diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f2a1ae3f790..15cd687530a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -330,7 +330,7 @@ class UTF32Test(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", # first byte of BOM read "", # second byte of BOM read @@ -352,6 +352,10 @@ class UTF32Test(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -386,7 +390,7 @@ class UTF32LETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "", @@ -404,6 +408,10 @@ class UTF32LETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -426,7 +434,7 @@ class UTF32BETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "", @@ -444,6 +452,10 @@ class UTF32BETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -494,7 +506,7 @@ class UTF16Test(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", # first byte of BOM read "", # second byte of BOM read => byteorder known @@ -506,6 +518,10 @@ class UTF16Test(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -543,7 +559,7 @@ class UTF16LETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "\x00", @@ -553,6 +569,10 @@ class UTF16LETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -582,7 +602,7 @@ class UTF16BETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "\x00", @@ -592,6 +612,10 @@ class UTF16BETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -621,7 +645,7 @@ class UTF8Test(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff\U00010000", [ "\x00", "\x00", @@ -634,6 +658,10 @@ class UTF8Test(ReadTest): "\x00\xff\u07ff\u0800", "\x00\xff\u07ff\u0800", "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff\U00010000", ] ) @@ -816,7 +844,7 @@ class UTF8SigTest(ReadTest): def test_partial(self): self.check_partial( - "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000", [ "", "", @@ -835,6 +863,10 @@ class UTF8SigTest(ReadTest): "\ufeff\x00\xff\u07ff\u0800", "\ufeff\x00\xff\u07ff\u0800", "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000", ] ) diff --git a/Misc/NEWS b/Misc/NEWS index b40d4c7f051..8a5d14d443d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ What's New in Python 3.3.1? Core and Builtins ----------------- +- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by + Amaury Forgeot d'Arc. + - Issue #16881: Fix Py_ARRAY_LENGTH macro for GCC < 3.1. - Issue #16856: Fix a segmentation fault from calling repr() on a dict with diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1522a16ba6e..b4fc0040b6e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5284,8 +5284,11 @@ PyUnicode_DecodeUTF16Stateful(const char *s, /* The remaining input chars are ignored if the callback chooses to skip the input */ case 1: + q -= 2; + if (consumed) + goto End; errmsg = "unexpected end of data"; - startinpos = ((const char *)q) - 2 - starts; + startinpos = ((const char *)q) - starts; endinpos = ((const char *)e) - starts; break; case 2: