mirror of
https://github.com/python/cpython.git
synced 2025-08-31 05:58:33 +00:00
Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
Amaury Forgeot d'Arc. Added tests for partial decoding of non-BMP characters.
This commit is contained in:
Serhiy Storchaka
parent
dec798eb46
commit
48e188e573
3 changed files with 47 additions and 9 deletions
|
|
@ -313,7 +313,7 @@ class UTF32Test(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"", # first byte of BOM read
|
"", # first byte of BOM read
|
||||||
"", # second byte of BOM read
|
"", # second byte of BOM read
|
||||||
|
|
@ -335,6 +335,10 @@ class UTF32Test(ReadTest):
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -369,7 +373,7 @@ class UTF32LETest(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"",
|
"",
|
||||||
"",
|
"",
|
||||||
|
|
@ -387,6 +391,10 @@ class UTF32LETest(ReadTest):
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -409,7 +417,7 @@ class UTF32BETest(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"",
|
"",
|
||||||
"",
|
"",
|
||||||
|
|
@ -427,6 +435,10 @@ class UTF32BETest(ReadTest):
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -477,7 +489,7 @@ class UTF16Test(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"", # first byte of BOM read
|
"", # first byte of BOM read
|
||||||
"", # second byte of BOM read => byteorder known
|
"", # second byte of BOM read => byteorder known
|
||||||
|
|
@ -489,6 +501,10 @@ class UTF16Test(ReadTest):
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -526,7 +542,7 @@ class UTF16LETest(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"",
|
"",
|
||||||
"\x00",
|
"\x00",
|
||||||
|
|
@ -536,6 +552,10 @@ class UTF16LETest(ReadTest):
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -565,7 +585,7 @@ class UTF16BETest(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"",
|
"",
|
||||||
"\x00",
|
"\x00",
|
||||||
|
|
@ -575,6 +595,10 @@ class UTF16BETest(ReadTest):
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100",
|
"\x00\xff\u0100",
|
||||||
"\x00\xff\u0100\uffff",
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff",
|
||||||
|
"\x00\xff\u0100\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -604,7 +628,7 @@ class UTF8Test(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\x00\xff\u07ff\u0800\uffff",
|
"\x00\xff\u07ff\u0800\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"\x00",
|
"\x00",
|
||||||
"\x00",
|
"\x00",
|
||||||
|
|
@ -617,6 +641,10 @@ class UTF8Test(ReadTest):
|
||||||
"\x00\xff\u07ff\u0800",
|
"\x00\xff\u07ff\u0800",
|
||||||
"\x00\xff\u07ff\u0800",
|
"\x00\xff\u07ff\u0800",
|
||||||
"\x00\xff\u07ff\u0800\uffff",
|
"\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\x00\xff\u07ff\u0800\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -694,7 +722,7 @@ class UTF8SigTest(ReadTest):
|
||||||
|
|
||||||
def test_partial(self):
|
def test_partial(self):
|
||||||
self.check_partial(
|
self.check_partial(
|
||||||
"\ufeff\x00\xff\u07ff\u0800\uffff",
|
"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
|
||||||
[
|
[
|
||||||
"",
|
"",
|
||||||
"",
|
"",
|
||||||
|
|
@ -713,6 +741,10 @@ class UTF8SigTest(ReadTest):
|
||||||
"\ufeff\x00\xff\u07ff\u0800",
|
"\ufeff\x00\xff\u07ff\u0800",
|
||||||
"\ufeff\x00\xff\u07ff\u0800",
|
"\ufeff\x00\xff\u07ff\u0800",
|
||||||
"\ufeff\x00\xff\u07ff\u0800\uffff",
|
"\ufeff\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\ufeff\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\ufeff\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\ufeff\x00\xff\u07ff\u0800\uffff",
|
||||||
|
"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,9 @@ What's New in Python 3.2.4
|
||||||
Core and Builtins
|
Core and Builtins
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
|
- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
|
||||||
|
Amaury Forgeot d'Arc.
|
||||||
|
|
||||||
- Issue #16367: Fix FileIO.readall() on Windows for files larger than 2 GB.
|
- Issue #16367: Fix FileIO.readall() on Windows for files larger than 2 GB.
|
||||||
|
|
||||||
- Issue #16455: On FreeBSD and Solaris, if the locale is C, the
|
- Issue #16455: On FreeBSD and Solaris, if the locale is C, the
|
||||||
|
|
|
||||||
|
|
@ -3573,8 +3573,11 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
|
||||||
|
|
||||||
/* UTF-16 code pair: */
|
/* UTF-16 code pair: */
|
||||||
if (e - q < 2) {
|
if (e - q < 2) {
|
||||||
|
q -= 2;
|
||||||
|
if (consumed)
|
||||||
|
break;
|
||||||
errmsg = "unexpected end of data";
|
errmsg = "unexpected end of data";
|
||||||
startinpos = (((const char *)q) - 2) - starts;
|
startinpos = ((const char *)q) - starts;
|
||||||
endinpos = ((const char *)e) - starts;
|
endinpos = ((const char *)e) - starts;
|
||||||
goto utf16Error;
|
goto utf16Error;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue