mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

bpo-24214: Fixed the UTF-8 incremental decoder. (GH-12603)

Browse source 
The bug occurred when the encoded surrogate character is passed
to the incremental decoder in two chunks.
This commit is contained in:
Serhiy Storchaka 2019-03-30 08:23:38 +02:00 • committed by GitHub
parent 38f4e468d4
commit 7a465cb5ee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 14 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Lib/test/test_codecs.py
View file

@ -406,6 +406,15 @@ class ReadTest(MixInCheckStateHandling):
self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"), self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
before + backslashreplace + after) before + backslashreplace + after)
def test_incremental_surrogatepass(self):
# Test incremental decoder for surrogatepass handler:
# see issue #24214
data = '\uD901'.encode(self.encoding, 'surrogatepass')
for i in range(1, len(data)):
dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
self.assertEqual(dec.decode(data[:i]), '')
self.assertEqual(dec.decode(data[i:], True), '\uD901')
class UTF32Test(ReadTest, unittest.TestCase): class UTF32Test(ReadTest, unittest.TestCase):
encoding = "utf-32" encoding = "utf-32"

2
Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst Normal file
View file

@ -0,0 +1,2 @@
Fixed support of the surrogatepass error handler in the UTF-8 incremental
decoder.

3
Objects/unicodeobject.c
View file

@ -4883,6 +4883,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
case 2: case 2:
case 3: case 3:
case 4: case 4:
if (s == end || consumed) {
goto End;
}
errmsg = "invalid continuation byte"; errmsg = "invalid continuation byte";
startinpos = s - starts; startinpos = s - starts;
endinpos = startinpos + ch - 1; endinpos = startinpos + ch - 1;