mirror of
https://github.com/python/cpython.git
synced 2025-08-03 16:39:00 +00:00
Update PyUnicode_DecodeUTF8 from RFC 2279 to RFC 3629.
1) #8271: when a byte sequence is invalid, only the start byte and all the valid continuation bytes are now replaced by U+FFFD, instead of replacing the number of bytes specified by the start byte. See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (pages 94-95); 2) 5- and 6-bytes-long UTF-8 sequences are now considered invalid (no changes in behavior); 3) Change the error messages "unexpected code byte" to "invalid start byte" and "invalid data" to "invalid continuation byte"; 4) Add an extensive set of tests in test_unicode; 5) Fix test_codeccallbacks because it was failing after this change.
This commit is contained in:
Ezio Melotti
parent
ead3c83ea9
commit
57221d02ba
4 changed files with 230 additions and 67 deletions
|
|
@ -942,6 +942,159 @@ class UnicodeTest(string_tests.CommonTest,
|
|||
# * strict decoding testing for all of the
|
||||
# UTF8_ERROR cases in PyUnicode_DecodeUTF8
|
||||
|
||||
def test_utf8_decode_valid_sequences(self):
|
||||
sequences = [
|
||||
# single byte
|
||||
(b'\x00', '\x00'), (b'a', 'a'), (b'\x7f', '\x7f'),
|
||||
# 2 bytes
|
||||
(b'\xc2\x80', '\x80'), (b'\xdf\xbf', '\u07ff'),
|
||||
# 3 bytes
|
||||
(b'\xe0\xa0\x80', '\u0800'), (b'\xed\x9f\xbf', '\ud7ff'),
|
||||
(b'\xee\x80\x80', '\uE000'), (b'\xef\xbf\xbf', '\uffff'),
|
||||
# 4 bytes
|
||||
(b'\xF0\x90\x80\x80', '\U00010000'),
|
||||
(b'\xf4\x8f\xbf\xbf', '\U0010FFFF')
|
||||
]
|
||||
for seq, res in sequences:
|
||||
self.assertEqual(seq.decode('utf-8'), res)
|
||||
|
||||
|
||||
def test_utf8_decode_invalid_sequences(self):
|
||||
# continuation bytes in a sequence of 2, 3, or 4 bytes
|
||||
continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)]
|
||||
# start bytes of a 2-byte sequence equivalent to codepoints < 0x7F
|
||||
invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)]
|
||||
# start bytes of a 4-byte sequence equivalent to codepoints > 0x10FFFF
|
||||
invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)]
|
||||
invalid_start_bytes = (
|
||||
continuation_bytes + invalid_2B_seq_start_bytes +
|
||||
invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)]
|
||||
)
|
||||
|
||||
for byte in invalid_start_bytes:
|
||||
self.assertRaises(UnicodeDecodeError, byte.decode, 'utf-8')
|
||||
|
||||
for sb in invalid_2B_seq_start_bytes:
|
||||
for cb in continuation_bytes:
|
||||
self.assertRaises(UnicodeDecodeError, (sb+cb).decode, 'utf-8')
|
||||
|
||||
for sb in invalid_4B_seq_start_bytes:
|
||||
for cb1 in continuation_bytes[:3]:
|
||||
for cb3 in continuation_bytes[:3]:
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(sb+cb1+b'\x80'+cb3).decode, 'utf-8')
|
||||
|
||||
for cb in [bytes([x]) for x in range(0x80, 0xA0)]:
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xE0'+cb+b'\x80').decode, 'utf-8')
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xE0'+cb+b'\xBF').decode, 'utf-8')
|
||||
# surrogates
|
||||
for cb in [bytes([x]) for x in range(0xA0, 0xC0)]:
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xED'+cb+b'\x80').decode, 'utf-8')
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xED'+cb+b'\xBF').decode, 'utf-8')
|
||||
for cb in [bytes([x]) for x in range(0x80, 0x90)]:
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xF0'+cb+b'\x80\x80').decode, 'utf-8')
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xF0'+cb+b'\xBF\xBF').decode, 'utf-8')
|
||||
for cb in [bytes([x]) for x in range(0x90, 0xC0)]:
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xF4'+cb+b'\x80\x80').decode, 'utf-8')
|
||||
self.assertRaises(UnicodeDecodeError,
|
||||
(b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8')
|
||||
|
||||
def test_issue8271(self):
|
||||
# Issue #8271: during the decoding of an invalid UTF-8 byte sequence,
|
||||
# only the start byte and the continuation byte(s) are now considered
|
||||
# invalid, instead of the number of bytes specified by the start byte.
|
||||
# See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95,
|
||||
# table 3-8, Row 2) for more information about the algorithm used.
|
||||
FFFD = '\ufffd'
|
||||
sequences = [
|
||||
# invalid start bytes
|
||||
(b'\x80', FFFD), # continuation byte
|
||||
(b'\x80\x80', FFFD*2), # 2 continuation bytes
|
||||
(b'\xc0', FFFD),
|
||||
(b'\xc0\xc0', FFFD*2),
|
||||
(b'\xc1', FFFD),
|
||||
(b'\xc1\xc0', FFFD*2),
|
||||
(b'\xc0\xc1', FFFD*2),
|
||||
# with start byte of a 2-byte sequence
|
||||
(b'\xc2', FFFD), # only the start byte
|
||||
(b'\xc2\xc2', FFFD*2), # 2 start bytes
|
||||
(b'\xc2\xc2\xc2', FFFD*3), # 2 start bytes
|
||||
(b'\xc2\x41', FFFD+'A'), # invalid continuation byte
|
||||
# with start byte of a 3-byte sequence
|
||||
(b'\xe1', FFFD), # only the start byte
|
||||
(b'\xe1\xe1', FFFD*2), # 2 start bytes
|
||||
(b'\xe1\xe1\xe1', FFFD*3), # 3 start bytes
|
||||
(b'\xe1\xe1\xe1\xe1', FFFD*4), # 4 start bytes
|
||||
(b'\xe1\x80', FFFD), # only 1 continuation byte
|
||||
(b'\xe1\x41', FFFD+'A'), # invalid continuation byte
|
||||
(b'\xe1\x41\x80', FFFD+'A'+FFFD), # invalid cb followed by valid cb
|
||||
(b'\xe1\x41\x41', FFFD+'AA'), # 2 invalid continuation bytes
|
||||
(b'\xe1\x80\x41', FFFD+'A'), # only 1 valid continuation byte
|
||||
(b'\xe1\x80\xe1\x41', FFFD*2+'A'), # 1 valid and the other invalid
|
||||
(b'\xe1\x41\xe1\x80', FFFD+'A'+FFFD), # 1 invalid and the other valid
|
||||
# with start byte of a 4-byte sequence
|
||||
(b'\xf1', FFFD), # only the start byte
|
||||
(b'\xf1\xf1', FFFD*2), # 2 start bytes
|
||||
(b'\xf1\xf1\xf1', FFFD*3), # 3 start bytes
|
||||
(b'\xf1\xf1\xf1\xf1', FFFD*4), # 4 start bytes
|
||||
(b'\xf1\xf1\xf1\xf1\xf1', FFFD*5), # 5 start bytes
|
||||
(b'\xf1\x80', FFFD), # only 1 continuation bytes
|
||||
(b'\xf1\x80\x80', FFFD), # only 2 continuation bytes
|
||||
(b'\xf1\x80\x41', FFFD+'A'), # 1 valid cb and 1 invalid
|
||||
(b'\xf1\x80\x41\x41', FFFD+'AA'), # 1 valid cb and 1 invalid
|
||||
(b'\xf1\x80\x80\x41', FFFD+'A'), # 2 valid cb and 1 invalid
|
||||
(b'\xf1\x41\x80', FFFD+'A'+FFFD), # 1 invalid cv and 1 valid
|
||||
(b'\xf1\x41\x80\x80', FFFD+'A'+FFFD*2), # 1 invalid cb and 2 invalid
|
||||
(b'\xf1\x41\x80\x41', FFFD+'A'+FFFD+'A'), # 2 invalid cb and 1 invalid
|
||||
(b'\xf1\x41\x41\x80', FFFD+'AA'+FFFD), # 1 valid cb and 1 invalid
|
||||
(b'\xf1\x41\xf1\x80', FFFD+'A'+FFFD),
|
||||
(b'\xf1\x41\x80\xf1', FFFD+'A'+FFFD*2),
|
||||
(b'\xf1\xf1\x80\x41', FFFD*2+'A'),
|
||||
(b'\xf1\x41\xf1\xf1', FFFD+'A'+FFFD*2),
|
||||
# with invalid start byte of a 4-byte sequence (rfc2279)
|
||||
(b'\xf5', FFFD), # only the start byte
|
||||
(b'\xf5\xf5', FFFD*2), # 2 start bytes
|
||||
(b'\xf5\x80', FFFD*2), # only 1 continuation byte
|
||||
(b'\xf5\x80\x80', FFFD*3), # only 2 continuation byte
|
||||
(b'\xf5\x80\x80\x80', FFFD*4), # 3 continuation bytes
|
||||
(b'\xf5\x80\x41', FFFD*2+'A'), # 1 valid cb and 1 invalid
|
||||
(b'\xf5\x80\x41\xf5', FFFD*2+'A'+FFFD),
|
||||
(b'\xf5\x41\x80\x80\x41', FFFD+'A'+FFFD*2+'A'),
|
||||
# with invalid start byte of a 5-byte sequence (rfc2279)
|
||||
(b'\xf8', FFFD), # only the start byte
|
||||
(b'\xf8\xf8', FFFD*2), # 2 start bytes
|
||||
(b'\xf8\x80', FFFD*2), # only one continuation byte
|
||||
(b'\xf8\x80\x41', FFFD*2 + 'A'), # 1 valid cb and 1 invalid
|
||||
(b'\xf8\x80\x80\x80\x80', FFFD*5), # invalid 5 bytes seq with 5 bytes
|
||||
# with invalid start byte of a 6-byte sequence (rfc2279)
|
||||
(b'\xfc', FFFD), # only the start byte
|
||||
(b'\xfc\xfc', FFFD*2), # 2 start bytes
|
||||
(b'\xfc\x80\x80', FFFD*3), # only 2 continuation bytes
|
||||
(b'\xfc\x80\x80\x80\x80\x80', FFFD*6), # 6 continuation bytes
|
||||
# invalid start byte
|
||||
(b'\xfe', FFFD),
|
||||
(b'\xfe\x80\x80', FFFD*3),
|
||||
# other sequences
|
||||
(b'\xf1\x80\x41\x42\x43', '\ufffd\x41\x42\x43'),
|
||||
(b'\xf1\x80\xff\x42\x43', '\ufffd\ufffd\x42\x43'),
|
||||
(b'\xf1\x80\xc2\x81\x43', '\ufffd\x81\x43'),
|
||||
(b'\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64',
|
||||
'\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'),
|
||||
]
|
||||
for n, (seq, res) in enumerate(sequences):
|
||||
self.assertRaises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict')
|
||||
self.assertEqual(seq.decode('utf-8', 'replace'), res)
|
||||
self.assertEqual((seq+b'b').decode('utf-8', 'replace'), res+'b')
|
||||
self.assertEqual(seq.decode('utf-8', 'ignore'),
|
||||
res.replace('\uFFFD', ''))
|
||||
|
||||
def test_codecs_idna(self):
|
||||
# Test whether trailing dot is preserved
|
||||
self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue