mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Issue #7820: The parser tokenizer restores all bytes in the right if the BOM

Browse source 
check fails.

Fix an assertion in pydebug mode.
This commit is contained in:
Victor Stinner 2010-03-02 23:20:02 +00:00
parent 0e717addd8
commit d23d3930ff
3 changed files with 46 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Lib/test/test_pep263.py
View file

@ -30,6 +30,17 @@ class PEP263Test(unittest.TestCase):
self.assertEqual(d['a'], d['b']) self.assertEqual(d['a'], d['b'])
self.assertEqual(len(d['a']), len(d['b'])) self.assertEqual(len(d['a']), len(d['b']))
def test_issue7820(self):
# Ensure that check_bom() restores all bytes in the right order if
# check_bom() fails in pydebug mode: a buffer starts with the first
# byte of a valid BOM, but next bytes are different
# one byte in common with the UTF-16-LE BOM
self.assertRaises(SyntaxError, eval, '\xff\x20')
# two bytes in common with the UTF-8 BOM
self.assertRaises(SyntaxError, eval, '\xef\xbb\x20')
def test_main(): def test_main():
test_support.run_unittest(PEP263Test) test_support.run_unittest(PEP263Test)

3
Misc/NEWS
View file

@ -12,6 +12,9 @@ What's New in Python 2.7 alpha 4?
Core and Builtins Core and Builtins
----------------- -----------------
- Issue #7820: The parser tokenizer restores all bytes in the right if
the BOM check fails.
- Issue #7309: Fix unchecked attribute access when converting - Issue #7309: Fix unchecked attribute access when converting
UnicodeEncodeError, UnicodeDecodeError, and UnicodeTranslateError to UnicodeEncodeError, UnicodeDecodeError, and UnicodeTranslateError to
strings. strings.

54
Parser/tokenizer.c
View file

@ -312,47 +312,57 @@ check_bom(int get_char(struct tok_state *),
int set_readline(struct tok_state *, const char *), int set_readline(struct tok_state *, const char *),
struct tok_state *tok) struct tok_state *tok)
{ {
int ch = get_char(tok); int ch1, ch2, ch3;
ch1 = get_char(tok);
tok->decoding_state = 1; tok->decoding_state = 1;
if (ch == EOF) { if (ch1 == EOF) {
return 1; return 1;
} else if (ch == 0xEF) { } else if (ch1 == 0xEF) {
ch = get_char(tok); ch2 = get_char(tok);
if (ch != 0xBB) if (ch2 != 0xBB) {
goto NON_BOM; unget_char(ch2, tok);
ch = get_char(tok); unget_char(ch1, tok);
if (ch != 0xBF) return 1;
goto NON_BOM; }
ch3 = get_char(tok);
if (ch3 != 0xBF) {
unget_char(ch3, tok);
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
#if 0 #if 0
/* Disable support for UTF-16 BOMs until a decision /* Disable support for UTF-16 BOMs until a decision
is made whether this needs to be supported. */ is made whether this needs to be supported. */
} else if (ch == 0xFE) { } else if (ch1 == 0xFE) {
ch = get_char(tok); ch2 = get_char(tok);
if (ch != 0xFF) if (ch2 != 0xFF) {
goto NON_BOM; unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-be")) if (!set_readline(tok, "utf-16-be"))
return 0; return 0;
tok->decoding_state = -1; tok->decoding_state = -1;
} else if (ch == 0xFF) { } else if (ch1 == 0xFF) {
ch = get_char(tok); ch2 = get_char(tok);
if (ch != 0xFE) if (ch2 != 0xFE) {
goto NON_BOM; unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-le")) if (!set_readline(tok, "utf-16-le"))
return 0; return 0;
tok->decoding_state = -1; tok->decoding_state = -1;
#endif #endif
} else { } else {
unget_char(ch, tok); unget_char(ch1, tok);
return 1; return 1;
} }
if (tok->encoding != NULL) if (tok->encoding != NULL)
PyMem_FREE(tok->encoding); PyMem_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */ tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1; return 1;
NON_BOM:
/* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
unget_char(0xFF, tok); /* XXX this will cause a syntax error */
return 1;
} }
/* Read a line of text from TOK into S, using the stream in TOK. /* Read a line of text from TOK into S, using the stream in TOK.