mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Issue #7820: The parser tokenizer restores all bytes in the right if the BOM

Browse source 
check fails.

Fix an assertion in pydebug mode.
This commit is contained in:
Victor Stinner 2010-03-02 23:20:02 +00:00
parent 0e717addd8
commit d23d3930ff
3 changed files with 46 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

54
Parser/tokenizer.c
View file

@ -312,47 +312,57 @@ check_bom(int get_char(struct tok_state *),
int set_readline(struct tok_state *, const char *),
struct tok_state *tok)
{
int ch = get_char(tok);
int ch1, ch2, ch3;
ch1 = get_char(tok);
tok->decoding_state = 1;
if (ch == EOF) {
if (ch1 == EOF) {
return 1;
} else if (ch == 0xEF) {
ch = get_char(tok);
if (ch != 0xBB)
goto NON_BOM;
ch = get_char(tok);
if (ch != 0xBF)
goto NON_BOM;
} else if (ch1 == 0xEF) {
ch2 = get_char(tok);
if (ch2 != 0xBB) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
ch3 = get_char(tok);
if (ch3 != 0xBF) {
unget_char(ch3, tok);
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
#if 0
/* Disable support for UTF-16 BOMs until a decision
is made whether this needs to be supported. */
} else if (ch == 0xFE) {
ch = get_char(tok);
if (ch != 0xFF)
goto NON_BOM;
} else if (ch1 == 0xFE) {
ch2 = get_char(tok);
if (ch2 != 0xFF) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-be"))
return 0;
tok->decoding_state = -1;
} else if (ch == 0xFF) {
ch = get_char(tok);
if (ch != 0xFE)
goto NON_BOM;
} else if (ch1 == 0xFF) {
ch2 = get_char(tok);
if (ch2 != 0xFE) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-le"))
return 0;
tok->decoding_state = -1;
#endif
} else {
unget_char(ch, tok);
unget_char(ch1, tok);
return 1;
}
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1;
NON_BOM:
/* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
unget_char(0xFF, tok); /* XXX this will cause a syntax error */
return 1;
}
/* Read a line of text from TOK into S, using the stream in TOK.