mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 00:23:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

bpo-46054: Fix parsing error when parsing non-utf8 characters in source files (GH-30068) (GH-30069)

Browse source 
(cherry picked from commit 4325a766f5)

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
This commit is contained in:
Miss Islington (bot) 2021-12-12 08:52:49 -08:00 • committed by GitHub
parent 438817fdd5
commit 94483f1e3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 19 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Lib/test/test_exceptions.py
View file

@ -2368,6 +2368,18 @@ class SyntaxErrorTests(unittest.TestCase):
finally:
unlink(TESTFN)
def test_non_utf8(self):
# Check non utf-8 characters
try:
with open(TESTFN, 'bw') as testfile:
testfile.write(b"\x89")
rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN)
err = err.decode('utf-8').splitlines()
self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1])
finally:
unlink(TESTFN)
def test_attributes_new_constructor(self):
args = ("bad.py", 1, 2, "abcdefg", 1, 100)
the_exception = SyntaxError("bad bad", args)

2
Misc/NEWS.d/next/Core and Builtins/2021-12-12-05-30-21.bpo-46054.2P-foG.rst Normal file
View file

@ -0,0 +1,2 @@
Fix parser error when parsing non-utf8 characters in source files. Patch by
Pablo Galindo.

9
Parser/tokenizer.c
View file

@ -818,10 +818,10 @@ tok_readline_raw(struct tok_state *tok)
tok_concatenate_interactive_new_line(tok, line) == -1) {
return 0;
}
if (*tok->inp == '\0') {
tok->inp = strchr(tok->inp, '\0');
if (tok->inp == tok->buf) {
return 0;
}
tok->inp = strchr(tok->inp, '\0');
} while (tok->inp[-1] != '\n');
return 1;
}
@ -983,13 +983,10 @@ tok_underflow_file(struct tok_state *tok) {
}
/* The default encoding is UTF-8, so make sure we don't have any
non-UTF-8 sequences in it. */
if (!tok->encoding
&& (tok->decoding_state != STATE_NORMAL || tok->lineno >= 2)) {
if (!ensure_utf8(tok->cur, tok)) {
if (!tok->encoding && !ensure_utf8(tok->cur, tok)) {
error_ret(tok);
return 0;
}
}
assert(tok->done == E_OK);
return tok->done == E_OK;
}