mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Issue #25388: Fixed tokenizer crash when processing undecodable source code

Browse source 
with a null byte.
This commit is contained in:
Serhiy Storchaka 2015-11-14 15:12:04 +02:00
commit 0304729ec4
3 changed files with 19 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Lib/test/test_compile.py
View file

@ -516,6 +516,16 @@ if 1:
res = script_helper.run_python_until_end(fn)[0] res = script_helper.run_python_until_end(fn)[0]
self.assertIn(b"Non-UTF-8", res.err) self.assertIn(b"Non-UTF-8", res.err)
def test_yet_more_evil_still_undecodable(self):
# Issue #25388
src = b"#\x00\n#\xfd\n"
with tempfile.TemporaryDirectory() as tmpd:
fn = os.path.join(tmpd, "bad.py")
with open(fn, "wb") as fp:
fp.write(src)
res = script_helper.run_python_until_end(fn)[0]
self.assertIn(b"Non-UTF-8", res.err)
@support.cpython_only @support.cpython_only
def test_compiler_recursion_limit(self): def test_compiler_recursion_limit(self):
# Expected limit is sys.getrecursionlimit() * the scaling factor # Expected limit is sys.getrecursionlimit() * the scaling factor

3
Misc/NEWS
View file

@ -10,6 +10,9 @@ Release date: XXXX-XX-XX
Core and Builtins Core and Builtins
----------------- -----------------
- Issue #25388: Fixed tokenizer crash when processing undecodable source code
with a null byte.
- Issue #25462: The hash of the key now is calculated only once in most - Issue #25462: The hash of the key now is calculated only once in most
operations in C implementation of OrderedDict. operations in C implementation of OrderedDict.

14
Parser/tokenizer.c
View file

@ -196,7 +196,8 @@ error_ret(struct tok_state *tok) /* XXX */
tok->decoding_erred = 1; tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
PyMem_FREE(tok->buf); PyMem_FREE(tok->buf);
tok->buf = NULL; tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
tok->done = E_DECODE;
return NULL; /* as if it were EOF */ return NULL; /* as if it were EOF */
} }
@ -952,11 +953,6 @@ tok_nextc(struct tok_state *tok)
} }
buflen = PyBytes_GET_SIZE(u); buflen = PyBytes_GET_SIZE(u);
buf = PyBytes_AS_STRING(u); buf = PyBytes_AS_STRING(u);
if (!buf) {
Py_DECREF(u);
tok->done = E_DECODE;
return EOF;
}
newtok = PyMem_MALLOC(buflen+1); newtok = PyMem_MALLOC(buflen+1);
strcpy(newtok, buf); strcpy(newtok, buf);
Py_DECREF(u); Py_DECREF(u);
@ -998,7 +994,6 @@ tok_nextc(struct tok_state *tok)
if (tok->buf != NULL) if (tok->buf != NULL)
PyMem_FREE(tok->buf); PyMem_FREE(tok->buf);
tok->buf = newtok; tok->buf = newtok;
tok->line_start = tok->buf;
tok->cur = tok->buf; tok->cur = tok->buf;
tok->line_start = tok->buf; tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0'); tok->inp = strchr(tok->buf, '\0');
@ -1021,7 +1016,8 @@ tok_nextc(struct tok_state *tok)
} }
if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf), if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
tok) == NULL) { tok) == NULL) {
tok->done = E_EOF; if (!tok->decoding_erred)
tok->done = E_EOF;
done = 1; done = 1;
} }
else { else {
@ -1055,6 +1051,8 @@ tok_nextc(struct tok_state *tok)
return EOF; return EOF;
} }
tok->buf = newbuf; tok->buf = newbuf;
tok->cur = tok->buf + cur;
tok->line_start = tok->cur;
tok->inp = tok->buf + curvalid; tok->inp = tok->buf + curvalid;
tok->end = tok->buf + newsize; tok->end = tok->buf + newsize;
tok->start = curstart < 0 ? NULL : tok->start = curstart < 0 ? NULL :