mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-24 11:44:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Issue #25388: Fixed tokenizer crash when processing undecodable source code

Browse source 
with a null byte.
This commit is contained in:
Serhiy Storchaka 2015-11-14 15:10:35 +02:00
parent 806fb25405
commit 0d441119f5
3 changed files with 19 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Parser/tokenizer.c
View file

@ -187,7 +187,8 @@ error_ret(struct tok_state *tok) /* XXX */
tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
PyMem_FREE(tok->buf);
tok->buf = NULL;
tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
tok->done = E_DECODE;
return NULL; /* as if it were EOF */
}
@ -943,11 +944,6 @@ tok_nextc(struct tok_state *tok)
}
buflen = PyBytes_GET_SIZE(u);
buf = PyBytes_AS_STRING(u);
if (!buf) {
Py_DECREF(u);
tok->done = E_DECODE;
return EOF;
}
newtok = PyMem_MALLOC(buflen+1);
strcpy(newtok, buf);
Py_DECREF(u);
@ -989,7 +985,6 @@ tok_nextc(struct tok_state *tok)
if (tok->buf != NULL)
PyMem_FREE(tok->buf);
tok->buf = newtok;
tok->line_start = tok->buf;
tok->cur = tok->buf;
tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0');
@ -1012,7 +1007,8 @@ tok_nextc(struct tok_state *tok)
}
if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
tok) == NULL) {
tok->done = E_EOF;
if (!tok->decoding_erred)
tok->done = E_EOF;
done = 1;
}
else {
@ -1046,6 +1042,8 @@ tok_nextc(struct tok_state *tok)
return EOF;
}
tok->buf = newbuf;
tok->cur = tok->buf + cur;
tok->line_start = tok->cur;
tok->inp = tok->buf + curvalid;
tok->end = tok->buf + newsize;
tok->start = curstart < 0 ? NULL :