mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 00:23:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

merge 3.4 (#24022)

Browse source
This commit is contained in:
Benjamin Peterson 2015-04-21 12:07:06 -04:00
commit 273a720f87
3 changed files with 18 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Parser/tokenizer.c
View file

@ -1307,6 +1307,8 @@ verify_identifier(struct tok_state *tok)
{
PyObject *s;
int result;
if (tok->decoding_erred)
return 0;
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
if (s == NULL || PyUnicode_READY(s) == -1) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
@ -1475,11 +1477,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
c = tok_nextc(tok);
}
tok_backup(tok, c);
if (nonascii &&
!verify_identifier(tok)) {
tok->done = E_IDENTIFIER;
if (nonascii && !verify_identifier(tok))
return ERRORTOKEN;
}
*p_start = tok->start;
*p_end = tok->cur;
return NAME;