mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 00:48:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Merged revisions 79723 via svnmerge from

Browse source 
svn+ssh://pythondev@svn.python.org/python/trunk

........
  r79723 | benjamin.peterson | 2010-04-03 17:48:51 -0500 (Sat, 03 Apr 2010) | 1 line

  ensure that the locale does not affect the tokenization of identifiers
........
This commit is contained in:
Benjamin Peterson 2010-04-03 22:55:48 +00:00
parent ace7108f86
commit 8daa49ee34
2 changed files with 13 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Parser/tokenizer.c
View file

@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok)
#else /* PGEN */
/* Ensure that the locale does not interfere with tokenization. */
static int
ascii_isalnum(int c)
{
return (('a' <= c && c <= 'z') ||
('A' <= c && c <= 'Z') ||
('0' <= c && c <= '9'));
}
static char *
error_ret(struct tok_state *tok) /* XXX */
{
@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
} while (t[0] == '\x20' || t[0] == '\t');
begin = t;
while (isalnum(Py_CHARMASK(t[0])) ||
while (ascii_isalnum(Py_CHARMASK(t[0])) ||
t[0] == '-' || t[0] == '_' || t[0] == '.')
t++;