mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-14 20:56:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Added PEP 3127 support to tokenize (with tests); added PEP 3127 to NEWS.

Browse source
This commit is contained in:
Eric Smith 2008-03-17 19:43:40 +00:00
parent 6f778cfb5b
commit 0aed07ad80
3 changed files with 17 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Lib/tokenize.py
View file

@ -51,9 +51,10 @@ Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Name = r'[a-zA-Z_]\w*'
Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
Octnumber = r'0[0-7]*[lL]?'
Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
Binnumber = r'0[bB][01]+[lL]?'
Decnumber = r'[1-9]\d*[lL]?'
Intnumber = group(Hexnumber, Octnumber, Decnumber)
Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Exponent = r'[eE][-+]?\d+'
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Expfloat = r'\d+' + Exponent