mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Patch 1420 by Ron Adam.

Browse source 
This adds support for bytes literals (b'...') to tokenize.py, and
removes support for unicode literals (u'...').
This commit is contained in:
Guido van Rossum 2007-11-12 17:40:10 +00:00
parent 1607278c26
commit 4fe72f9b03
4 changed files with 58 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

38
Lib/test/output/test_tokenize
View file

@ -342,59 +342,59 @@ test_tokenize
112,3-112,4: NEWLINE '\n' 112,3-112,4: NEWLINE '\n'
113,0-113,1: NAME 'x' 113,0-113,1: NAME 'x'
113,2-113,3: OP '=' 113,2-113,3: OP '='
113,4-113,10: STRING "u'abc'" 113,4-113,10: STRING "b'abc'"
113,11-113,12: OP '+' 113,11-113,12: OP '+'
113,13-113,19: STRING "U'ABC'" 113,13-113,19: STRING "B'ABC'"
113,19-113,20: NEWLINE '\n' 113,19-113,20: NEWLINE '\n'
114,0-114,1: NAME 'y' 114,0-114,1: NAME 'y'
114,2-114,3: OP '=' 114,2-114,3: OP '='
114,4-114,10: STRING 'u"abc"' 114,4-114,10: STRING 'b"abc"'
114,11-114,12: OP '+' 114,11-114,12: OP '+'
114,13-114,19: STRING 'U"ABC"' 114,13-114,19: STRING 'B"ABC"'
114,19-114,20: NEWLINE '\n' 114,19-114,20: NEWLINE '\n'
115,0-115,1: NAME 'x' 115,0-115,1: NAME 'x'
115,2-115,3: OP '=' 115,2-115,3: OP '='
115,4-115,11: STRING "ur'abc'" 115,4-115,11: STRING "br'abc'"
115,12-115,13: OP '+' 115,12-115,13: OP '+'
115,14-115,21: STRING "Ur'ABC'" 115,14-115,21: STRING "Br'ABC'"
115,22-115,23: OP '+' 115,22-115,23: OP '+'
115,24-115,31: STRING "uR'ABC'" 115,24-115,31: STRING "bR'ABC'"
115,32-115,33: OP '+' 115,32-115,33: OP '+'
115,34-115,41: STRING "UR'ABC'" 115,34-115,41: STRING "BR'ABC'"
115,41-115,42: NEWLINE '\n' 115,41-115,42: NEWLINE '\n'
116,0-116,1: NAME 'y' 116,0-116,1: NAME 'y'
116,2-116,3: OP '=' 116,2-116,3: OP '='
116,4-116,11: STRING 'ur"abc"' 116,4-116,11: STRING 'br"abc"'
116,12-116,13: OP '+' 116,12-116,13: OP '+'
116,14-116,21: STRING 'Ur"ABC"' 116,14-116,21: STRING 'Br"ABC"'
116,22-116,23: OP '+' 116,22-116,23: OP '+'
116,24-116,31: STRING 'uR"ABC"' 116,24-116,31: STRING 'bR"ABC"'
116,32-116,33: OP '+' 116,32-116,33: OP '+'
116,34-116,41: STRING 'UR"ABC"' 116,34-116,41: STRING 'BR"ABC"'
116,41-116,42: NEWLINE '\n' 116,41-116,42: NEWLINE '\n'
117,0-117,1: NAME 'x' 117,0-117,1: NAME 'x'
117,2-117,3: OP '=' 117,2-117,3: OP '='
117,4-117,10: STRING "ur'\\\\'" 117,4-117,10: STRING "br'\\\\'"
117,11-117,12: OP '+' 117,11-117,12: OP '+'
117,13-117,19: STRING "UR'\\\\'" 117,13-117,19: STRING "BR'\\\\'"
117,19-117,20: NEWLINE '\n' 117,19-117,20: NEWLINE '\n'
118,0-118,1: NAME 'x' 118,0-118,1: NAME 'x'
118,2-118,3: OP '=' 118,2-118,3: OP '='
118,4-118,10: STRING "ur'\\''" 118,4-118,10: STRING "br'\\''"
118,11-118,12: OP '+' 118,11-118,12: OP '+'
118,13-118,15: STRING "''" 118,13-118,15: STRING "''"
118,15-118,16: NEWLINE '\n' 118,15-118,16: NEWLINE '\n'
119,0-119,1: NAME 'y' 119,0-119,1: NAME 'y'
119,2-119,3: OP '=' 119,2-119,3: OP '='
119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''" 119,4-121,6: STRING "br'''\nfoo bar \\\\\nbaz'''"
121,7-121,8: OP '+' 121,7-121,8: OP '+'
121,9-122,6: STRING "UR'''\nfoo'''" 121,9-122,6: STRING "BR'''\nfoo'''"
122,6-122,7: NEWLINE '\n' 122,6-122,7: NEWLINE '\n'
123,0-123,1: NAME 'y' 123,0-123,1: NAME 'y'
123,2-123,3: OP '=' 123,2-123,3: OP '='
123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""' 123,4-125,3: STRING 'Br"""foo\nbar \\\\ baz\n"""'
125,4-125,5: OP '+' 125,4-125,5: OP '+'
125,6-126,3: STRING "uR'''spam\n'''" 125,6-126,3: STRING "bR'''spam\n'''"
126,3-126,4: NEWLINE '\n' 126,3-126,4: NEWLINE '\n'
127,0-127,1: NL '\n' 127,0-127,1: NL '\n'
128,0-128,13: COMMENT '# Indentation' 128,0-128,13: COMMENT '# Indentation'

13
Lib/test/test_tokenize.py
View file

@ -183,6 +183,13 @@ def test_main():
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
# Validate the tokenize_tests.txt file.
# This makes sure it compiles, and displays any errors in it.
f = open(findfile('tokenize_tests.txt'))
sf = f.read()
f.close()
cf = compile(sf, 'tokenize_tests.txt', 'exec')
# This displays the tokenization of tokenize_tests.py to stdout, and # This displays the tokenization of tokenize_tests.py to stdout, and
# regrtest.py checks that this equals the expected output (in the # regrtest.py checks that this equals the expected output (in the
# test/output/ directory). # test/output/ directory).
@ -190,10 +197,12 @@ def test_main():
tokenize(f.readline) tokenize(f.readline)
f.close() f.close()
# Now run test_roundtrip() over tokenize_test.py too, and over all # Now run test_roundtrip() over test_tokenize.py too, and over all
# (if the "compiler" resource is enabled) or a small random sample (if # (if the "compiler" resource is enabled) or a small random sample (if
# "compiler" is not enabled) of the test*.py files. # "compiler" is not enabled) of the test*.py files.
f = findfile('tokenize_tests.txt') f = findfile('test_tokenize.py')
if verbose:
print(' round trip: ', f, file=sys.__stdout__)
test_roundtrip(f) test_roundtrip(f)
testdir = os.path.dirname(f) or os.curdir testdir = os.path.dirname(f) or os.curdir

20
Lib/test/tokenize_tests.txt
View file

@ -110,19 +110,19 @@ y = r"""foo
bar \\ baz bar \\ baz
""" + R'''spam """ + R'''spam
''' '''
x = u'abc' + U'ABC' x = b'abc' + B'ABC'
y = u"abc" + U"ABC" y = b"abc" + B"ABC"
x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC' x = br'abc' + Br'ABC' + bR'ABC' + BR'ABC'
y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC" y = br"abc" + Br"ABC" + bR"ABC" + BR"ABC"
x = ur'\\' + UR'\\' x = br'\\' + BR'\\'
x = ur'\'' + '' x = br'\'' + ''
y = ur''' y = br'''
foo bar \\ foo bar \\
baz''' + UR''' baz''' + BR'''
foo''' foo'''
y = Ur"""foo y = Br"""foo
bar \\ baz bar \\ baz
""" + uR'''spam """ + bR'''spam
''' '''
# Indentation # Indentation

36
Lib/tokenize.py
View file

@ -69,10 +69,10 @@ Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
# Tail end of """ string. # Tail end of """ string.
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""') Triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""')
# Single-line ' or " string. # Single-line ' or " string.
String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", String = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
# Because of leftmost-then-longest match semantics, be sure to put the # Because of leftmost-then-longest match semantics, be sure to put the
# longest operators first (e.g., if = came before ==, == would get # longest operators first (e.g., if = came before ==, == would get
@ -90,9 +90,9 @@ PlainToken = group(Number, Funny, String, Name)
Token = Ignore + PlainToken Token = Ignore + PlainToken
# First (or only) line of ' or " string. # First (or only) line of ' or " string.
ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
group("'", r'\\\r?\n'), group("'", r'\\\r?\n'),
r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n')) group('"', r'\\\r?\n'))
PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoExtras = group(r'\\\r?\n', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
@ -102,28 +102,28 @@ tokenprog, pseudoprog, single3prog, double3prog = map(
endprogs = {"'": re.compile(Single), '"': re.compile(Double), endprogs = {"'": re.compile(Single), '"': re.compile(Double),
"'''": single3prog, '"""': double3prog, "'''": single3prog, '"""': double3prog,
"r'''": single3prog, 'r"""': double3prog, "r'''": single3prog, 'r"""': double3prog,
"u'''": single3prog, 'u"""': double3prog, "b'''": single3prog, 'b"""': double3prog,
"ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog,
"R'''": single3prog, 'R"""': double3prog, "R'''": single3prog, 'R"""': double3prog,
"U'''": single3prog, 'U"""': double3prog, "B'''": single3prog, 'B"""': double3prog,
"uR'''": single3prog, 'uR"""': double3prog, "bR'''": single3prog, 'bR"""': double3prog,
"Ur'''": single3prog, 'Ur"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog,
"UR'''": single3prog, 'UR"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog,
'r': None, 'R': None, 'u': None, 'U': None} 'r': None, 'R': None, 'b': None, 'B': None}
triple_quoted = {} triple_quoted = {}
for t in ("'''", '"""', for t in ("'''", '"""',
"r'''", 'r"""', "R'''", 'R"""', "r'''", 'r"""', "R'''", 'R"""',
"u'''", 'u"""', "U'''", 'U"""', "b'''", 'b"""', "B'''", 'B"""',
"ur'''", 'ur"""', "Ur'''", 'Ur"""', "br'''", 'br"""', "Br'''", 'Br"""',
"uR'''", 'uR"""', "UR'''", 'UR"""'): "bR'''", 'bR"""', "BR'''", 'BR"""'):
triple_quoted[t] = t triple_quoted[t] = t
single_quoted = {} single_quoted = {}
for t in ("'", '"', for t in ("'", '"',
"r'", 'r"', "R'", 'R"', "r'", 'r"', "R'", 'R"',
"u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"',
"ur'", 'ur"', "Ur'", 'Ur"', "br'", 'br"', "Br'", 'Br"',
"uR'", 'uR"', "UR'", 'UR"' ): "bR'", 'bR"', "BR'", 'BR"' ):
single_quoted[t] = t single_quoted[t] = t
tabsize = 8 tabsize = 8