mirror of
https://github.com/python/cpython.git
synced 2025-11-01 18:51:43 +00:00
Move test_tokenize to doctest.
Done as GHOP 238 by Josip Dzolonga.
This commit is contained in:
Brett Cannon
parent
66865d2ebd
commit
b8d37359cd
4 changed files with 488 additions and 832 deletions
|
|
@ -1,684 +0,0 @@
|
|||
test_tokenize
|
||||
1,0-1,34: COMMENT "# Tests for the 'tokenize' module."
|
||||
1,34-1,35: NL '\n'
|
||||
2,0-2,42: COMMENT '# Large bits stolen from test_grammar.py. '
|
||||
2,42-2,43: NL '\n'
|
||||
3,0-3,1: NL '\n'
|
||||
4,0-4,10: COMMENT '# Comments'
|
||||
4,10-4,11: NL '\n'
|
||||
5,0-5,3: STRING '"#"'
|
||||
5,3-5,4: NEWLINE '\n'
|
||||
6,0-6,2: COMMENT "#'"
|
||||
6,2-6,3: NL '\n'
|
||||
7,0-7,2: COMMENT '#"'
|
||||
7,2-7,3: NL '\n'
|
||||
8,0-8,2: COMMENT '#\\'
|
||||
8,2-8,3: NL '\n'
|
||||
9,7-9,8: COMMENT '#'
|
||||
9,8-9,9: NL '\n'
|
||||
10,4-10,9: COMMENT '# abc'
|
||||
10,9-10,10: NL '\n'
|
||||
11,0-12,4: STRING "'''#\n#'''"
|
||||
12,4-12,5: NEWLINE '\n'
|
||||
13,0-13,1: NL '\n'
|
||||
14,0-14,1: NAME 'x'
|
||||
14,2-14,3: OP '='
|
||||
14,4-14,5: NUMBER '1'
|
||||
14,7-14,8: COMMENT '#'
|
||||
14,8-14,9: NEWLINE '\n'
|
||||
15,0-15,1: NL '\n'
|
||||
16,0-16,24: COMMENT '# Balancing continuation'
|
||||
16,24-16,25: NL '\n'
|
||||
17,0-17,1: NL '\n'
|
||||
18,0-18,1: NAME 'a'
|
||||
18,2-18,3: OP '='
|
||||
18,4-18,5: OP '('
|
||||
18,5-18,6: NUMBER '3'
|
||||
18,6-18,7: OP ','
|
||||
18,8-18,9: NUMBER '4'
|
||||
18,9-18,10: OP ','
|
||||
18,10-18,11: NL '\n'
|
||||
19,2-19,3: NUMBER '5'
|
||||
19,3-19,4: OP ','
|
||||
19,5-19,6: NUMBER '6'
|
||||
19,6-19,7: OP ')'
|
||||
19,7-19,8: NEWLINE '\n'
|
||||
20,0-20,1: NAME 'y'
|
||||
20,2-20,3: OP '='
|
||||
20,4-20,5: OP '['
|
||||
20,5-20,6: NUMBER '3'
|
||||
20,6-20,7: OP ','
|
||||
20,8-20,9: NUMBER '4'
|
||||
20,9-20,10: OP ','
|
||||
20,10-20,11: NL '\n'
|
||||
21,2-21,3: NUMBER '5'
|
||||
21,3-21,4: OP ']'
|
||||
21,4-21,5: NEWLINE '\n'
|
||||
22,0-22,1: NAME 'z'
|
||||
22,2-22,3: OP '='
|
||||
22,4-22,5: OP '{'
|
||||
22,5-22,8: STRING "'a'"
|
||||
22,8-22,9: OP ':'
|
||||
22,9-22,10: NUMBER '5'
|
||||
22,10-22,11: OP ','
|
||||
22,11-22,12: NL '\n'
|
||||
23,2-23,5: STRING "'b'"
|
||||
23,5-23,6: OP ':'
|
||||
23,6-23,7: NUMBER '6'
|
||||
23,7-23,8: OP '}'
|
||||
23,8-23,9: NEWLINE '\n'
|
||||
24,0-24,1: NAME 'x'
|
||||
24,2-24,3: OP '='
|
||||
24,4-24,5: OP '('
|
||||
24,5-24,8: NAME 'len'
|
||||
24,8-24,9: OP '('
|
||||
24,9-24,10: OP '`'
|
||||
24,10-24,11: NAME 'y'
|
||||
24,11-24,12: OP '`'
|
||||
24,12-24,13: OP ')'
|
||||
24,14-24,15: OP '+'
|
||||
24,16-24,17: NUMBER '5'
|
||||
24,17-24,18: OP '*'
|
||||
24,18-24,19: NAME 'x'
|
||||
24,20-24,21: OP '-'
|
||||
24,22-24,23: NAME 'a'
|
||||
24,23-24,24: OP '['
|
||||
24,24-24,25: NL '\n'
|
||||
25,3-25,4: NUMBER '3'
|
||||
25,5-25,6: OP ']'
|
||||
25,6-25,7: NL '\n'
|
||||
26,3-26,4: OP '-'
|
||||
26,5-26,6: NAME 'x'
|
||||
26,7-26,8: OP '+'
|
||||
26,9-26,12: NAME 'len'
|
||||
26,12-26,13: OP '('
|
||||
26,13-26,14: OP '{'
|
||||
26,14-26,15: NL '\n'
|
||||
27,3-27,4: OP '}'
|
||||
27,4-27,5: NL '\n'
|
||||
28,4-28,5: OP ')'
|
||||
28,5-28,6: NL '\n'
|
||||
29,2-29,3: OP ')'
|
||||
29,3-29,4: NEWLINE '\n'
|
||||
30,0-30,1: NL '\n'
|
||||
31,0-31,36: COMMENT '# Backslash means line continuation:'
|
||||
31,36-31,37: NL '\n'
|
||||
32,0-32,1: NAME 'x'
|
||||
32,2-32,3: OP '='
|
||||
32,4-32,5: NUMBER '1'
|
||||
33,0-33,1: OP '+'
|
||||
33,2-33,3: NUMBER '1'
|
||||
33,3-33,4: NEWLINE '\n'
|
||||
34,0-34,1: NL '\n'
|
||||
35,0-35,54: COMMENT '# Backslash does not means continuation in comments :\\'
|
||||
35,54-35,55: NL '\n'
|
||||
36,0-36,1: NAME 'x'
|
||||
36,2-36,3: OP '='
|
||||
36,4-36,5: NUMBER '0'
|
||||
36,5-36,6: NEWLINE '\n'
|
||||
37,0-37,1: NL '\n'
|
||||
38,0-38,19: COMMENT '# Ordinary integers'
|
||||
38,19-38,20: NL '\n'
|
||||
39,0-39,4: NUMBER '0xff'
|
||||
39,5-39,7: OP '<>'
|
||||
39,8-39,11: NUMBER '255'
|
||||
39,11-39,12: NEWLINE '\n'
|
||||
40,0-40,4: NUMBER '0377'
|
||||
40,5-40,7: OP '<>'
|
||||
40,8-40,11: NUMBER '255'
|
||||
40,11-40,12: NEWLINE '\n'
|
||||
41,0-41,10: NUMBER '2147483647'
|
||||
41,13-41,15: OP '!='
|
||||
41,16-41,28: NUMBER '017777777777'
|
||||
41,28-41,29: NEWLINE '\n'
|
||||
42,0-42,1: OP '-'
|
||||
42,1-42,11: NUMBER '2147483647'
|
||||
42,11-42,12: OP '-'
|
||||
42,12-42,13: NUMBER '1'
|
||||
42,14-42,16: OP '!='
|
||||
42,17-42,29: NUMBER '020000000000'
|
||||
42,29-42,30: NEWLINE '\n'
|
||||
43,0-43,12: NUMBER '037777777777'
|
||||
43,13-43,15: OP '!='
|
||||
43,16-43,17: OP '-'
|
||||
43,17-43,18: NUMBER '1'
|
||||
43,18-43,19: NEWLINE '\n'
|
||||
44,0-44,10: NUMBER '0xffffffff'
|
||||
44,11-44,13: OP '!='
|
||||
44,14-44,15: OP '-'
|
||||
44,15-44,16: NUMBER '1'
|
||||
44,16-44,17: NEWLINE '\n'
|
||||
45,0-45,1: NL '\n'
|
||||
46,0-46,15: COMMENT '# Long integers'
|
||||
46,15-46,16: NL '\n'
|
||||
47,0-47,1: NAME 'x'
|
||||
47,2-47,3: OP '='
|
||||
47,4-47,6: NUMBER '0L'
|
||||
47,6-47,7: NEWLINE '\n'
|
||||
48,0-48,1: NAME 'x'
|
||||
48,2-48,3: OP '='
|
||||
48,4-48,6: NUMBER '0l'
|
||||
48,6-48,7: NEWLINE '\n'
|
||||
49,0-49,1: NAME 'x'
|
||||
49,2-49,3: OP '='
|
||||
49,4-49,23: NUMBER '0xffffffffffffffffL'
|
||||
49,23-49,24: NEWLINE '\n'
|
||||
50,0-50,1: NAME 'x'
|
||||
50,2-50,3: OP '='
|
||||
50,4-50,23: NUMBER '0xffffffffffffffffl'
|
||||
50,23-50,24: NEWLINE '\n'
|
||||
51,0-51,1: NAME 'x'
|
||||
51,2-51,3: OP '='
|
||||
51,4-51,23: NUMBER '077777777777777777L'
|
||||
51,23-51,24: NEWLINE '\n'
|
||||
52,0-52,1: NAME 'x'
|
||||
52,2-52,3: OP '='
|
||||
52,4-52,23: NUMBER '077777777777777777l'
|
||||
52,23-52,24: NEWLINE '\n'
|
||||
53,0-53,1: NAME 'x'
|
||||
53,2-53,3: OP '='
|
||||
53,4-53,35: NUMBER '123456789012345678901234567890L'
|
||||
53,35-53,36: NEWLINE '\n'
|
||||
54,0-54,1: NAME 'x'
|
||||
54,2-54,3: OP '='
|
||||
54,4-54,35: NUMBER '123456789012345678901234567890l'
|
||||
54,35-54,36: NEWLINE '\n'
|
||||
55,0-55,1: NL '\n'
|
||||
56,0-56,24: COMMENT '# Floating-point numbers'
|
||||
56,24-56,25: NL '\n'
|
||||
57,0-57,1: NAME 'x'
|
||||
57,2-57,3: OP '='
|
||||
57,4-57,8: NUMBER '3.14'
|
||||
57,8-57,9: NEWLINE '\n'
|
||||
58,0-58,1: NAME 'x'
|
||||
58,2-58,3: OP '='
|
||||
58,4-58,8: NUMBER '314.'
|
||||
58,8-58,9: NEWLINE '\n'
|
||||
59,0-59,1: NAME 'x'
|
||||
59,2-59,3: OP '='
|
||||
59,4-59,9: NUMBER '0.314'
|
||||
59,9-59,10: NEWLINE '\n'
|
||||
60,0-60,17: COMMENT '# XXX x = 000.314'
|
||||
60,17-60,18: NL '\n'
|
||||
61,0-61,1: NAME 'x'
|
||||
61,2-61,3: OP '='
|
||||
61,4-61,8: NUMBER '.314'
|
||||
61,8-61,9: NEWLINE '\n'
|
||||
62,0-62,1: NAME 'x'
|
||||
62,2-62,3: OP '='
|
||||
62,4-62,8: NUMBER '3e14'
|
||||
62,8-62,9: NEWLINE '\n'
|
||||
63,0-63,1: NAME 'x'
|
||||
63,2-63,3: OP '='
|
||||
63,4-63,8: NUMBER '3E14'
|
||||
63,8-63,9: NEWLINE '\n'
|
||||
64,0-64,1: NAME 'x'
|
||||
64,2-64,3: OP '='
|
||||
64,4-64,9: NUMBER '3e-14'
|
||||
64,9-64,10: NEWLINE '\n'
|
||||
65,0-65,1: NAME 'x'
|
||||
65,2-65,3: OP '='
|
||||
65,4-65,9: NUMBER '3e+14'
|
||||
65,9-65,10: NEWLINE '\n'
|
||||
66,0-66,1: NAME 'x'
|
||||
66,2-66,3: OP '='
|
||||
66,4-66,9: NUMBER '3.e14'
|
||||
66,9-66,10: NEWLINE '\n'
|
||||
67,0-67,1: NAME 'x'
|
||||
67,2-67,3: OP '='
|
||||
67,4-67,9: NUMBER '.3e14'
|
||||
67,9-67,10: NEWLINE '\n'
|
||||
68,0-68,1: NAME 'x'
|
||||
68,2-68,3: OP '='
|
||||
68,4-68,9: NUMBER '3.1e4'
|
||||
68,9-68,10: NEWLINE '\n'
|
||||
69,0-69,1: NL '\n'
|
||||
70,0-70,17: COMMENT '# String literals'
|
||||
70,17-70,18: NL '\n'
|
||||
71,0-71,1: NAME 'x'
|
||||
71,2-71,3: OP '='
|
||||
71,4-71,6: STRING "''"
|
||||
71,6-71,7: OP ';'
|
||||
71,8-71,9: NAME 'y'
|
||||
71,10-71,11: OP '='
|
||||
71,12-71,14: STRING '""'
|
||||
71,14-71,15: OP ';'
|
||||
71,15-71,16: NEWLINE '\n'
|
||||
72,0-72,1: NAME 'x'
|
||||
72,2-72,3: OP '='
|
||||
72,4-72,8: STRING "'\\''"
|
||||
72,8-72,9: OP ';'
|
||||
72,10-72,11: NAME 'y'
|
||||
72,12-72,13: OP '='
|
||||
72,14-72,17: STRING '"\'"'
|
||||
72,17-72,18: OP ';'
|
||||
72,18-72,19: NEWLINE '\n'
|
||||
73,0-73,1: NAME 'x'
|
||||
73,2-73,3: OP '='
|
||||
73,4-73,7: STRING '\'"\''
|
||||
73,7-73,8: OP ';'
|
||||
73,9-73,10: NAME 'y'
|
||||
73,11-73,12: OP '='
|
||||
73,13-73,17: STRING '"\\""'
|
||||
73,17-73,18: OP ';'
|
||||
73,18-73,19: NEWLINE '\n'
|
||||
74,0-74,1: NAME 'x'
|
||||
74,2-74,3: OP '='
|
||||
74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
|
||||
74,32-74,33: NEWLINE '\n'
|
||||
75,0-75,1: NAME 'y'
|
||||
75,2-75,3: OP '='
|
||||
75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
|
||||
75,31-75,32: NEWLINE '\n'
|
||||
76,0-76,1: NAME 'x'
|
||||
76,2-76,3: OP '='
|
||||
76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
|
||||
76,32-76,33: NEWLINE '\n'
|
||||
77,0-77,1: NAME 'y'
|
||||
77,2-77,3: OP '='
|
||||
77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
|
||||
77,31-77,32: NEWLINE '\n'
|
||||
78,0-78,1: NAME 'x'
|
||||
78,2-78,3: OP '='
|
||||
78,4-83,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
|
||||
83,3-83,4: NEWLINE '\n'
|
||||
84,0-84,1: NAME 'y'
|
||||
84,2-84,3: OP '='
|
||||
84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
|
||||
84,63-84,64: NEWLINE '\n'
|
||||
85,0-85,1: NAME 'y'
|
||||
85,2-85,3: OP '='
|
||||
85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
|
||||
90,3-90,4: OP ';'
|
||||
90,4-90,5: NEWLINE '\n'
|
||||
91,0-91,1: NAME 'y'
|
||||
91,2-91,3: OP '='
|
||||
91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
|
||||
96,1-96,2: OP ';'
|
||||
96,2-96,3: NEWLINE '\n'
|
||||
97,0-97,1: NAME 'y'
|
||||
97,2-97,3: OP '='
|
||||
97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
|
||||
102,1-102,2: OP ';'
|
||||
102,2-102,3: NEWLINE '\n'
|
||||
103,0-103,1: NAME 'x'
|
||||
103,2-103,3: OP '='
|
||||
103,4-103,9: STRING "r'\\\\'"
|
||||
103,10-103,11: OP '+'
|
||||
103,12-103,17: STRING "R'\\\\'"
|
||||
103,17-103,18: NEWLINE '\n'
|
||||
104,0-104,1: NAME 'x'
|
||||
104,2-104,3: OP '='
|
||||
104,4-104,9: STRING "r'\\''"
|
||||
104,10-104,11: OP '+'
|
||||
104,12-104,14: STRING "''"
|
||||
104,14-104,15: NEWLINE '\n'
|
||||
105,0-105,1: NAME 'y'
|
||||
105,2-105,3: OP '='
|
||||
105,4-107,6: STRING "r'''\nfoo bar \\\\\nbaz'''"
|
||||
107,7-107,8: OP '+'
|
||||
107,9-108,6: STRING "R'''\nfoo'''"
|
||||
108,6-108,7: NEWLINE '\n'
|
||||
109,0-109,1: NAME 'y'
|
||||
109,2-109,3: OP '='
|
||||
109,4-111,3: STRING 'r"""foo\nbar \\\\ baz\n"""'
|
||||
111,4-111,5: OP '+'
|
||||
111,6-112,3: STRING "R'''spam\n'''"
|
||||
112,3-112,4: NEWLINE '\n'
|
||||
113,0-113,1: NAME 'x'
|
||||
113,2-113,3: OP '='
|
||||
113,4-113,10: STRING "u'abc'"
|
||||
113,11-113,12: OP '+'
|
||||
113,13-113,19: STRING "U'ABC'"
|
||||
113,19-113,20: NEWLINE '\n'
|
||||
114,0-114,1: NAME 'y'
|
||||
114,2-114,3: OP '='
|
||||
114,4-114,10: STRING 'u"abc"'
|
||||
114,11-114,12: OP '+'
|
||||
114,13-114,19: STRING 'U"ABC"'
|
||||
114,19-114,20: NEWLINE '\n'
|
||||
115,0-115,1: NAME 'x'
|
||||
115,2-115,3: OP '='
|
||||
115,4-115,11: STRING "ur'abc'"
|
||||
115,12-115,13: OP '+'
|
||||
115,14-115,21: STRING "Ur'ABC'"
|
||||
115,22-115,23: OP '+'
|
||||
115,24-115,31: STRING "uR'ABC'"
|
||||
115,32-115,33: OP '+'
|
||||
115,34-115,41: STRING "UR'ABC'"
|
||||
115,41-115,42: NEWLINE '\n'
|
||||
116,0-116,1: NAME 'y'
|
||||
116,2-116,3: OP '='
|
||||
116,4-116,11: STRING 'ur"abc"'
|
||||
116,12-116,13: OP '+'
|
||||
116,14-116,21: STRING 'Ur"ABC"'
|
||||
116,22-116,23: OP '+'
|
||||
116,24-116,31: STRING 'uR"ABC"'
|
||||
116,32-116,33: OP '+'
|
||||
116,34-116,41: STRING 'UR"ABC"'
|
||||
116,41-116,42: NEWLINE '\n'
|
||||
117,0-117,1: NAME 'x'
|
||||
117,2-117,3: OP '='
|
||||
117,4-117,10: STRING "ur'\\\\'"
|
||||
117,11-117,12: OP '+'
|
||||
117,13-117,19: STRING "UR'\\\\'"
|
||||
117,19-117,20: NEWLINE '\n'
|
||||
118,0-118,1: NAME 'x'
|
||||
118,2-118,3: OP '='
|
||||
118,4-118,10: STRING "ur'\\''"
|
||||
118,11-118,12: OP '+'
|
||||
118,13-118,15: STRING "''"
|
||||
118,15-118,16: NEWLINE '\n'
|
||||
119,0-119,1: NAME 'y'
|
||||
119,2-119,3: OP '='
|
||||
119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''"
|
||||
121,7-121,8: OP '+'
|
||||
121,9-122,6: STRING "UR'''\nfoo'''"
|
||||
122,6-122,7: NEWLINE '\n'
|
||||
123,0-123,1: NAME 'y'
|
||||
123,2-123,3: OP '='
|
||||
123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""'
|
||||
125,4-125,5: OP '+'
|
||||
125,6-126,3: STRING "uR'''spam\n'''"
|
||||
126,3-126,4: NEWLINE '\n'
|
||||
127,0-127,1: NL '\n'
|
||||
128,0-128,13: COMMENT '# Indentation'
|
||||
128,13-128,14: NL '\n'
|
||||
129,0-129,2: NAME 'if'
|
||||
129,3-129,4: NUMBER '1'
|
||||
129,4-129,5: OP ':'
|
||||
129,5-129,6: NEWLINE '\n'
|
||||
130,0-130,4: INDENT ' '
|
||||
130,4-130,5: NAME 'x'
|
||||
130,6-130,7: OP '='
|
||||
130,8-130,9: NUMBER '2'
|
||||
130,9-130,10: NEWLINE '\n'
|
||||
131,0-131,0: DEDENT ''
|
||||
131,0-131,2: NAME 'if'
|
||||
131,3-131,4: NUMBER '1'
|
||||
131,4-131,5: OP ':'
|
||||
131,5-131,6: NEWLINE '\n'
|
||||
132,0-132,8: INDENT ' '
|
||||
132,8-132,9: NAME 'x'
|
||||
132,10-132,11: OP '='
|
||||
132,12-132,13: NUMBER '2'
|
||||
132,13-132,14: NEWLINE '\n'
|
||||
133,0-133,0: DEDENT ''
|
||||
133,0-133,2: NAME 'if'
|
||||
133,3-133,4: NUMBER '1'
|
||||
133,4-133,5: OP ':'
|
||||
133,5-133,6: NEWLINE '\n'
|
||||
134,0-134,4: INDENT ' '
|
||||
134,4-134,9: NAME 'while'
|
||||
134,10-134,11: NUMBER '0'
|
||||
134,11-134,12: OP ':'
|
||||
134,12-134,13: NEWLINE '\n'
|
||||
135,0-135,5: INDENT ' '
|
||||
135,5-135,7: NAME 'if'
|
||||
135,8-135,9: NUMBER '0'
|
||||
135,9-135,10: OP ':'
|
||||
135,10-135,11: NEWLINE '\n'
|
||||
136,0-136,11: INDENT ' '
|
||||
136,11-136,12: NAME 'x'
|
||||
136,13-136,14: OP '='
|
||||
136,15-136,16: NUMBER '2'
|
||||
136,16-136,17: NEWLINE '\n'
|
||||
137,5-137,5: DEDENT ''
|
||||
137,5-137,6: NAME 'x'
|
||||
137,7-137,8: OP '='
|
||||
137,9-137,10: NUMBER '2'
|
||||
137,10-137,11: NEWLINE '\n'
|
||||
138,0-138,0: DEDENT ''
|
||||
138,0-138,0: DEDENT ''
|
||||
138,0-138,2: NAME 'if'
|
||||
138,3-138,4: NUMBER '0'
|
||||
138,4-138,5: OP ':'
|
||||
138,5-138,6: NEWLINE '\n'
|
||||
139,0-139,2: INDENT ' '
|
||||
139,2-139,4: NAME 'if'
|
||||
139,5-139,6: NUMBER '2'
|
||||
139,6-139,7: OP ':'
|
||||
139,7-139,8: NEWLINE '\n'
|
||||
140,0-140,3: INDENT ' '
|
||||
140,3-140,8: NAME 'while'
|
||||
140,9-140,10: NUMBER '0'
|
||||
140,10-140,11: OP ':'
|
||||
140,11-140,12: NEWLINE '\n'
|
||||
141,0-141,8: INDENT ' '
|
||||
141,8-141,10: NAME 'if'
|
||||
141,11-141,12: NUMBER '1'
|
||||
141,12-141,13: OP ':'
|
||||
141,13-141,14: NEWLINE '\n'
|
||||
142,0-142,10: INDENT ' '
|
||||
142,10-142,11: NAME 'x'
|
||||
142,12-142,13: OP '='
|
||||
142,14-142,15: NUMBER '2'
|
||||
142,15-142,16: NEWLINE '\n'
|
||||
143,0-143,1: NL '\n'
|
||||
144,0-144,11: COMMENT '# Operators'
|
||||
144,11-144,12: NL '\n'
|
||||
145,0-145,1: NL '\n'
|
||||
146,0-146,0: DEDENT ''
|
||||
146,0-146,0: DEDENT ''
|
||||
146,0-146,0: DEDENT ''
|
||||
146,0-146,0: DEDENT ''
|
||||
146,0-146,3: NAME 'def'
|
||||
146,4-146,7: NAME 'd22'
|
||||
146,7-146,8: OP '('
|
||||
146,8-146,9: NAME 'a'
|
||||
146,9-146,10: OP ','
|
||||
146,11-146,12: NAME 'b'
|
||||
146,12-146,13: OP ','
|
||||
146,14-146,15: NAME 'c'
|
||||
146,15-146,16: OP '='
|
||||
146,16-146,17: NUMBER '1'
|
||||
146,17-146,18: OP ','
|
||||
146,19-146,20: NAME 'd'
|
||||
146,20-146,21: OP '='
|
||||
146,21-146,22: NUMBER '2'
|
||||
146,22-146,23: OP ')'
|
||||
146,23-146,24: OP ':'
|
||||
146,25-146,29: NAME 'pass'
|
||||
146,29-146,30: NEWLINE '\n'
|
||||
147,0-147,3: NAME 'def'
|
||||
147,4-147,8: NAME 'd01v'
|
||||
147,8-147,9: OP '('
|
||||
147,9-147,10: NAME 'a'
|
||||
147,10-147,11: OP '='
|
||||
147,11-147,12: NUMBER '1'
|
||||
147,12-147,13: OP ','
|
||||
147,14-147,15: OP '*'
|
||||
147,15-147,20: NAME 'restt'
|
||||
147,20-147,21: OP ','
|
||||
147,22-147,24: OP '**'
|
||||
147,24-147,29: NAME 'restd'
|
||||
147,29-147,30: OP ')'
|
||||
147,30-147,31: OP ':'
|
||||
147,32-147,36: NAME 'pass'
|
||||
147,36-147,37: NEWLINE '\n'
|
||||
148,0-148,1: NL '\n'
|
||||
149,0-149,1: OP '('
|
||||
149,1-149,2: NAME 'x'
|
||||
149,2-149,3: OP ','
|
||||
149,4-149,5: NAME 'y'
|
||||
149,5-149,6: OP ')'
|
||||
149,7-149,9: OP '<>'
|
||||
149,10-149,11: OP '('
|
||||
149,11-149,12: OP '{'
|
||||
149,12-149,15: STRING "'a'"
|
||||
149,15-149,16: OP ':'
|
||||
149,16-149,17: NUMBER '1'
|
||||
149,17-149,18: OP '}'
|
||||
149,18-149,19: OP ','
|
||||
149,20-149,21: OP '{'
|
||||
149,21-149,24: STRING "'b'"
|
||||
149,24-149,25: OP ':'
|
||||
149,25-149,26: NUMBER '2'
|
||||
149,26-149,27: OP '}'
|
||||
149,27-149,28: OP ')'
|
||||
149,28-149,29: NEWLINE '\n'
|
||||
150,0-150,1: NL '\n'
|
||||
151,0-151,12: COMMENT '# comparison'
|
||||
151,12-151,13: NL '\n'
|
||||
152,0-152,2: NAME 'if'
|
||||
152,3-152,4: NUMBER '1'
|
||||
152,5-152,6: OP '<'
|
||||
152,7-152,8: NUMBER '1'
|
||||
152,9-152,10: OP '>'
|
||||
152,11-152,12: NUMBER '1'
|
||||
152,13-152,15: OP '=='
|
||||
152,16-152,17: NUMBER '1'
|
||||
152,18-152,20: OP '>='
|
||||
152,21-152,22: NUMBER '1'
|
||||
152,23-152,25: OP '<='
|
||||
152,26-152,27: NUMBER '1'
|
||||
152,28-152,30: OP '<>'
|
||||
152,31-152,32: NUMBER '1'
|
||||
152,33-152,35: OP '!='
|
||||
152,36-152,37: NUMBER '1'
|
||||
152,38-152,40: NAME 'in'
|
||||
152,41-152,42: NUMBER '1'
|
||||
152,43-152,46: NAME 'not'
|
||||
152,47-152,49: NAME 'in'
|
||||
152,50-152,51: NUMBER '1'
|
||||
152,52-152,54: NAME 'is'
|
||||
152,55-152,56: NUMBER '1'
|
||||
152,57-152,59: NAME 'is'
|
||||
152,60-152,63: NAME 'not'
|
||||
152,64-152,65: NUMBER '1'
|
||||
152,65-152,66: OP ':'
|
||||
152,67-152,71: NAME 'pass'
|
||||
152,71-152,72: NEWLINE '\n'
|
||||
153,0-153,1: NL '\n'
|
||||
154,0-154,8: COMMENT '# binary'
|
||||
154,8-154,9: NL '\n'
|
||||
155,0-155,1: NAME 'x'
|
||||
155,2-155,3: OP '='
|
||||
155,4-155,5: NUMBER '1'
|
||||
155,6-155,7: OP '&'
|
||||
155,8-155,9: NUMBER '1'
|
||||
155,9-155,10: NEWLINE '\n'
|
||||
156,0-156,1: NAME 'x'
|
||||
156,2-156,3: OP '='
|
||||
156,4-156,5: NUMBER '1'
|
||||
156,6-156,7: OP '^'
|
||||
156,8-156,9: NUMBER '1'
|
||||
156,9-156,10: NEWLINE '\n'
|
||||
157,0-157,1: NAME 'x'
|
||||
157,2-157,3: OP '='
|
||||
157,4-157,5: NUMBER '1'
|
||||
157,6-157,7: OP '|'
|
||||
157,8-157,9: NUMBER '1'
|
||||
157,9-157,10: NEWLINE '\n'
|
||||
158,0-158,1: NL '\n'
|
||||
159,0-159,7: COMMENT '# shift'
|
||||
159,7-159,8: NL '\n'
|
||||
160,0-160,1: NAME 'x'
|
||||
160,2-160,3: OP '='
|
||||
160,4-160,5: NUMBER '1'
|
||||
160,6-160,8: OP '<<'
|
||||
160,9-160,10: NUMBER '1'
|
||||
160,11-160,13: OP '>>'
|
||||
160,14-160,15: NUMBER '1'
|
||||
160,15-160,16: NEWLINE '\n'
|
||||
161,0-161,1: NL '\n'
|
||||
162,0-162,10: COMMENT '# additive'
|
||||
162,10-162,11: NL '\n'
|
||||
163,0-163,1: NAME 'x'
|
||||
163,2-163,3: OP '='
|
||||
163,4-163,5: NUMBER '1'
|
||||
163,6-163,7: OP '-'
|
||||
163,8-163,9: NUMBER '1'
|
||||
163,10-163,11: OP '+'
|
||||
163,12-163,13: NUMBER '1'
|
||||
163,14-163,15: OP '-'
|
||||
163,16-163,17: NUMBER '1'
|
||||
163,18-163,19: OP '+'
|
||||
163,20-163,21: NUMBER '1'
|
||||
163,21-163,22: NEWLINE '\n'
|
||||
164,0-164,1: NL '\n'
|
||||
165,0-165,16: COMMENT '# multiplicative'
|
||||
165,16-165,17: NL '\n'
|
||||
166,0-166,1: NAME 'x'
|
||||
166,2-166,3: OP '='
|
||||
166,4-166,5: NUMBER '1'
|
||||
166,6-166,7: OP '/'
|
||||
166,8-166,9: NUMBER '1'
|
||||
166,10-166,11: OP '*'
|
||||
166,12-166,13: NUMBER '1'
|
||||
166,14-166,15: OP '%'
|
||||
166,16-166,17: NUMBER '1'
|
||||
166,17-166,18: NEWLINE '\n'
|
||||
167,0-167,1: NL '\n'
|
||||
168,0-168,7: COMMENT '# unary'
|
||||
168,7-168,8: NL '\n'
|
||||
169,0-169,1: NAME 'x'
|
||||
169,2-169,3: OP '='
|
||||
169,4-169,5: OP '~'
|
||||
169,5-169,6: NUMBER '1'
|
||||
169,7-169,8: OP '^'
|
||||
169,9-169,10: NUMBER '1'
|
||||
169,11-169,12: OP '&'
|
||||
169,13-169,14: NUMBER '1'
|
||||
169,15-169,16: OP '|'
|
||||
169,17-169,18: NUMBER '1'
|
||||
169,19-169,20: OP '&'
|
||||
169,21-169,22: NUMBER '1'
|
||||
169,23-169,24: OP '^'
|
||||
169,25-169,26: OP '-'
|
||||
169,26-169,27: NUMBER '1'
|
||||
169,27-169,28: NEWLINE '\n'
|
||||
170,0-170,1: NAME 'x'
|
||||
170,2-170,3: OP '='
|
||||
170,4-170,5: OP '-'
|
||||
170,5-170,6: NUMBER '1'
|
||||
170,6-170,7: OP '*'
|
||||
170,7-170,8: NUMBER '1'
|
||||
170,8-170,9: OP '/'
|
||||
170,9-170,10: NUMBER '1'
|
||||
170,11-170,12: OP '+'
|
||||
170,13-170,14: NUMBER '1'
|
||||
170,14-170,15: OP '*'
|
||||
170,15-170,16: NUMBER '1'
|
||||
170,17-170,18: OP '-'
|
||||
170,19-170,20: OP '-'
|
||||
170,20-170,21: OP '-'
|
||||
170,21-170,22: OP '-'
|
||||
170,22-170,23: NUMBER '1'
|
||||
170,23-170,24: OP '*'
|
||||
170,24-170,25: NUMBER '1'
|
||||
170,25-170,26: NEWLINE '\n'
|
||||
171,0-171,1: NL '\n'
|
||||
172,0-172,10: COMMENT '# selector'
|
||||
172,10-172,11: NL '\n'
|
||||
173,0-173,6: NAME 'import'
|
||||
173,7-173,10: NAME 'sys'
|
||||
173,10-173,11: OP ','
|
||||
173,12-173,16: NAME 'time'
|
||||
173,16-173,17: NEWLINE '\n'
|
||||
174,0-174,1: NAME 'x'
|
||||
174,2-174,3: OP '='
|
||||
174,4-174,7: NAME 'sys'
|
||||
174,7-174,8: OP '.'
|
||||
174,8-174,15: NAME 'modules'
|
||||
174,15-174,16: OP '['
|
||||
174,16-174,22: STRING "'time'"
|
||||
174,22-174,23: OP ']'
|
||||
174,23-174,24: OP '.'
|
||||
174,24-174,28: NAME 'time'
|
||||
174,28-174,29: OP '('
|
||||
174,29-174,30: OP ')'
|
||||
174,30-174,31: NEWLINE '\n'
|
||||
175,0-175,1: NL '\n'
|
||||
176,0-176,1: OP '@'
|
||||
176,1-176,13: NAME 'staticmethod'
|
||||
176,13-176,14: NEWLINE '\n'
|
||||
177,0-177,3: NAME 'def'
|
||||
177,4-177,7: NAME 'foo'
|
||||
177,7-177,8: OP '('
|
||||
177,8-177,9: OP ')'
|
||||
177,9-177,10: OP ':'
|
||||
177,11-177,15: NAME 'pass'
|
||||
177,15-177,16: NEWLINE '\n'
|
||||
178,0-178,1: NL '\n'
|
||||
179,0-179,0: ENDMARKER ''
|
||||
|
|
@ -1,112 +1,502 @@
|
|||
"""Tests for the tokenize module.
|
||||
doctests = """
|
||||
Tests for the tokenize module.
|
||||
|
||||
The tests were originally written in the old Python style, where the
|
||||
test output was compared to a golden file. This docstring represents
|
||||
the first steps towards rewriting the entire test as a doctest.
|
||||
>>> import glob, random, sys
|
||||
|
||||
The tests can be really simple. Given a small fragment of source
|
||||
code, print out a table with the tokens. The ENDMARK is omitted for
|
||||
The tests can be really simple. Given a small fragment of source
|
||||
code, print out a table with thokens. The ENDMARK is omitted for
|
||||
brevity.
|
||||
|
||||
>>> dump_tokens("1 + 1")
|
||||
NUMBER '1' (1, 0) (1, 1)
|
||||
OP '+' (1, 2) (1, 3)
|
||||
NUMBER '1' (1, 4) (1, 5)
|
||||
>>> dump_tokens("1 + 1")
|
||||
NUMBER '1' (1, 0) (1, 1)
|
||||
OP '+' (1, 2) (1, 3)
|
||||
NUMBER '1' (1, 4) (1, 5)
|
||||
|
||||
A comment generates a token here, unlike in the parser module. The
|
||||
comment token is followed by an NL or a NEWLINE token, depending on
|
||||
whether the line contains the completion of a statement.
|
||||
>>> dump_tokens("if False:\\n"
|
||||
... " # NL\\n"
|
||||
... " True = False # NEWLINE\\n")
|
||||
NAME 'if' (1, 0) (1, 2)
|
||||
NAME 'False' (1, 3) (1, 8)
|
||||
OP ':' (1, 8) (1, 9)
|
||||
NEWLINE '\\n' (1, 9) (1, 10)
|
||||
COMMENT '# NL' (2, 4) (2, 8)
|
||||
NL '\\n' (2, 8) (2, 9)
|
||||
INDENT ' ' (3, 0) (3, 4)
|
||||
NAME 'True' (3, 4) (3, 8)
|
||||
OP '=' (3, 9) (3, 10)
|
||||
NAME 'False' (3, 11) (3, 16)
|
||||
COMMENT '# NEWLINE' (3, 17) (3, 26)
|
||||
NEWLINE '\\n' (3, 26) (3, 27)
|
||||
DEDENT '' (4, 0) (4, 0)
|
||||
|
||||
>>> dump_tokens("if False:\\n"
|
||||
... " # NL\\n"
|
||||
... " True = False # NEWLINE\\n")
|
||||
NAME 'if' (1, 0) (1, 2)
|
||||
NAME 'False' (1, 3) (1, 8)
|
||||
OP ':' (1, 8) (1, 9)
|
||||
NEWLINE '\\n' (1, 9) (1, 10)
|
||||
COMMENT '# NL' (2, 4) (2, 8)
|
||||
NL '\\n' (2, 8) (2, 9)
|
||||
INDENT ' ' (3, 0) (3, 4)
|
||||
NAME 'True' (3, 4) (3, 8)
|
||||
OP '=' (3, 9) (3, 10)
|
||||
NAME 'False' (3, 11) (3, 16)
|
||||
COMMENT '# NEWLINE' (3, 17) (3, 26)
|
||||
NEWLINE '\\n' (3, 26) (3, 27)
|
||||
DEDENT '' (4, 0) (4, 0)
|
||||
>>> indent_error_file = \"""
|
||||
... def k(x):
|
||||
... x += 2
|
||||
... x += 5
|
||||
... \"""
|
||||
|
||||
>>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
IndentationError: unindent does not match any outer indentation level
|
||||
|
||||
There will be a bunch more tests of specific source patterns.
|
||||
Test roundtrip for `untokenize`. `f` is an open file or a string. The source
|
||||
code in f is tokenized, converted back to source code via tokenize.untokenize(),
|
||||
and tokenized again from the latter. The test fails if the second tokenization
|
||||
doesn't match the first.
|
||||
|
||||
The tokenize module also defines an untokenize function that should
|
||||
regenerate the original program text from the tokens.
|
||||
>>> def roundtrip(f):
|
||||
... if isinstance(f, str): f = StringIO(f)
|
||||
... token_list = list(generate_tokens(f.readline))
|
||||
... f.close()
|
||||
... tokens1 = [tok[:2] for tok in token_list]
|
||||
... new_text = untokenize(tokens1)
|
||||
... readline = iter(new_text.splitlines(1)).next
|
||||
... tokens2 = [tok[:2] for tok in generate_tokens(readline)]
|
||||
... return tokens1 == tokens2
|
||||
...
|
||||
|
||||
There are some standard formatting practices that are easy to get right.
|
||||
There are some standard formattig practises that are easy to get right.
|
||||
|
||||
>>> roundtrip("if x == 1:\\n"
|
||||
... " print x\\n")
|
||||
if x == 1:
|
||||
print x
|
||||
>>> roundtrip("if x == 1:\\n"
|
||||
... " print x\\n")
|
||||
True
|
||||
|
||||
>>> roundtrip("# This is a comment\\n# This also")
|
||||
True
|
||||
|
||||
Some people use different formatting conventions, which makes
|
||||
untokenize a little trickier. Note that this test involves trailing
|
||||
whitespace after the colon. Note that we use hex escapes to make the
|
||||
two trailing blanks apparent in the expected output.
|
||||
untokenize a little trickier. Note that this test involves trailing
|
||||
whitespace after the colon. Note that we use hex escapes to make the
|
||||
two trailing blanks apperant in the expected output.
|
||||
|
||||
>>> roundtrip("if x == 1 : \\n"
|
||||
... " print x\\n")
|
||||
if x == 1 :\x20\x20
|
||||
print x
|
||||
>>> roundtrip("if x == 1 : \\n"
|
||||
... " print x\\n")
|
||||
True
|
||||
|
||||
Comments need to go in the right place.
|
||||
>>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
|
||||
>>> roundtrip(open(f))
|
||||
True
|
||||
|
||||
>>> roundtrip("if x == 1:\\n"
|
||||
... " # A comment by itself.\\n"
|
||||
... " print x # Comment here, too.\\n"
|
||||
... " # Another comment.\\n"
|
||||
... "after_if = True\\n")
|
||||
if x == 1:
|
||||
# A comment by itself.
|
||||
print x # Comment here, too.
|
||||
# Another comment.
|
||||
after_if = True
|
||||
>>> roundtrip("if x == 1:\\n"
|
||||
... " # A comment by itself.\\n"
|
||||
... " print x # Comment here, too.\\n"
|
||||
... " # Another comment.\\n"
|
||||
... "after_if = True\\n")
|
||||
True
|
||||
|
||||
>>> roundtrip("if (x # The comments need to go in the right place\\n"
|
||||
... " == 1):\\n"
|
||||
... " print 'x == 1'\\n")
|
||||
if (x # The comments need to go in the right place
|
||||
== 1):
|
||||
print 'x == 1'
|
||||
>>> roundtrip("if (x # The comments need to go in the right place\\n"
|
||||
... " == 1):\\n"
|
||||
... " print 'x==1'\\n")
|
||||
True
|
||||
|
||||
>>> roundtrip("class Test: # A comment here\\n"
|
||||
... " # A comment with weird indent\\n"
|
||||
... " after_com = 5\\n"
|
||||
... " def x(m): return m*5 # a one liner\\n"
|
||||
... " def y(m): # A whitespace after the colon\\n"
|
||||
... " return y*4 # 3-space indent\\n")
|
||||
True
|
||||
|
||||
Some error-handling code
|
||||
|
||||
>>> roundtrip("try: import somemodule\\n"
|
||||
... "except ImportError: # comment\\n"
|
||||
... " print 'Can not import' # comment2\\n"
|
||||
... "else: print 'Loaded'\\n")
|
||||
True
|
||||
|
||||
Balancing contunuation
|
||||
|
||||
>>> roundtrip("a = (3,4, \\n"
|
||||
... "5,6)\\n"
|
||||
... "y = [3, 4,\\n"
|
||||
... "5]\\n"
|
||||
... "z = {'a': 5,\\n"
|
||||
... "'b':15, 'c':True}\\n"
|
||||
... "x = len(y) + 5 - a[\\n"
|
||||
... "3] - a[2]\\n"
|
||||
... "+ len(z) - z[\\n"
|
||||
... "'b']\\n")
|
||||
True
|
||||
|
||||
Ordinary integers and binary operators
|
||||
|
||||
>>> dump_tokens("0xff <= 255")
|
||||
NUMBER '0xff' (1, 0) (1, 4)
|
||||
OP '<=' (1, 5) (1, 7)
|
||||
NUMBER '255' (1, 8) (1, 11)
|
||||
>>> dump_tokens("01234567 > ~0x15")
|
||||
NUMBER '01234567' (1, 0) (1, 8)
|
||||
OP '>' (1, 9) (1, 10)
|
||||
OP '~' (1, 11) (1, 12)
|
||||
NUMBER '0x15' (1, 12) (1, 16)
|
||||
>>> dump_tokens("2134568 != 01231515")
|
||||
NUMBER '2134568' (1, 0) (1, 7)
|
||||
OP '!=' (1, 8) (1, 10)
|
||||
NUMBER '01231515' (1, 11) (1, 19)
|
||||
>>> dump_tokens("(-124561-1) & 0200000000")
|
||||
OP '(' (1, 0) (1, 1)
|
||||
OP '-' (1, 1) (1, 2)
|
||||
NUMBER '124561' (1, 2) (1, 8)
|
||||
OP '-' (1, 8) (1, 9)
|
||||
NUMBER '1' (1, 9) (1, 10)
|
||||
OP ')' (1, 10) (1, 11)
|
||||
OP '&' (1, 12) (1, 13)
|
||||
NUMBER '0200000000' (1, 14) (1, 24)
|
||||
>>> dump_tokens("0xdeadbeef != -1")
|
||||
NUMBER '0xdeadbeef' (1, 0) (1, 10)
|
||||
OP '!=' (1, 11) (1, 13)
|
||||
OP '-' (1, 14) (1, 15)
|
||||
NUMBER '1' (1, 15) (1, 16)
|
||||
>>> dump_tokens("0xdeadc0de & 012345")
|
||||
NUMBER '0xdeadc0de' (1, 0) (1, 10)
|
||||
OP '&' (1, 11) (1, 12)
|
||||
NUMBER '012345' (1, 13) (1, 19)
|
||||
>>> dump_tokens("0xFF & 0x15 | 1234")
|
||||
NUMBER '0xFF' (1, 0) (1, 4)
|
||||
OP '&' (1, 5) (1, 6)
|
||||
NUMBER '0x15' (1, 7) (1, 11)
|
||||
OP '|' (1, 12) (1, 13)
|
||||
NUMBER '1234' (1, 14) (1, 18)
|
||||
|
||||
Long integers
|
||||
|
||||
>>> dump_tokens("x = 0L")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '0L' (1, 4) (1, 6)
|
||||
>>> dump_tokens("x = 0xfffffffffff")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '0xffffffffff (1, 4) (1, 17)
|
||||
>>> dump_tokens("x = 123141242151251616110l")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '123141242151 (1, 4) (1, 26)
|
||||
>>> dump_tokens("x = -15921590215012591L")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
OP '-' (1, 4) (1, 5)
|
||||
NUMBER '159215902150 (1, 5) (1, 23)
|
||||
|
||||
Floating point numbers
|
||||
|
||||
>>> dump_tokens("x = 3.14159")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '3.14159' (1, 4) (1, 11)
|
||||
>>> dump_tokens("x = 314159.")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '314159.' (1, 4) (1, 11)
|
||||
>>> dump_tokens("x = .314159")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '.314159' (1, 4) (1, 11)
|
||||
>>> dump_tokens("x = 3e14159")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '3e14159' (1, 4) (1, 11)
|
||||
>>> dump_tokens("x = 3E123")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '3E123' (1, 4) (1, 9)
|
||||
>>> dump_tokens("x+y = 3e-1230")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '+' (1, 1) (1, 2)
|
||||
NAME 'y' (1, 2) (1, 3)
|
||||
OP '=' (1, 4) (1, 5)
|
||||
NUMBER '3e-1230' (1, 6) (1, 13)
|
||||
>>> dump_tokens("x = 3.14e159")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '3.14e159' (1, 4) (1, 12)
|
||||
|
||||
String literals
|
||||
|
||||
>>> dump_tokens("x = ''; y = \\\"\\\"")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING "''" (1, 4) (1, 6)
|
||||
OP ';' (1, 6) (1, 7)
|
||||
NAME 'y' (1, 8) (1, 9)
|
||||
OP '=' (1, 10) (1, 11)
|
||||
STRING '""' (1, 12) (1, 14)
|
||||
>>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING '\\'"\\'' (1, 4) (1, 7)
|
||||
OP ';' (1, 7) (1, 8)
|
||||
NAME 'y' (1, 9) (1, 10)
|
||||
OP '=' (1, 11) (1, 12)
|
||||
STRING '"\\'"' (1, 13) (1, 16)
|
||||
>>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING '"doesn\\'t "' (1, 4) (1, 14)
|
||||
NAME 'shrink' (1, 14) (1, 20)
|
||||
STRING '", does it"' (1, 20) (1, 31)
|
||||
>>> dump_tokens("x = u'abc' + U'ABC'")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING "u'abc'" (1, 4) (1, 10)
|
||||
OP '+' (1, 11) (1, 12)
|
||||
STRING "U'ABC'" (1, 13) (1, 19)
|
||||
>>> dump_tokens('y = u"ABC" + U"ABC"')
|
||||
NAME 'y' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING 'u"ABC"' (1, 4) (1, 10)
|
||||
OP '+' (1, 11) (1, 12)
|
||||
STRING 'U"ABC"' (1, 13) (1, 19)
|
||||
>>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING "ur'abc'" (1, 4) (1, 11)
|
||||
OP '+' (1, 12) (1, 13)
|
||||
STRING "Ur'ABC'" (1, 14) (1, 21)
|
||||
OP '+' (1, 22) (1, 23)
|
||||
STRING "uR'ABC'" (1, 24) (1, 31)
|
||||
OP '+' (1, 32) (1, 33)
|
||||
STRING "UR'ABC'" (1, 34) (1, 41)
|
||||
>>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
|
||||
NAME 'y' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
STRING 'ur"abc"' (1, 4) (1, 11)
|
||||
OP '+' (1, 12) (1, 13)
|
||||
STRING 'Ur"ABC"' (1, 14) (1, 21)
|
||||
OP '+' (1, 22) (1, 23)
|
||||
STRING 'uR"ABC"' (1, 24) (1, 31)
|
||||
OP '+' (1, 32) (1, 33)
|
||||
STRING 'UR"ABC"' (1, 34) (1, 41)
|
||||
|
||||
Operators
|
||||
|
||||
>>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
|
||||
NAME 'def' (1, 0) (1, 3)
|
||||
NAME 'd22' (1, 4) (1, 7)
|
||||
OP '(' (1, 7) (1, 8)
|
||||
NAME 'a' (1, 8) (1, 9)
|
||||
OP ',' (1, 9) (1, 10)
|
||||
NAME 'b' (1, 11) (1, 12)
|
||||
OP ',' (1, 12) (1, 13)
|
||||
NAME 'c' (1, 14) (1, 15)
|
||||
OP '=' (1, 15) (1, 16)
|
||||
NUMBER '2' (1, 16) (1, 17)
|
||||
OP ',' (1, 17) (1, 18)
|
||||
NAME 'd' (1, 19) (1, 20)
|
||||
OP '=' (1, 20) (1, 21)
|
||||
NUMBER '2' (1, 21) (1, 22)
|
||||
OP ',' (1, 22) (1, 23)
|
||||
OP '*' (1, 24) (1, 25)
|
||||
NAME 'k' (1, 25) (1, 26)
|
||||
OP ')' (1, 26) (1, 27)
|
||||
OP ':' (1, 27) (1, 28)
|
||||
NAME 'pass' (1, 29) (1, 33)
|
||||
>>> dump_tokens("def d01v_(a=1, *k, **w): pass")
|
||||
NAME 'def' (1, 0) (1, 3)
|
||||
NAME 'd01v_' (1, 4) (1, 9)
|
||||
OP '(' (1, 9) (1, 10)
|
||||
NAME 'a' (1, 10) (1, 11)
|
||||
OP '=' (1, 11) (1, 12)
|
||||
NUMBER '1' (1, 12) (1, 13)
|
||||
OP ',' (1, 13) (1, 14)
|
||||
OP '*' (1, 15) (1, 16)
|
||||
NAME 'k' (1, 16) (1, 17)
|
||||
OP ',' (1, 17) (1, 18)
|
||||
OP '**' (1, 19) (1, 21)
|
||||
NAME 'w' (1, 21) (1, 22)
|
||||
OP ')' (1, 22) (1, 23)
|
||||
OP ':' (1, 23) (1, 24)
|
||||
NAME 'pass' (1, 25) (1, 29)
|
||||
|
||||
Comparison
|
||||
|
||||
>>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
|
||||
... "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
|
||||
NAME 'if' (1, 0) (1, 2)
|
||||
NUMBER '1' (1, 3) (1, 4)
|
||||
OP '<' (1, 5) (1, 6)
|
||||
NUMBER '1' (1, 7) (1, 8)
|
||||
OP '>' (1, 9) (1, 10)
|
||||
NUMBER '1' (1, 11) (1, 12)
|
||||
OP '==' (1, 13) (1, 15)
|
||||
NUMBER '1' (1, 16) (1, 17)
|
||||
OP '>=' (1, 18) (1, 20)
|
||||
NUMBER '5' (1, 21) (1, 22)
|
||||
OP '<=' (1, 23) (1, 25)
|
||||
NUMBER '0x15' (1, 26) (1, 30)
|
||||
OP '<=' (1, 31) (1, 33)
|
||||
NUMBER '0x12' (1, 34) (1, 38)
|
||||
OP '!=' (1, 39) (1, 41)
|
||||
NUMBER '1' (1, 42) (1, 43)
|
||||
NAME 'and' (1, 44) (1, 47)
|
||||
NUMBER '5' (1, 48) (1, 49)
|
||||
NAME 'in' (1, 50) (1, 52)
|
||||
NUMBER '1' (1, 53) (1, 54)
|
||||
NAME 'not' (1, 55) (1, 58)
|
||||
NAME 'in' (1, 59) (1, 61)
|
||||
NUMBER '1' (1, 62) (1, 63)
|
||||
NAME 'is' (1, 64) (1, 66)
|
||||
NUMBER '1' (1, 67) (1, 68)
|
||||
NAME 'or' (1, 69) (1, 71)
|
||||
NUMBER '5' (1, 72) (1, 73)
|
||||
NAME 'is' (1, 74) (1, 76)
|
||||
NAME 'not' (1, 77) (1, 80)
|
||||
NUMBER '1' (1, 81) (1, 82)
|
||||
OP ':' (1, 82) (1, 83)
|
||||
NAME 'pass' (1, 84) (1, 88)
|
||||
|
||||
Shift
|
||||
|
||||
>>> dump_tokens("x = 1 << 1 >> 5")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '1' (1, 4) (1, 5)
|
||||
OP '<<' (1, 6) (1, 8)
|
||||
NUMBER '1' (1, 9) (1, 10)
|
||||
OP '>>' (1, 11) (1, 13)
|
||||
NUMBER '5' (1, 14) (1, 15)
|
||||
|
||||
Additive
|
||||
|
||||
>>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '1' (1, 4) (1, 5)
|
||||
OP '-' (1, 6) (1, 7)
|
||||
NAME 'y' (1, 8) (1, 9)
|
||||
OP '+' (1, 10) (1, 11)
|
||||
NUMBER '15' (1, 12) (1, 14)
|
||||
OP '-' (1, 15) (1, 16)
|
||||
NUMBER '01' (1, 17) (1, 19)
|
||||
OP '+' (1, 20) (1, 21)
|
||||
NUMBER '0x124' (1, 22) (1, 27)
|
||||
OP '+' (1, 28) (1, 29)
|
||||
NAME 'z' (1, 30) (1, 31)
|
||||
OP '+' (1, 32) (1, 33)
|
||||
NAME 'a' (1, 34) (1, 35)
|
||||
OP '[' (1, 35) (1, 36)
|
||||
NUMBER '5' (1, 36) (1, 37)
|
||||
OP ']' (1, 37) (1, 38)
|
||||
|
||||
Multiplicative
|
||||
|
||||
>>> dump_tokens("x = 1//1*1/5*12%0x12")
|
||||
NAME 'x' (1, 0) (1, 1)
|
||||
OP '=' (1, 2) (1, 3)
|
||||
NUMBER '1' (1, 4) (1, 5)
|
||||
OP '//' (1, 5) (1, 7)
|
||||
NUMBER '1' (1, 7) (1, 8)
|
||||
OP '*' (1, 8) (1, 9)
|
||||
NUMBER '1' (1, 9) (1, 10)
|
||||
OP '/' (1, 10) (1, 11)
|
||||
NUMBER '5' (1, 11) (1, 12)
|
||||
OP '*' (1, 12) (1, 13)
|
||||
NUMBER '12' (1, 13) (1, 15)
|
||||
OP '%' (1, 15) (1, 16)
|
||||
NUMBER '0x12' (1, 16) (1, 20)
|
||||
|
||||
Unary
|
||||
|
||||
>>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
|
||||
OP '~' (1, 0) (1, 1)
|
||||
NUMBER '1' (1, 1) (1, 2)
|
||||
OP '^' (1, 3) (1, 4)
|
||||
NUMBER '1' (1, 5) (1, 6)
|
||||
OP '&' (1, 7) (1, 8)
|
||||
NUMBER '1' (1, 9) (1, 10)
|
||||
OP '|' (1, 11) (1, 12)
|
||||
NUMBER '1' (1, 12) (1, 13)
|
||||
OP '^' (1, 14) (1, 15)
|
||||
OP '-' (1, 16) (1, 17)
|
||||
NUMBER '1' (1, 17) (1, 18)
|
||||
>>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
|
||||
OP '-' (1, 0) (1, 1)
|
||||
NUMBER '1' (1, 1) (1, 2)
|
||||
OP '*' (1, 2) (1, 3)
|
||||
NUMBER '1' (1, 3) (1, 4)
|
||||
OP '/' (1, 4) (1, 5)
|
||||
NUMBER '1' (1, 5) (1, 6)
|
||||
OP '+' (1, 6) (1, 7)
|
||||
NUMBER '1' (1, 7) (1, 8)
|
||||
OP '*' (1, 8) (1, 9)
|
||||
NUMBER '1' (1, 9) (1, 10)
|
||||
OP '//' (1, 10) (1, 12)
|
||||
NUMBER '1' (1, 12) (1, 13)
|
||||
OP '-' (1, 14) (1, 15)
|
||||
OP '-' (1, 16) (1, 17)
|
||||
OP '-' (1, 17) (1, 18)
|
||||
OP '-' (1, 18) (1, 19)
|
||||
NUMBER '1' (1, 19) (1, 20)
|
||||
OP '**' (1, 20) (1, 22)
|
||||
NUMBER '1' (1, 22) (1, 23)
|
||||
|
||||
Selector
|
||||
|
||||
>>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
|
||||
NAME 'import' (1, 0) (1, 6)
|
||||
NAME 'sys' (1, 7) (1, 10)
|
||||
OP ',' (1, 10) (1, 11)
|
||||
NAME 'time' (1, 12) (1, 16)
|
||||
NEWLINE '\\n' (1, 16) (1, 17)
|
||||
NAME 'x' (2, 0) (2, 1)
|
||||
OP '=' (2, 2) (2, 3)
|
||||
NAME 'sys' (2, 4) (2, 7)
|
||||
OP '.' (2, 7) (2, 8)
|
||||
NAME 'modules' (2, 8) (2, 15)
|
||||
OP '[' (2, 15) (2, 16)
|
||||
STRING "'time'" (2, 16) (2, 22)
|
||||
OP ']' (2, 22) (2, 23)
|
||||
OP '.' (2, 23) (2, 24)
|
||||
NAME 'time' (2, 24) (2, 28)
|
||||
OP '(' (2, 28) (2, 29)
|
||||
OP ')' (2, 29) (2, 30)
|
||||
|
||||
Methods
|
||||
|
||||
>>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
|
||||
OP '@' (1, 0) (1, 1)
|
||||
NAME 'staticmethod (1, 1) (1, 13)
|
||||
NEWLINE '\\n' (1, 13) (1, 14)
|
||||
NAME 'def' (2, 0) (2, 3)
|
||||
NAME 'foo' (2, 4) (2, 7)
|
||||
OP '(' (2, 7) (2, 8)
|
||||
NAME 'x' (2, 8) (2, 9)
|
||||
OP ',' (2, 9) (2, 10)
|
||||
NAME 'y' (2, 10) (2, 11)
|
||||
OP ')' (2, 11) (2, 12)
|
||||
OP ':' (2, 12) (2, 13)
|
||||
NAME 'pass' (2, 14) (2, 18)
|
||||
|
||||
Backslash means line continuation, except for comments
|
||||
|
||||
>>> roundtrip("x=1+\\\\n"
|
||||
... "1\\n"
|
||||
... "# This is a comment\\\\n"
|
||||
... "# This also\\n")
|
||||
True
|
||||
>>> roundtrip("# Comment \\\\nx = 0")
|
||||
True
|
||||
|
||||
>>>
|
||||
>>> tempdir = os.path.dirname(f) or os.curdir
|
||||
>>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
|
||||
>>> if not test_support.is_resource_enabled("compiler"):
|
||||
... testfiles = random.sample(testfiles, 10)
|
||||
...
|
||||
>>> for testfile in testfiles:
|
||||
... if not roundtrip(open(testfile)): break
|
||||
... else: True
|
||||
True
|
||||
"""
|
||||
|
||||
import os, glob, random, time, sys
|
||||
from cStringIO import StringIO
|
||||
from test.test_support import (verbose, findfile, is_resource_enabled,
|
||||
TestFailed)
|
||||
from tokenize import (tokenize, generate_tokens, untokenize, tok_name,
|
||||
ENDMARKER, NUMBER, NAME, OP, STRING, COMMENT)
|
||||
|
||||
# How much time in seconds can pass before we print a 'Still working' message.
|
||||
_PRINT_WORKING_MSG_INTERVAL = 5 * 60
|
||||
|
||||
# Test roundtrip for `untokenize`. `f` is a file path. The source code in f
|
||||
# is tokenized, converted back to source code via tokenize.untokenize(),
|
||||
# and tokenized again from the latter. The test fails if the second
|
||||
# tokenization doesn't match the first.
|
||||
def test_roundtrip(f):
|
||||
## print 'Testing:', f
|
||||
fobj = open(f)
|
||||
try:
|
||||
fulltok = list(generate_tokens(fobj.readline))
|
||||
finally:
|
||||
fobj.close()
|
||||
|
||||
t1 = [tok[:2] for tok in fulltok]
|
||||
newtext = untokenize(t1)
|
||||
readline = iter(newtext.splitlines(1)).next
|
||||
t2 = [tok[:2] for tok in generate_tokens(readline)]
|
||||
if t1 != t2:
|
||||
raise TestFailed("untokenize() roundtrip failed for %r" % f)
|
||||
from test import test_support
|
||||
from tokenize import (tokenize, untokenize, generate_tokens, NUMBER, NAME, OP,
|
||||
STRING, ENDMARKER, tok_name)
|
||||
from StringIO import StringIO
|
||||
import os
|
||||
|
||||
def dump_tokens(s):
|
||||
"""Print out the tokens in s in a table format.
|
||||
|
|
@ -118,12 +508,7 @@ def dump_tokens(s):
|
|||
if type == ENDMARKER:
|
||||
break
|
||||
type = tok_name[type]
|
||||
print "%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals()
|
||||
|
||||
def roundtrip(s):
|
||||
f = StringIO(s)
|
||||
source = untokenize(generate_tokens(f.readline))
|
||||
print source,
|
||||
print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
|
||||
|
||||
# This is an example from the docs, set up as a doctest.
|
||||
def decistmt(s):
|
||||
|
|
@ -163,61 +548,13 @@ def decistmt(s):
|
|||
result.append((toknum, tokval))
|
||||
return untokenize(result)
|
||||
|
||||
|
||||
__test__ = {"doctests" : doctests, 'decistmt': decistmt}
|
||||
|
||||
|
||||
def test_main():
|
||||
if verbose:
|
||||
print 'starting...'
|
||||
|
||||
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
|
||||
|
||||
# This displays the tokenization of tokenize_tests.py to stdout, and
|
||||
# regrtest.py checks that this equals the expected output (in the
|
||||
# test/output/ directory).
|
||||
f = open(findfile('tokenize_tests' + os.extsep + 'txt'))
|
||||
tokenize(f.readline)
|
||||
f.close()
|
||||
|
||||
# Now run test_roundtrip() over tokenize_test.py too, and over all
|
||||
# (if the "compiler" resource is enabled) or a small random sample (if
|
||||
# "compiler" is not enabled) of the test*.py files.
|
||||
f = findfile('tokenize_tests' + os.extsep + 'txt')
|
||||
test_roundtrip(f)
|
||||
|
||||
testdir = os.path.dirname(f) or os.curdir
|
||||
testfiles = glob.glob(testdir + os.sep + 'test*.py')
|
||||
if not is_resource_enabled('compiler'):
|
||||
testfiles = random.sample(testfiles, 10)
|
||||
|
||||
for f in testfiles:
|
||||
# Print still working message since this test can be really slow
|
||||
if next_time <= time.time():
|
||||
next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
|
||||
print >>sys.__stdout__, ' test_main still working, be patient...'
|
||||
sys.__stdout__.flush()
|
||||
|
||||
test_roundtrip(f)
|
||||
|
||||
# Test detecton of IndentationError.
|
||||
sampleBadText = """\
|
||||
def foo():
|
||||
bar
|
||||
baz
|
||||
"""
|
||||
|
||||
try:
|
||||
for tok in generate_tokens(StringIO(sampleBadText).readline):
|
||||
pass
|
||||
except IndentationError:
|
||||
pass
|
||||
else:
|
||||
raise TestFailed("Did not detect IndentationError:")
|
||||
|
||||
# Run the doctests in this module.
|
||||
from test import test_tokenize # i.e., this module
|
||||
from test.test_support import run_doctest
|
||||
run_doctest(test_tokenize, verbose)
|
||||
|
||||
if verbose:
|
||||
print 'finished'
|
||||
from test import test_tokenize
|
||||
test_support.run_doctest(test_tokenize, True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_main()
|
||||
|
|
|
|||
|
|
@ -178,6 +178,7 @@ Luke Dunstan
|
|||
Andy Dustman
|
||||
Gary Duzan
|
||||
Eugene Dvurechenski
|
||||
Josip Dzolonga
|
||||
Maxim Dzumanenko
|
||||
Hans Eckardt
|
||||
Grant Edwards
|
||||
|
|
|
|||
|
|
@ -50,6 +50,8 @@ Library
|
|||
Tests
|
||||
-----
|
||||
|
||||
- GHOP 238: Convert test_tokenize to use doctest.
|
||||
|
||||
- GHOP 237: Rewrite test_thread using unittest.
|
||||
|
||||
- Patch #2232: os.tmpfile might fail on Windows if the user has no
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue