Show '\011', '\012', and '\015' as '\t', '\n', '\r' in strings.

Switch from octal escapes to hex escapes for other nonprintable characters.
This commit is contained in:
Ka-Ping Yee 2001-01-24 17:19:08 +00:00
parent 726b78ecb8
commit fa004ad36c
13 changed files with 221 additions and 205 deletions

View file

@ -41,5 +41,5 @@ Example:
\begin{verbatim} \begin{verbatim}
>>> import linecache >>> import linecache
>>> linecache.getline('/etc/passwd', 4) >>> linecache.getline('/etc/passwd', 4)
'sys:x:3:3:sys:/dev:/bin/sh\012' 'sys:x:3:3:sys:/dev:/bin/sh\n'
\end{verbatim} \end{verbatim}

View file

@ -265,7 +265,7 @@ Example:
>>> import locale >>> import locale
>>> loc = locale.setlocale(locale.LC_ALL) # get current locale >>> loc = locale.setlocale(locale.LC_ALL) # get current locale
>>> locale.setlocale(locale.LC_ALL, 'de') # use German locale >>> locale.setlocale(locale.LC_ALL, 'de') # use German locale
>>> locale.strcoll('f\344n', 'foo') # compare a string containing an umlaut >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut
>>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale
>>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale
>>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale

View file

@ -25,14 +25,14 @@ the spammish repetition'}:
>>> m.update("Nobody inspects") >>> m.update("Nobody inspects")
>>> m.update(" the spammish repetition") >>> m.update(" the spammish repetition")
>>> m.digest() >>> m.digest()
'\273d\234\203\335\036\245\311\331\336\311\241\215\360\377\351' '\xbbd\x9c\x83\xdd\x1e\xa5\xc9\xd9\xde\xc9\xa1\x8d\xf0\xff\xe9'
\end{verbatim} \end{verbatim}
More condensed: More condensed:
\begin{verbatim} \begin{verbatim}
>>> md5.new("Nobody inspects the spammish repetition").digest() >>> md5.new("Nobody inspects the spammish repetition").digest()
'\273d\234\203\335\036\245\311\331\336\311\241\215\360\377\351' '\xbbd\x9c\x83\xdd\x1e\xa5\xc9\xd9\xde\xc9\xa1\x8d\xf0\xff\xe9'
\end{verbatim} \end{verbatim}
\begin{funcdesc}{new}{\optional{arg}} \begin{funcdesc}{new}{\optional{arg}}

View file

@ -432,7 +432,7 @@ buried deep in nested tuples.
(297, (297,
(298, (298,
(299, (299,
(300, (3, '"""Some documentation.\012"""'))))))))))))))))), (300, (3, '"""Some documentation.\n"""'))))))))))))))))),
(4, ''))), (4, ''))),
(4, ''), (4, ''),
(0, '')) (0, ''))
@ -537,7 +537,7 @@ module docstring from the parse tree created previously is easy:
>>> found >>> found
1 1
>>> vars >>> vars
{'docstring': '"""Some documentation.\012"""'} {'docstring': '"""Some documentation.\n"""'}
\end{verbatim} \end{verbatim}
Once specific data can be extracted from a location where it is Once specific data can be extracted from a location where it is

View file

@ -272,11 +272,11 @@ Start element: parent {'id': 'top'}
Start element: child1 {'name': 'paul'} Start element: child1 {'name': 'paul'}
Character data: 'Text goes here' Character data: 'Text goes here'
End element: child1 End element: child1
Character data: '\012' Character data: '\n'
Start element: child2 {'name': 'fred'} Start element: child2 {'name': 'fred'}
Character data: 'More text' Character data: 'More text'
End element: child2 End element: child2
Character data: '\012' Character data: '\n'
End element: parent End element: parent
\end{verbatim} \end{verbatim}

View file

@ -68,17 +68,17 @@ An example usage:
>>> import rotor >>> import rotor
>>> rt = rotor.newrotor('key', 12) >>> rt = rotor.newrotor('key', 12)
>>> rt.encrypt('bar') >>> rt.encrypt('bar')
'\2534\363' '\xab4\xf3'
>>> rt.encryptmore('bar') >>> rt.encryptmore('bar')
'\357\375$' '\xef\xfd$'
>>> rt.encrypt('bar') >>> rt.encrypt('bar')
'\2534\363' '\xab4\xf3'
>>> rt.decrypt('\2534\363') >>> rt.decrypt('\xab4\xf3')
'bar' 'bar'
>>> rt.decryptmore('\357\375$') >>> rt.decryptmore('\xef\xfd$')
'bar' 'bar'
>>> rt.decrypt('\357\375$') >>> rt.decrypt('\xef\xfd$')
'l(\315' 'l(\xcd'
>>> del rt >>> del rt
\end{verbatim} \end{verbatim}

View file

@ -168,8 +168,8 @@ big-endian machine):
\begin{verbatim} \begin{verbatim}
>>> from struct import * >>> from struct import *
>>> pack('hhl', 1, 2, 3) >>> pack('hhl', 1, 2, 3)
'\000\001\000\002\000\000\000\003' '\x00\x01\x00\x02\x00\x00\x00\x03'
>>> unpack('hhl', '\000\001\000\002\000\000\000\003') >>> unpack('hhl', '\x00\x01\x00\x02\x00\x00\x00\x03')
(1, 2, 3) (1, 2, 3)
>>> calcsize('hhl') >>> calcsize('hhl')
8 8

View file

@ -801,24 +801,24 @@ Apart from these standard encodings, Python provides a whole set of
other ways of creating Unicode strings on the basis of a known other ways of creating Unicode strings on the basis of a known
encoding. encoding.
The builtin \function{unicode()}\bifuncindex{unicode} provides access The built-in function \function{unicode()}\bifuncindex{unicode} provides access
to all registered Unicode codecs (COders and DECoders). Some of the to all registered Unicode codecs (COders and DECoders). Some of the
more well known encodings which these codecs can convert are more well known encodings which these codecs can convert are
\emph{Latin-1}, \emph{ASCII}, \emph{UTF-8} and \emph{UTF-16}. The latter two \emph{Latin-1}, \emph{ASCII}, \emph{UTF-8} and \emph{UTF-16}. The latter two
are variable length encodings which permit to store Unicode characters are variable-length encodings which store Unicode characters
in 8 or 16 bits. Python uses UTF-8 as default encoding. This becomes in blocks of 8 or 16 bits. To print a Unicode string or write it to a file,
noticeable when printing Unicode strings or writing them to files. you must convert it to a string with the \method{encode()} method.
\begin{verbatim} \begin{verbatim}
>>> u"äöü" >>> u"äöü"
u'\344\366\374' u'\344\366\374'
>>> str(u"äöü") >>> u"äöü".encode('UTF-8')
'\303\244\303\266\303\274' '\303\244\303\266\303\274'
\end{verbatim} \end{verbatim}
If you have data in a specific encoding and want to produce a If you have data in a specific encoding and want to produce a
corresponding Unicode string from it, you can use the corresponding Unicode string from it, you can use the
\function{unicode()} builtin with the encoding name as second \function{unicode()} function with the encoding name as second
argument. argument.
\begin{verbatim} \begin{verbatim}
@ -826,14 +826,6 @@ argument.
u'\344\366\374' u'\344\366\374'
\end{verbatim} \end{verbatim}
To convert the Unicode string back into a string using the original
encoding, the objects provide an \method{encode()} method.
\begin{verbatim}
>>> u"äöü".encode('UTF-8')
'\303\244\303\266\303\274'
\end{verbatim}
\subsection{Lists \label{lists}} \subsection{Lists \label{lists}}

View file

@ -24,18 +24,18 @@ Value interpolation too deeply recursive:
Testing for parsing errors... Testing for parsing errors...
Caught expected exception: File contains parsing errors: <???> Caught expected exception: File contains parsing errors: <???>
[line 2]: ' extra-spaces: splat\012' [line 2]: ' extra-spaces: splat\n'
Caught expected exception: File contains parsing errors: <???> Caught expected exception: File contains parsing errors: <???>
[line 2]: ' extra-spaces= splat\012' [line 2]: ' extra-spaces= splat\n'
Caught expected exception: File contains parsing errors: <???> Caught expected exception: File contains parsing errors: <???>
[line 2]: 'option-without-value\012' [line 2]: 'option-without-value\n'
Caught expected exception: File contains parsing errors: <???> Caught expected exception: File contains parsing errors: <???>
[line 2]: ':value-without-option-name\012' [line 2]: ':value-without-option-name\n'
Caught expected exception: File contains parsing errors: <???> Caught expected exception: File contains parsing errors: <???>
[line 2]: '=value-without-option-name\012' [line 2]: '=value-without-option-name\n'
Caught expected exception: File contains no section headers. Caught expected exception: File contains no section headers.
file: <???>, line: 1 file: <???>, line: 1
'No Section!\012' 'No Section!\n'
Testing query interface... Testing query interface...
[] []

View file

@ -6,9 +6,9 @@ Set-Cookie: chips=ahoy;
Set-Cookie: vienna=finger; Set-Cookie: vienna=finger;
chips 'ahoy' 'ahoy' chips 'ahoy' 'ahoy'
Set-Cookie: chips=ahoy; Set-Cookie: chips=ahoy;
<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=\012;'> <SimpleCookie: keebler='E=mc2; L="Loves"; fudge=\n;'>
Set-Cookie: keebler="E=mc2; L=\"Loves\"; fudge=\012;"; Set-Cookie: keebler="E=mc2; L=\"Loves\"; fudge=\012;";
keebler 'E=mc2; L="Loves"; fudge=\012;' 'E=mc2; L="Loves"; fudge=\012;' keebler 'E=mc2; L="Loves"; fudge=\n;' 'E=mc2; L="Loves"; fudge=\n;'
Set-Cookie: keebler="E=mc2; L=\"Loves\"; fudge=\012;"; Set-Cookie: keebler="E=mc2; L=\"Loves\"; fudge=\012;";
Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme; Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme;

View file

@ -1,26 +1,26 @@
test_tokenize test_tokenize
1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\012" 1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\n"
2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \012' 2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \n'
3,0-3,1: NL '\012' 3,0-3,1: NL '\n'
4,0-4,11: COMMENT '# Comments\012' 4,0-4,11: COMMENT '# Comments\n'
5,0-5,3: STRING '"#"' 5,0-5,3: STRING '"#"'
5,3-5,4: NEWLINE '\012' 5,3-5,4: NEWLINE '\n'
6,0-6,3: COMMENT "#'\012" 6,0-6,3: COMMENT "#'\n"
7,0-7,3: COMMENT '#"\012' 7,0-7,3: COMMENT '#"\n'
8,0-8,3: COMMENT '#\\\012' 8,0-8,3: COMMENT '#\\\n'
9,7-9,9: COMMENT '#\012' 9,7-9,9: COMMENT '#\n'
10,4-10,10: COMMENT '# abc\012' 10,4-10,10: COMMENT '# abc\n'
11,0-12,4: STRING "'''#\012#'''" 11,0-12,4: STRING "'''#\n#'''"
12,4-12,5: NEWLINE '\012' 12,4-12,5: NEWLINE '\n'
13,0-13,1: NL '\012' 13,0-13,1: NL '\n'
14,0-14,1: NAME 'x' 14,0-14,1: NAME 'x'
14,2-14,3: OP '=' 14,2-14,3: OP '='
14,4-14,5: NUMBER '1' 14,4-14,5: NUMBER '1'
14,7-14,8: COMMENT '#' 14,7-14,8: COMMENT '#'
14,8-14,9: NEWLINE '\012' 14,8-14,9: NEWLINE '\n'
15,0-15,1: NL '\012' 15,0-15,1: NL '\n'
16,0-16,25: COMMENT '# Balancing continuation\012' 16,0-16,25: COMMENT '# Balancing continuation\n'
17,0-17,1: NL '\012' 17,0-17,1: NL '\n'
18,0-18,1: NAME 'a' 18,0-18,1: NAME 'a'
18,2-18,3: OP '=' 18,2-18,3: OP '='
18,4-18,5: OP '(' 18,4-18,5: OP '('
@ -28,12 +28,12 @@ test_tokenize
18,6-18,7: OP ',' 18,6-18,7: OP ','
18,8-18,9: NUMBER '4' 18,8-18,9: NUMBER '4'
18,9-18,10: OP ',' 18,9-18,10: OP ','
18,10-18,11: NL '\012' 18,10-18,11: NL '\n'
19,2-19,3: NUMBER '5' 19,2-19,3: NUMBER '5'
19,3-19,4: OP ',' 19,3-19,4: OP ','
19,5-19,6: NUMBER '6' 19,5-19,6: NUMBER '6'
19,6-19,7: OP ')' 19,6-19,7: OP ')'
19,7-19,8: NEWLINE '\012' 19,7-19,8: NEWLINE '\n'
20,0-20,1: NAME 'y' 20,0-20,1: NAME 'y'
20,2-20,3: OP '=' 20,2-20,3: OP '='
20,4-20,5: OP '[' 20,4-20,5: OP '['
@ -41,10 +41,10 @@ test_tokenize
20,6-20,7: OP ',' 20,6-20,7: OP ','
20,8-20,9: NUMBER '4' 20,8-20,9: NUMBER '4'
20,9-20,10: OP ',' 20,9-20,10: OP ','
20,10-20,11: NL '\012' 20,10-20,11: NL '\n'
21,2-21,3: NUMBER '5' 21,2-21,3: NUMBER '5'
21,3-21,4: OP ']' 21,3-21,4: OP ']'
21,4-21,5: NEWLINE '\012' 21,4-21,5: NEWLINE '\n'
22,0-22,1: NAME 'z' 22,0-22,1: NAME 'z'
22,2-22,3: OP '=' 22,2-22,3: OP '='
22,4-22,5: OP '{' 22,4-22,5: OP '{'
@ -52,12 +52,12 @@ test_tokenize
22,8-22,9: OP ':' 22,8-22,9: OP ':'
22,9-22,10: NUMBER '5' 22,9-22,10: NUMBER '5'
22,10-22,11: OP ',' 22,10-22,11: OP ','
22,11-22,12: NL '\012' 22,11-22,12: NL '\n'
23,2-23,5: STRING "'b'" 23,2-23,5: STRING "'b'"
23,5-23,6: OP ':' 23,5-23,6: OP ':'
23,6-23,7: NUMBER '6' 23,6-23,7: NUMBER '6'
23,7-23,8: OP '}' 23,7-23,8: OP '}'
23,8-23,9: NEWLINE '\012' 23,8-23,9: NEWLINE '\n'
24,0-24,1: NAME 'x' 24,0-24,1: NAME 'x'
24,2-24,3: OP '=' 24,2-24,3: OP '='
24,4-24,5: OP '(' 24,4-24,5: OP '('
@ -74,151 +74,151 @@ test_tokenize
24,20-24,21: OP '-' 24,20-24,21: OP '-'
24,22-24,23: NAME 'a' 24,22-24,23: NAME 'a'
24,23-24,24: OP '[' 24,23-24,24: OP '['
24,24-24,25: NL '\012' 24,24-24,25: NL '\n'
25,3-25,4: NUMBER '3' 25,3-25,4: NUMBER '3'
25,5-25,6: OP ']' 25,5-25,6: OP ']'
25,6-25,7: NL '\012' 25,6-25,7: NL '\n'
26,3-26,4: OP '-' 26,3-26,4: OP '-'
26,5-26,6: NAME 'x' 26,5-26,6: NAME 'x'
26,7-26,8: OP '+' 26,7-26,8: OP '+'
26,9-26,12: NAME 'len' 26,9-26,12: NAME 'len'
26,12-26,13: OP '(' 26,12-26,13: OP '('
26,13-26,14: OP '{' 26,13-26,14: OP '{'
26,14-26,15: NL '\012' 26,14-26,15: NL '\n'
27,3-27,4: OP '}' 27,3-27,4: OP '}'
27,4-27,5: NL '\012' 27,4-27,5: NL '\n'
28,4-28,5: OP ')' 28,4-28,5: OP ')'
28,5-28,6: NL '\012' 28,5-28,6: NL '\n'
29,2-29,3: OP ')' 29,2-29,3: OP ')'
29,3-29,4: NEWLINE '\012' 29,3-29,4: NEWLINE '\n'
30,0-30,1: NL '\012' 30,0-30,1: NL '\n'
31,0-31,37: COMMENT '# Backslash means line continuation:\012' 31,0-31,37: COMMENT '# Backslash means line continuation:\n'
32,0-32,1: NAME 'x' 32,0-32,1: NAME 'x'
32,2-32,3: OP '=' 32,2-32,3: OP '='
32,4-32,5: NUMBER '1' 32,4-32,5: NUMBER '1'
33,0-33,1: OP '+' 33,0-33,1: OP '+'
33,2-33,3: NUMBER '1' 33,2-33,3: NUMBER '1'
33,3-33,4: NEWLINE '\012' 33,3-33,4: NEWLINE '\n'
34,0-34,1: NL '\012' 34,0-34,1: NL '\n'
35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\012' 35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\n'
36,0-36,1: NAME 'x' 36,0-36,1: NAME 'x'
36,2-36,3: OP '=' 36,2-36,3: OP '='
36,4-36,5: NUMBER '0' 36,4-36,5: NUMBER '0'
36,5-36,6: NEWLINE '\012' 36,5-36,6: NEWLINE '\n'
37,0-37,1: NL '\012' 37,0-37,1: NL '\n'
38,0-38,20: COMMENT '# Ordinary integers\012' 38,0-38,20: COMMENT '# Ordinary integers\n'
39,0-39,4: NUMBER '0xff' 39,0-39,4: NUMBER '0xff'
39,5-39,7: OP '<>' 39,5-39,7: OP '<>'
39,8-39,11: NUMBER '255' 39,8-39,11: NUMBER '255'
39,11-39,12: NEWLINE '\012' 39,11-39,12: NEWLINE '\n'
40,0-40,4: NUMBER '0377' 40,0-40,4: NUMBER '0377'
40,5-40,7: OP '<>' 40,5-40,7: OP '<>'
40,8-40,11: NUMBER '255' 40,8-40,11: NUMBER '255'
40,11-40,12: NEWLINE '\012' 40,11-40,12: NEWLINE '\n'
41,0-41,10: NUMBER '2147483647' 41,0-41,10: NUMBER '2147483647'
41,13-41,15: OP '!=' 41,13-41,15: OP '!='
41,16-41,28: NUMBER '017777777777' 41,16-41,28: NUMBER '017777777777'
41,28-41,29: NEWLINE '\012' 41,28-41,29: NEWLINE '\n'
42,0-42,1: OP '-' 42,0-42,1: OP '-'
42,1-42,11: NUMBER '2147483647' 42,1-42,11: NUMBER '2147483647'
42,11-42,12: OP '-' 42,11-42,12: OP '-'
42,12-42,13: NUMBER '1' 42,12-42,13: NUMBER '1'
42,14-42,16: OP '!=' 42,14-42,16: OP '!='
42,17-42,29: NUMBER '020000000000' 42,17-42,29: NUMBER '020000000000'
42,29-42,30: NEWLINE '\012' 42,29-42,30: NEWLINE '\n'
43,0-43,12: NUMBER '037777777777' 43,0-43,12: NUMBER '037777777777'
43,13-43,15: OP '!=' 43,13-43,15: OP '!='
43,16-43,17: OP '-' 43,16-43,17: OP '-'
43,17-43,18: NUMBER '1' 43,17-43,18: NUMBER '1'
43,18-43,19: NEWLINE '\012' 43,18-43,19: NEWLINE '\n'
44,0-44,10: NUMBER '0xffffffff' 44,0-44,10: NUMBER '0xffffffff'
44,11-44,13: OP '!=' 44,11-44,13: OP '!='
44,14-44,15: OP '-' 44,14-44,15: OP '-'
44,15-44,16: NUMBER '1' 44,15-44,16: NUMBER '1'
44,16-44,17: NEWLINE '\012' 44,16-44,17: NEWLINE '\n'
45,0-45,1: NL '\012' 45,0-45,1: NL '\n'
46,0-46,16: COMMENT '# Long integers\012' 46,0-46,16: COMMENT '# Long integers\n'
47,0-47,1: NAME 'x' 47,0-47,1: NAME 'x'
47,2-47,3: OP '=' 47,2-47,3: OP '='
47,4-47,6: NUMBER '0L' 47,4-47,6: NUMBER '0L'
47,6-47,7: NEWLINE '\012' 47,6-47,7: NEWLINE '\n'
48,0-48,1: NAME 'x' 48,0-48,1: NAME 'x'
48,2-48,3: OP '=' 48,2-48,3: OP '='
48,4-48,6: NUMBER '0l' 48,4-48,6: NUMBER '0l'
48,6-48,7: NEWLINE '\012' 48,6-48,7: NEWLINE '\n'
49,0-49,1: NAME 'x' 49,0-49,1: NAME 'x'
49,2-49,3: OP '=' 49,2-49,3: OP '='
49,4-49,23: NUMBER '0xffffffffffffffffL' 49,4-49,23: NUMBER '0xffffffffffffffffL'
49,23-49,24: NEWLINE '\012' 49,23-49,24: NEWLINE '\n'
50,0-50,1: NAME 'x' 50,0-50,1: NAME 'x'
50,2-50,3: OP '=' 50,2-50,3: OP '='
50,4-50,23: NUMBER '0xffffffffffffffffl' 50,4-50,23: NUMBER '0xffffffffffffffffl'
50,23-50,24: NEWLINE '\012' 50,23-50,24: NEWLINE '\n'
51,0-51,1: NAME 'x' 51,0-51,1: NAME 'x'
51,2-51,3: OP '=' 51,2-51,3: OP '='
51,4-51,23: NUMBER '077777777777777777L' 51,4-51,23: NUMBER '077777777777777777L'
51,23-51,24: NEWLINE '\012' 51,23-51,24: NEWLINE '\n'
52,0-52,1: NAME 'x' 52,0-52,1: NAME 'x'
52,2-52,3: OP '=' 52,2-52,3: OP '='
52,4-52,23: NUMBER '077777777777777777l' 52,4-52,23: NUMBER '077777777777777777l'
52,23-52,24: NEWLINE '\012' 52,23-52,24: NEWLINE '\n'
53,0-53,1: NAME 'x' 53,0-53,1: NAME 'x'
53,2-53,3: OP '=' 53,2-53,3: OP '='
53,4-53,35: NUMBER '123456789012345678901234567890L' 53,4-53,35: NUMBER '123456789012345678901234567890L'
53,35-53,36: NEWLINE '\012' 53,35-53,36: NEWLINE '\n'
54,0-54,1: NAME 'x' 54,0-54,1: NAME 'x'
54,2-54,3: OP '=' 54,2-54,3: OP '='
54,4-54,35: NUMBER '123456789012345678901234567890l' 54,4-54,35: NUMBER '123456789012345678901234567890l'
54,35-54,36: NEWLINE '\012' 54,35-54,36: NEWLINE '\n'
55,0-55,1: NL '\012' 55,0-55,1: NL '\n'
56,0-56,25: COMMENT '# Floating-point numbers\012' 56,0-56,25: COMMENT '# Floating-point numbers\n'
57,0-57,1: NAME 'x' 57,0-57,1: NAME 'x'
57,2-57,3: OP '=' 57,2-57,3: OP '='
57,4-57,8: NUMBER '3.14' 57,4-57,8: NUMBER '3.14'
57,8-57,9: NEWLINE '\012' 57,8-57,9: NEWLINE '\n'
58,0-58,1: NAME 'x' 58,0-58,1: NAME 'x'
58,2-58,3: OP '=' 58,2-58,3: OP '='
58,4-58,8: NUMBER '314.' 58,4-58,8: NUMBER '314.'
58,8-58,9: NEWLINE '\012' 58,8-58,9: NEWLINE '\n'
59,0-59,1: NAME 'x' 59,0-59,1: NAME 'x'
59,2-59,3: OP '=' 59,2-59,3: OP '='
59,4-59,9: NUMBER '0.314' 59,4-59,9: NUMBER '0.314'
59,9-59,10: NEWLINE '\012' 59,9-59,10: NEWLINE '\n'
60,0-60,18: COMMENT '# XXX x = 000.314\012' 60,0-60,18: COMMENT '# XXX x = 000.314\n'
61,0-61,1: NAME 'x' 61,0-61,1: NAME 'x'
61,2-61,3: OP '=' 61,2-61,3: OP '='
61,4-61,8: NUMBER '.314' 61,4-61,8: NUMBER '.314'
61,8-61,9: NEWLINE '\012' 61,8-61,9: NEWLINE '\n'
62,0-62,1: NAME 'x' 62,0-62,1: NAME 'x'
62,2-62,3: OP '=' 62,2-62,3: OP '='
62,4-62,8: NUMBER '3e14' 62,4-62,8: NUMBER '3e14'
62,8-62,9: NEWLINE '\012' 62,8-62,9: NEWLINE '\n'
63,0-63,1: NAME 'x' 63,0-63,1: NAME 'x'
63,2-63,3: OP '=' 63,2-63,3: OP '='
63,4-63,8: NUMBER '3E14' 63,4-63,8: NUMBER '3E14'
63,8-63,9: NEWLINE '\012' 63,8-63,9: NEWLINE '\n'
64,0-64,1: NAME 'x' 64,0-64,1: NAME 'x'
64,2-64,3: OP '=' 64,2-64,3: OP '='
64,4-64,9: NUMBER '3e-14' 64,4-64,9: NUMBER '3e-14'
64,9-64,10: NEWLINE '\012' 64,9-64,10: NEWLINE '\n'
65,0-65,1: NAME 'x' 65,0-65,1: NAME 'x'
65,2-65,3: OP '=' 65,2-65,3: OP '='
65,4-65,9: NUMBER '3e+14' 65,4-65,9: NUMBER '3e+14'
65,9-65,10: NEWLINE '\012' 65,9-65,10: NEWLINE '\n'
66,0-66,1: NAME 'x' 66,0-66,1: NAME 'x'
66,2-66,3: OP '=' 66,2-66,3: OP '='
66,4-66,9: NUMBER '3.e14' 66,4-66,9: NUMBER '3.e14'
66,9-66,10: NEWLINE '\012' 66,9-66,10: NEWLINE '\n'
67,0-67,1: NAME 'x' 67,0-67,1: NAME 'x'
67,2-67,3: OP '=' 67,2-67,3: OP '='
67,4-67,9: NUMBER '.3e14' 67,4-67,9: NUMBER '.3e14'
67,9-67,10: NEWLINE '\012' 67,9-67,10: NEWLINE '\n'
68,0-68,1: NAME 'x' 68,0-68,1: NAME 'x'
68,2-68,3: OP '=' 68,2-68,3: OP '='
68,4-68,9: NUMBER '3.1e4' 68,4-68,9: NUMBER '3.1e4'
68,9-68,10: NEWLINE '\012' 68,9-68,10: NEWLINE '\n'
69,0-69,1: NL '\012' 69,0-69,1: NL '\n'
70,0-70,18: COMMENT '# String literals\012' 70,0-70,18: COMMENT '# String literals\n'
71,0-71,1: NAME 'x' 71,0-71,1: NAME 'x'
71,2-71,3: OP '=' 71,2-71,3: OP '='
71,4-71,6: STRING "''" 71,4-71,6: STRING "''"
@ -227,7 +227,7 @@ test_tokenize
71,10-71,11: OP '=' 71,10-71,11: OP '='
71,12-71,14: STRING '""' 71,12-71,14: STRING '""'
71,14-71,15: OP ';' 71,14-71,15: OP ';'
71,15-71,16: NEWLINE '\012' 71,15-71,16: NEWLINE '\n'
72,0-72,1: NAME 'x' 72,0-72,1: NAME 'x'
72,2-72,3: OP '=' 72,2-72,3: OP '='
72,4-72,8: STRING "'\\''" 72,4-72,8: STRING "'\\''"
@ -236,7 +236,7 @@ test_tokenize
72,12-72,13: OP '=' 72,12-72,13: OP '='
72,14-72,17: STRING '"\'"' 72,14-72,17: STRING '"\'"'
72,17-72,18: OP ';' 72,17-72,18: OP ';'
72,18-72,19: NEWLINE '\012' 72,18-72,19: NEWLINE '\n'
73,0-73,1: NAME 'x' 73,0-73,1: NAME 'x'
73,2-73,3: OP '=' 73,2-73,3: OP '='
73,4-73,7: STRING '\'"\'' 73,4-73,7: STRING '\'"\''
@ -245,82 +245,82 @@ test_tokenize
73,11-73,12: OP '=' 73,11-73,12: OP '='
73,13-73,17: STRING '"\\""' 73,13-73,17: STRING '"\\""'
73,17-73,18: OP ';' 73,17-73,18: OP ';'
73,18-73,19: NEWLINE '\012' 73,18-73,19: NEWLINE '\n'
74,0-74,1: NAME 'x' 74,0-74,1: NAME 'x'
74,2-74,3: OP '=' 74,2-74,3: OP '='
74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"' 74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
74,32-74,33: NEWLINE '\012' 74,32-74,33: NEWLINE '\n'
75,0-75,1: NAME 'y' 75,0-75,1: NAME 'y'
75,2-75,3: OP '=' 75,2-75,3: OP '='
75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\'' 75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
75,31-75,32: NEWLINE '\012' 75,31-75,32: NEWLINE '\n'
76,0-76,1: NAME 'x' 76,0-76,1: NAME 'x'
76,2-76,3: OP '=' 76,2-76,3: OP '='
76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"' 76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
76,32-76,33: NEWLINE '\012' 76,32-76,33: NEWLINE '\n'
77,0-77,1: NAME 'y' 77,0-77,1: NAME 'y'
77,2-77,3: OP '=' 77,2-77,3: OP '='
77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\'' 77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
77,31-77,32: NEWLINE '\012' 77,31-77,32: NEWLINE '\n'
78,0-78,1: NAME 'x' 78,0-78,1: NAME 'x'
78,2-78,3: OP '=' 78,2-78,3: OP '='
78,4-83,3: STRING '"""\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012"""' 78,4-83,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
83,3-83,4: NEWLINE '\012' 83,3-83,4: NEWLINE '\n'
84,0-84,1: NAME 'y' 84,0-84,1: NAME 'y'
84,2-84,3: OP '=' 84,2-84,3: OP '='
84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\'' 84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
84,63-84,64: NEWLINE '\012' 84,63-84,64: NEWLINE '\n'
85,0-85,1: NAME 'y' 85,0-85,1: NAME 'y'
85,2-85,3: OP '=' 85,2-85,3: OP '='
85,4-90,3: STRING '\'\'\'\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012\'\'\'' 85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
90,3-90,4: OP ';' 90,3-90,4: OP ';'
90,4-90,5: NEWLINE '\012' 90,4-90,5: NEWLINE '\n'
91,0-91,1: NAME 'y' 91,0-91,1: NAME 'y'
91,2-91,3: OP '=' 91,2-91,3: OP '='
91,4-96,1: STRING '"\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \'lazy\' dog.\\n\\\012"' 91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
96,1-96,2: OP ';' 96,1-96,2: OP ';'
96,2-96,3: NEWLINE '\012' 96,2-96,3: NEWLINE '\n'
97,0-97,1: NAME 'y' 97,0-97,1: NAME 'y'
97,2-97,3: OP '=' 97,2-97,3: OP '='
97,4-102,1: STRING '\'\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \\\'lazy\\\' dog.\\n\\\012\'' 97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
102,1-102,2: OP ';' 102,1-102,2: OP ';'
102,2-102,3: NEWLINE '\012' 102,2-102,3: NEWLINE '\n'
103,0-103,1: NAME 'x' 103,0-103,1: NAME 'x'
103,2-103,3: OP '=' 103,2-103,3: OP '='
103,4-103,9: STRING "r'\\\\'" 103,4-103,9: STRING "r'\\\\'"
103,10-103,11: OP '+' 103,10-103,11: OP '+'
103,12-103,17: STRING "R'\\\\'" 103,12-103,17: STRING "R'\\\\'"
103,17-103,18: NEWLINE '\012' 103,17-103,18: NEWLINE '\n'
104,0-104,1: NAME 'x' 104,0-104,1: NAME 'x'
104,2-104,3: OP '=' 104,2-104,3: OP '='
104,4-104,9: STRING "r'\\''" 104,4-104,9: STRING "r'\\''"
104,10-104,11: OP '+' 104,10-104,11: OP '+'
104,12-104,14: STRING "''" 104,12-104,14: STRING "''"
104,14-104,15: NEWLINE '\012' 104,14-104,15: NEWLINE '\n'
105,0-105,1: NAME 'y' 105,0-105,1: NAME 'y'
105,2-105,3: OP '=' 105,2-105,3: OP '='
105,4-107,6: STRING "r'''\012foo bar \\\\\012baz'''" 105,4-107,6: STRING "r'''\nfoo bar \\\\\nbaz'''"
107,7-107,8: OP '+' 107,7-107,8: OP '+'
107,9-108,6: STRING "R'''\012foo'''" 107,9-108,6: STRING "R'''\nfoo'''"
108,6-108,7: NEWLINE '\012' 108,6-108,7: NEWLINE '\n'
109,0-109,1: NAME 'y' 109,0-109,1: NAME 'y'
109,2-109,3: OP '=' 109,2-109,3: OP '='
109,4-111,3: STRING 'r"""foo\012bar \\\\ baz\012"""' 109,4-111,3: STRING 'r"""foo\nbar \\\\ baz\n"""'
111,4-111,5: OP '+' 111,4-111,5: OP '+'
111,6-112,3: STRING "R'''spam\012'''" 111,6-112,3: STRING "R'''spam\n'''"
112,3-112,4: NEWLINE '\012' 112,3-112,4: NEWLINE '\n'
113,0-113,1: NAME 'x' 113,0-113,1: NAME 'x'
113,2-113,3: OP '=' 113,2-113,3: OP '='
113,4-113,10: STRING "u'abc'" 113,4-113,10: STRING "u'abc'"
113,11-113,12: OP '+' 113,11-113,12: OP '+'
113,13-113,19: STRING "U'ABC'" 113,13-113,19: STRING "U'ABC'"
113,19-113,20: NEWLINE '\012' 113,19-113,20: NEWLINE '\n'
114,0-114,1: NAME 'y' 114,0-114,1: NAME 'y'
114,2-114,3: OP '=' 114,2-114,3: OP '='
114,4-114,10: STRING 'u"abc"' 114,4-114,10: STRING 'u"abc"'
114,11-114,12: OP '+' 114,11-114,12: OP '+'
114,13-114,19: STRING 'U"ABC"' 114,13-114,19: STRING 'U"ABC"'
114,19-114,20: NEWLINE '\012' 114,19-114,20: NEWLINE '\n'
115,0-115,1: NAME 'x' 115,0-115,1: NAME 'x'
115,2-115,3: OP '=' 115,2-115,3: OP '='
115,4-115,11: STRING "ur'abc'" 115,4-115,11: STRING "ur'abc'"
@ -330,7 +330,7 @@ test_tokenize
115,24-115,31: STRING "uR'ABC'" 115,24-115,31: STRING "uR'ABC'"
115,32-115,33: OP '+' 115,32-115,33: OP '+'
115,34-115,41: STRING "UR'ABC'" 115,34-115,41: STRING "UR'ABC'"
115,41-115,42: NEWLINE '\012' 115,41-115,42: NEWLINE '\n'
116,0-116,1: NAME 'y' 116,0-116,1: NAME 'y'
116,2-116,3: OP '=' 116,2-116,3: OP '='
116,4-116,11: STRING 'ur"abc"' 116,4-116,11: STRING 'ur"abc"'
@ -340,106 +340,106 @@ test_tokenize
116,24-116,31: STRING 'uR"ABC"' 116,24-116,31: STRING 'uR"ABC"'
116,32-116,33: OP '+' 116,32-116,33: OP '+'
116,34-116,41: STRING 'UR"ABC"' 116,34-116,41: STRING 'UR"ABC"'
116,41-116,42: NEWLINE '\012' 116,41-116,42: NEWLINE '\n'
117,0-117,1: NAME 'x' 117,0-117,1: NAME 'x'
117,2-117,3: OP '=' 117,2-117,3: OP '='
117,4-117,10: STRING "ur'\\\\'" 117,4-117,10: STRING "ur'\\\\'"
117,11-117,12: OP '+' 117,11-117,12: OP '+'
117,13-117,19: STRING "UR'\\\\'" 117,13-117,19: STRING "UR'\\\\'"
117,19-117,20: NEWLINE '\012' 117,19-117,20: NEWLINE '\n'
118,0-118,1: NAME 'x' 118,0-118,1: NAME 'x'
118,2-118,3: OP '=' 118,2-118,3: OP '='
118,4-118,10: STRING "ur'\\''" 118,4-118,10: STRING "ur'\\''"
118,11-118,12: OP '+' 118,11-118,12: OP '+'
118,13-118,15: STRING "''" 118,13-118,15: STRING "''"
118,15-118,16: NEWLINE '\012' 118,15-118,16: NEWLINE '\n'
119,0-119,1: NAME 'y' 119,0-119,1: NAME 'y'
119,2-119,3: OP '=' 119,2-119,3: OP '='
119,4-121,6: STRING "ur'''\012foo bar \\\\\012baz'''" 119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''"
121,7-121,8: OP '+' 121,7-121,8: OP '+'
121,9-122,6: STRING "UR'''\012foo'''" 121,9-122,6: STRING "UR'''\nfoo'''"
122,6-122,7: NEWLINE '\012' 122,6-122,7: NEWLINE '\n'
123,0-123,1: NAME 'y' 123,0-123,1: NAME 'y'
123,2-123,3: OP '=' 123,2-123,3: OP '='
123,4-125,3: STRING 'Ur"""foo\012bar \\\\ baz\012"""' 123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""'
125,4-125,5: OP '+' 125,4-125,5: OP '+'
125,6-126,3: STRING "uR'''spam\012'''" 125,6-126,3: STRING "uR'''spam\n'''"
126,3-126,4: NEWLINE '\012' 126,3-126,4: NEWLINE '\n'
127,0-127,1: NL '\012' 127,0-127,1: NL '\n'
128,0-128,14: COMMENT '# Indentation\012' 128,0-128,14: COMMENT '# Indentation\n'
129,0-129,2: NAME 'if' 129,0-129,2: NAME 'if'
129,3-129,4: NUMBER '1' 129,3-129,4: NUMBER '1'
129,4-129,5: OP ':' 129,4-129,5: OP ':'
129,5-129,6: NEWLINE '\012' 129,5-129,6: NEWLINE '\n'
130,0-130,4: INDENT ' ' 130,0-130,4: INDENT ' '
130,4-130,5: NAME 'x' 130,4-130,5: NAME 'x'
130,6-130,7: OP '=' 130,6-130,7: OP '='
130,8-130,9: NUMBER '2' 130,8-130,9: NUMBER '2'
130,9-130,10: NEWLINE '\012' 130,9-130,10: NEWLINE '\n'
131,0-131,0: DEDENT '' 131,0-131,0: DEDENT ''
131,0-131,2: NAME 'if' 131,0-131,2: NAME 'if'
131,3-131,4: NUMBER '1' 131,3-131,4: NUMBER '1'
131,4-131,5: OP ':' 131,4-131,5: OP ':'
131,5-131,6: NEWLINE '\012' 131,5-131,6: NEWLINE '\n'
132,0-132,8: INDENT ' ' 132,0-132,8: INDENT ' '
132,8-132,9: NAME 'x' 132,8-132,9: NAME 'x'
132,10-132,11: OP '=' 132,10-132,11: OP '='
132,12-132,13: NUMBER '2' 132,12-132,13: NUMBER '2'
132,13-132,14: NEWLINE '\012' 132,13-132,14: NEWLINE '\n'
133,0-133,0: DEDENT '' 133,0-133,0: DEDENT ''
133,0-133,2: NAME 'if' 133,0-133,2: NAME 'if'
133,3-133,4: NUMBER '1' 133,3-133,4: NUMBER '1'
133,4-133,5: OP ':' 133,4-133,5: OP ':'
133,5-133,6: NEWLINE '\012' 133,5-133,6: NEWLINE '\n'
134,0-134,4: INDENT ' ' 134,0-134,4: INDENT ' '
134,4-134,9: NAME 'while' 134,4-134,9: NAME 'while'
134,10-134,11: NUMBER '0' 134,10-134,11: NUMBER '0'
134,11-134,12: OP ':' 134,11-134,12: OP ':'
134,12-134,13: NEWLINE '\012' 134,12-134,13: NEWLINE '\n'
135,0-135,5: INDENT ' ' 135,0-135,5: INDENT ' '
135,5-135,7: NAME 'if' 135,5-135,7: NAME 'if'
135,8-135,9: NUMBER '0' 135,8-135,9: NUMBER '0'
135,9-135,10: OP ':' 135,9-135,10: OP ':'
135,10-135,11: NEWLINE '\012' 135,10-135,11: NEWLINE '\n'
136,0-136,11: INDENT ' ' 136,0-136,11: INDENT ' '
136,11-136,12: NAME 'x' 136,11-136,12: NAME 'x'
136,13-136,14: OP '=' 136,13-136,14: OP '='
136,15-136,16: NUMBER '2' 136,15-136,16: NUMBER '2'
136,16-136,17: NEWLINE '\012' 136,16-136,17: NEWLINE '\n'
137,5-137,5: DEDENT '' 137,5-137,5: DEDENT ''
137,5-137,6: NAME 'x' 137,5-137,6: NAME 'x'
137,7-137,8: OP '=' 137,7-137,8: OP '='
137,9-137,10: NUMBER '2' 137,9-137,10: NUMBER '2'
137,10-137,11: NEWLINE '\012' 137,10-137,11: NEWLINE '\n'
138,0-138,0: DEDENT '' 138,0-138,0: DEDENT ''
138,0-138,0: DEDENT '' 138,0-138,0: DEDENT ''
138,0-138,2: NAME 'if' 138,0-138,2: NAME 'if'
138,3-138,4: NUMBER '0' 138,3-138,4: NUMBER '0'
138,4-138,5: OP ':' 138,4-138,5: OP ':'
138,5-138,6: NEWLINE '\012' 138,5-138,6: NEWLINE '\n'
139,0-139,2: INDENT ' ' 139,0-139,2: INDENT ' '
139,2-139,4: NAME 'if' 139,2-139,4: NAME 'if'
139,5-139,6: NUMBER '2' 139,5-139,6: NUMBER '2'
139,6-139,7: OP ':' 139,6-139,7: OP ':'
139,7-139,8: NEWLINE '\012' 139,7-139,8: NEWLINE '\n'
140,0-140,3: INDENT ' ' 140,0-140,3: INDENT ' '
140,3-140,8: NAME 'while' 140,3-140,8: NAME 'while'
140,9-140,10: NUMBER '0' 140,9-140,10: NUMBER '0'
140,10-140,11: OP ':' 140,10-140,11: OP ':'
140,11-140,12: NEWLINE '\012' 140,11-140,12: NEWLINE '\n'
141,0-141,8: INDENT ' ' 141,0-141,8: INDENT ' '
141,8-141,10: NAME 'if' 141,8-141,10: NAME 'if'
141,11-141,12: NUMBER '1' 141,11-141,12: NUMBER '1'
141,12-141,13: OP ':' 141,12-141,13: OP ':'
141,13-141,14: NEWLINE '\012' 141,13-141,14: NEWLINE '\n'
142,0-142,10: INDENT ' ' 142,0-142,10: INDENT ' '
142,10-142,11: NAME 'x' 142,10-142,11: NAME 'x'
142,12-142,13: OP '=' 142,12-142,13: OP '='
142,14-142,15: NUMBER '2' 142,14-142,15: NUMBER '2'
142,15-142,16: NEWLINE '\012' 142,15-142,16: NEWLINE '\n'
143,0-143,1: NL '\012' 143,0-143,1: NL '\n'
144,0-144,12: COMMENT '# Operators\012' 144,0-144,12: COMMENT '# Operators\n'
145,0-145,1: NL '\012' 145,0-145,1: NL '\n'
146,0-146,0: DEDENT '' 146,0-146,0: DEDENT ''
146,0-146,0: DEDENT '' 146,0-146,0: DEDENT ''
146,0-146,0: DEDENT '' 146,0-146,0: DEDENT ''
@ -461,7 +461,7 @@ test_tokenize
146,22-146,23: OP ')' 146,22-146,23: OP ')'
146,23-146,24: OP ':' 146,23-146,24: OP ':'
146,25-146,29: NAME 'pass' 146,25-146,29: NAME 'pass'
146,29-146,30: NEWLINE '\012' 146,29-146,30: NEWLINE '\n'
147,0-147,3: NAME 'def' 147,0-147,3: NAME 'def'
147,4-147,8: NAME 'd01v' 147,4-147,8: NAME 'd01v'
147,8-147,9: OP '(' 147,8-147,9: OP '('
@ -477,8 +477,8 @@ test_tokenize
147,27-147,28: OP ')' 147,27-147,28: OP ')'
147,28-147,29: OP ':' 147,28-147,29: OP ':'
147,30-147,34: NAME 'pass' 147,30-147,34: NAME 'pass'
147,34-147,35: NEWLINE '\012' 147,34-147,35: NEWLINE '\n'
148,0-148,1: NL '\012' 148,0-148,1: NL '\n'
149,0-149,1: OP '(' 149,0-149,1: OP '('
149,1-149,2: NAME 'x' 149,1-149,2: NAME 'x'
149,2-149,3: OP ',' 149,2-149,3: OP ','
@ -498,9 +498,9 @@ test_tokenize
149,25-149,26: NUMBER '2' 149,25-149,26: NUMBER '2'
149,26-149,27: OP '}' 149,26-149,27: OP '}'
149,27-149,28: OP ')' 149,27-149,28: OP ')'
149,28-149,29: NEWLINE '\012' 149,28-149,29: NEWLINE '\n'
150,0-150,1: NL '\012' 150,0-150,1: NL '\n'
151,0-151,13: COMMENT '# comparison\012' 151,0-151,13: COMMENT '# comparison\n'
152,0-152,2: NAME 'if' 152,0-152,2: NAME 'if'
152,3-152,4: NUMBER '1' 152,3-152,4: NUMBER '1'
152,5-152,6: OP '<' 152,5-152,6: OP '<'
@ -529,29 +529,29 @@ test_tokenize
152,64-152,65: NUMBER '1' 152,64-152,65: NUMBER '1'
152,65-152,66: OP ':' 152,65-152,66: OP ':'
152,67-152,71: NAME 'pass' 152,67-152,71: NAME 'pass'
152,71-152,72: NEWLINE '\012' 152,71-152,72: NEWLINE '\n'
153,0-153,1: NL '\012' 153,0-153,1: NL '\n'
154,0-154,9: COMMENT '# binary\012' 154,0-154,9: COMMENT '# binary\n'
155,0-155,1: NAME 'x' 155,0-155,1: NAME 'x'
155,2-155,3: OP '=' 155,2-155,3: OP '='
155,4-155,5: NUMBER '1' 155,4-155,5: NUMBER '1'
155,6-155,7: OP '&' 155,6-155,7: OP '&'
155,8-155,9: NUMBER '1' 155,8-155,9: NUMBER '1'
155,9-155,10: NEWLINE '\012' 155,9-155,10: NEWLINE '\n'
156,0-156,1: NAME 'x' 156,0-156,1: NAME 'x'
156,2-156,3: OP '=' 156,2-156,3: OP '='
156,4-156,5: NUMBER '1' 156,4-156,5: NUMBER '1'
156,6-156,7: OP '^' 156,6-156,7: OP '^'
156,8-156,9: NUMBER '1' 156,8-156,9: NUMBER '1'
156,9-156,10: NEWLINE '\012' 156,9-156,10: NEWLINE '\n'
157,0-157,1: NAME 'x' 157,0-157,1: NAME 'x'
157,2-157,3: OP '=' 157,2-157,3: OP '='
157,4-157,5: NUMBER '1' 157,4-157,5: NUMBER '1'
157,6-157,7: OP '|' 157,6-157,7: OP '|'
157,8-157,9: NUMBER '1' 157,8-157,9: NUMBER '1'
157,9-157,10: NEWLINE '\012' 157,9-157,10: NEWLINE '\n'
158,0-158,1: NL '\012' 158,0-158,1: NL '\n'
159,0-159,8: COMMENT '# shift\012' 159,0-159,8: COMMENT '# shift\n'
160,0-160,1: NAME 'x' 160,0-160,1: NAME 'x'
160,2-160,3: OP '=' 160,2-160,3: OP '='
160,4-160,5: NUMBER '1' 160,4-160,5: NUMBER '1'
@ -559,9 +559,9 @@ test_tokenize
160,9-160,10: NUMBER '1' 160,9-160,10: NUMBER '1'
160,11-160,13: OP '>>' 160,11-160,13: OP '>>'
160,14-160,15: NUMBER '1' 160,14-160,15: NUMBER '1'
160,15-160,16: NEWLINE '\012' 160,15-160,16: NEWLINE '\n'
161,0-161,1: NL '\012' 161,0-161,1: NL '\n'
162,0-162,11: COMMENT '# additive\012' 162,0-162,11: COMMENT '# additive\n'
163,0-163,1: NAME 'x' 163,0-163,1: NAME 'x'
163,2-163,3: OP '=' 163,2-163,3: OP '='
163,4-163,5: NUMBER '1' 163,4-163,5: NUMBER '1'
@ -573,9 +573,9 @@ test_tokenize
163,16-163,17: NUMBER '1' 163,16-163,17: NUMBER '1'
163,18-163,19: OP '+' 163,18-163,19: OP '+'
163,20-163,21: NUMBER '1' 163,20-163,21: NUMBER '1'
163,21-163,22: NEWLINE '\012' 163,21-163,22: NEWLINE '\n'
164,0-164,1: NL '\012' 164,0-164,1: NL '\n'
165,0-165,17: COMMENT '# multiplicative\012' 165,0-165,17: COMMENT '# multiplicative\n'
166,0-166,1: NAME 'x' 166,0-166,1: NAME 'x'
166,2-166,3: OP '=' 166,2-166,3: OP '='
166,4-166,5: NUMBER '1' 166,4-166,5: NUMBER '1'
@ -585,9 +585,9 @@ test_tokenize
166,12-166,13: NUMBER '1' 166,12-166,13: NUMBER '1'
166,14-166,15: OP '%' 166,14-166,15: OP '%'
166,16-166,17: NUMBER '1' 166,16-166,17: NUMBER '1'
166,17-166,18: NEWLINE '\012' 166,17-166,18: NEWLINE '\n'
167,0-167,1: NL '\012' 167,0-167,1: NL '\n'
168,0-168,8: COMMENT '# unary\012' 168,0-168,8: COMMENT '# unary\n'
169,0-169,1: NAME 'x' 169,0-169,1: NAME 'x'
169,2-169,3: OP '=' 169,2-169,3: OP '='
169,4-169,5: OP '~' 169,4-169,5: OP '~'
@ -603,7 +603,7 @@ test_tokenize
169,23-169,24: OP '^' 169,23-169,24: OP '^'
169,25-169,26: OP '-' 169,25-169,26: OP '-'
169,26-169,27: NUMBER '1' 169,26-169,27: NUMBER '1'
169,27-169,28: NEWLINE '\012' 169,27-169,28: NEWLINE '\n'
170,0-170,1: NAME 'x' 170,0-170,1: NAME 'x'
170,2-170,3: OP '=' 170,2-170,3: OP '='
170,4-170,5: OP '-' 170,4-170,5: OP '-'
@ -623,14 +623,14 @@ test_tokenize
170,22-170,23: NUMBER '1' 170,22-170,23: NUMBER '1'
170,23-170,24: OP '*' 170,23-170,24: OP '*'
170,24-170,25: NUMBER '1' 170,24-170,25: NUMBER '1'
170,25-170,26: NEWLINE '\012' 170,25-170,26: NEWLINE '\n'
171,0-171,1: NL '\012' 171,0-171,1: NL '\n'
172,0-172,11: COMMENT '# selector\012' 172,0-172,11: COMMENT '# selector\n'
173,0-173,6: NAME 'import' 173,0-173,6: NAME 'import'
173,7-173,10: NAME 'sys' 173,7-173,10: NAME 'sys'
173,10-173,11: OP ',' 173,10-173,11: OP ','
173,12-173,16: NAME 'time' 173,12-173,16: NAME 'time'
173,16-173,17: NEWLINE '\012' 173,16-173,17: NEWLINE '\n'
174,0-174,1: NAME 'x' 174,0-174,1: NAME 'x'
174,2-174,3: OP '=' 174,2-174,3: OP '='
174,4-174,7: NAME 'sys' 174,4-174,7: NAME 'sys'
@ -643,6 +643,6 @@ test_tokenize
174,24-174,28: NAME 'time' 174,24-174,28: NAME 'time'
174,28-174,29: OP '(' 174,28-174,29: OP '('
174,29-174,30: OP ')' 174,29-174,30: OP ')'
174,30-174,31: NEWLINE '\012' 174,30-174,31: NEWLINE '\n'
175,0-175,1: NL '\012' 175,0-175,1: NL '\n'
176,0-176,0: ENDMARKER '' 176,0-176,0: ENDMARKER ''

View file

@ -335,8 +335,14 @@ string_print(PyStringObject *op, FILE *fp, int flags)
c = op->ob_sval[i]; c = op->ob_sval[i];
if (c == quote || c == '\\') if (c == quote || c == '\\')
fprintf(fp, "\\%c", c); fprintf(fp, "\\%c", c);
else if (c < ' ' || c >= 0177) else if (c == '\t')
fprintf(fp, "\\%03o", c & 0377); fprintf(fp, "\\t");
else if (c == '\n')
fprintf(fp, "\\n");
else if (c == '\r')
fprintf(fp, "\\r");
else if (c < ' ' || c >= 0x7f)
fprintf(fp, "\\x%02x", c & 0xff);
else else
fputc(c, fp); fputc(c, fp);
} }
@ -374,10 +380,15 @@ string_repr(register PyStringObject *op)
c = op->ob_sval[i]; c = op->ob_sval[i];
if (c == quote || c == '\\') if (c == quote || c == '\\')
*p++ = '\\', *p++ = c; *p++ = '\\', *p++ = c;
else if (c < ' ' || c >= 0177) { else if (c == '\t')
sprintf(p, "\\%03o", c & 0377); *p++ = '\\', *p++ = 't';
while (*p != '\0') else if (c == '\n')
p++; *p++ = '\\', *p++ = 'n';
else if (c == '\r')
*p++ = '\\', *p++ = 'r';
else if (c < ' ' || c >= 0x7f) {
sprintf(p, "\\x%02x", c & 0xff);
p += 4;
} }
else else
*p++ = c; *p++ = c;

View file

@ -1343,7 +1343,7 @@ PyObject *unicodeescape_string(const Py_UNICODE *s,
char *p; char *p;
char *q; char *q;
static const char *hexdigit = "0123456789ABCDEF"; static const char *hexdigit = "0123456789abcdef";
repr = PyString_FromStringAndSize(NULL, 2 + 6*size + 1); repr = PyString_FromStringAndSize(NULL, 2 + 6*size + 1);
if (repr == NULL) if (repr == NULL)
@ -1372,12 +1372,25 @@ PyObject *unicodeescape_string(const Py_UNICODE *s,
*p++ = hexdigit[(ch >> 4) & 0xf]; *p++ = hexdigit[(ch >> 4) & 0xf];
*p++ = hexdigit[ch & 15]; *p++ = hexdigit[ch & 15];
} }
/* Map non-printable US ASCII to '\ooo' */ /* Map special whitespace to '\t', \n', '\r' */
else if (ch == '\t') {
*p++ = '\\';
*p++ = 't';
}
else if (ch == '\n') {
*p++ = '\\';
*p++ = 'n';
}
else if (ch == '\r') {
*p++ = '\\';
*p++ = 'r';
}
/* Map non-printable US ASCII to '\xhh' */
else if (ch < ' ' || ch >= 128) { else if (ch < ' ' || ch >= 128) {
*p++ = '\\'; *p++ = '\\';
*p++ = hexdigit[(ch >> 6) & 7]; *p++ = 'x';
*p++ = hexdigit[(ch >> 3) & 7]; *p++ = hexdigit[(ch >> 4) & 0xf];
*p++ = hexdigit[ch & 7]; *p++ = hexdigit[ch & 15];
} }
/* Copy everything else as-is */ /* Copy everything else as-is */
else else
@ -1498,7 +1511,7 @@ PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
char *p; char *p;
char *q; char *q;
static const char *hexdigit = "0123456789ABCDEF"; static const char *hexdigit = "0123456789abcdef";
repr = PyString_FromStringAndSize(NULL, 6 * size); repr = PyString_FromStringAndSize(NULL, 6 * size);
if (repr == NULL) if (repr == NULL)