mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 00:48:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Issue #15054: Fix incorrect tokenization of 'b' string literals.

Browse source 
Patch by Serhiy Storchaka.
This commit is contained in:
Meador Inge 2012-06-16 21:49:08 -05:00
parent 2d9db1dfce
commit 8d5c0b8c19
3 changed files with 81 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

76
Lib/test/test_tokenize.py
View file

@ -289,6 +289,82 @@ String literals
OP '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
>>> dump_tokens("u'abc' + U'abc'")
ENCODING 'utf-8' (0, 0) (0, 0)
STRING "u'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
>>> dump_tokens('u"abc" + U"abc"')
ENCODING 'utf-8' (0, 0) (0, 0)
STRING 'u"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
>>> dump_tokens("ur'abc' + uR'abc' + Ur'abc' + UR'abc'")
ENCODING 'utf-8' (0, 0) (0, 0)
STRING "ur'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "uR'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Ur'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "UR'abc'" (1, 30) (1, 37)
>>> dump_tokens('ur"abc" + uR"abc" + Ur"abc" + UR"abc"')
ENCODING 'utf-8' (0, 0) (0, 0)
STRING 'ur"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'uR"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Ur"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'UR"abc"' (1, 30) (1, 37)
>>> dump_tokens("b'abc' + B'abc'")
ENCODING 'utf-8' (0, 0) (0, 0)
STRING "b'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
>>> dump_tokens('b"abc" + B"abc"')
ENCODING 'utf-8' (0, 0) (0, 0)
STRING 'b"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
>>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
ENCODING 'utf-8' (0, 0) (0, 0)
STRING "br'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
>>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
ENCODING 'utf-8' (0, 0) (0, 0)
STRING 'br"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
>>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'")
ENCODING 'utf-8' (0, 0) (0, 0)
STRING "rb'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
>>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"')
ENCODING 'utf-8' (0, 0) (0, 0)
STRING 'rb"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
Operators
>>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")