mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-25 21:11:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Issue2495: tokenize.untokenize did not insert space between two consecutive string literals:

Browse source 
"" "" => """", which is invalid code.

Will backport
This commit is contained in:
Amaury Forgeot d'Arc 2008-03-27 23:23:54 +00:00
parent 853e44ca8c
commit da0c025a43
3 changed files with 22 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Lib/tokenize.py
View file

@ -210,12 +210,21 @@ class Untokenizer:
tokval += ' '
if toknum in (NEWLINE, NL):
startline = True
prevstring = False
for tok in iterable:
toknum, tokval = tok[:2]
if toknum in (NAME, NUMBER):
tokval += ' '
# Insert a space between two consecutive strings
if toknum == STRING:
if prevstring:
tokval = ' ' + tokval
prevstring = True
else:
prevstring = False
if toknum == INDENT:
indents.append(tokval)
continue
@ -244,7 +253,7 @@ def untokenize(iterable):
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
t2 = [tok[:2] for tok in generate_tokens(readline)]
assert t1 == t2
"""
ut = Untokenizer()