mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.

Browse source
This commit is contained in:
Ezio Melotti 2012-11-03 17:30:51 +02:00
parent d7bae5e85a
commit 7d24b1698a
4 changed files with 11 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Lib/tokenize.py
View file

@ -95,7 +95,7 @@ ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
group("'", r'\\\r?\n'),
r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
tokenprog, pseudoprog, single3prog, double3prog = map(
@ -362,6 +362,8 @@ def generate_tokens(readline):
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
if start == end:
continue
token, initial = line[start:end], line[start]
if initial in numchars or \