mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

some cleanups

Browse source
This commit is contained in:
Benjamin Peterson 2009-10-15 03:05:39 +00:00
parent e52657220c
commit ca2d2529ce
1 changed files with 10 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Lib/tokenize.py
View file

@ -23,15 +23,15 @@ function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.""" each time a new token is found."""
__author__ = 'Ka-Ping Yee <ping@lfw.org>' __author__ = 'Ka-Ping Yee <ping@lfw.org>'
__credits__ = \ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger' 'Skip Montanaro, Raymond Hettinger')
import string, re import string, re
from token import * from token import *
import token import token
__all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize", __all__ = [x for x in dir(token) if not x.startswith("_")]
"generate_tokens", "NL", "untokenize"] __all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
del x del x
del token del token
@ -288,7 +288,7 @@ def generate_tokens(readline):
line = readline() line = readline()
except StopIteration: except StopIteration:
line = '' line = ''
lnum = lnum + 1 lnum += 1
pos, max = 0, len(line) pos, max = 0, len(line)
if contstr: # continued string if contstr: # continued string
@ -317,14 +317,14 @@ def generate_tokens(readline):
column = 0 column = 0
while pos < max: # measure leading whitespace while pos < max: # measure leading whitespace
if line[pos] == ' ': if line[pos] == ' ':
column = column + 1 column += 1
elif line[pos] == '\t': elif line[pos] == '\t':
column = (column//tabsize + 1)*tabsize column = (column//tabsize + 1)*tabsize
elif line[pos] == '\f': elif line[pos] == '\f':
column = 0 column = 0
else: else:
break break
pos = pos + 1 pos += 1
if pos == max: if pos == max:
break break
@ -403,14 +403,14 @@ def generate_tokens(readline):
continued = 1 continued = 1
else: else:
if initial in '([{': if initial in '([{':
parenlev = parenlev + 1 parenlev += 1
elif initial in ')]}': elif initial in ')]}':
parenlev = parenlev - 1 parenlev -= 1
yield (OP, token, spos, epos, line) yield (OP, token, spos, epos, line)
else: else:
yield (ERRORTOKEN, line[pos], yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line) (lnum, pos), (lnum, pos+1), line)
pos = pos + 1 pos += 1
for indent in indents[1:]: # pop remaining indent levels for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (DEDENT, '', (lnum, 0), (lnum, 0), '')