mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-07 15:42:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 18

bpo-30377: Simplify handling of COMMENT and NL in tokenize.py (#1607)

Browse source
This commit is contained in:
Albert-Jan Nijburg 2017-05-24 12:31:57 +01:00 • committed by Serhiy Storchaka
parent a17a2f52c4
commit c471ca448c
2 changed files with 12 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Lib/test/test_tokenize.py
View file

@ -39,6 +39,7 @@ class TokenizeTest(TestCase):
""") """)
self.check_tokenize("if False:\n" self.check_tokenize("if False:\n"
" # NL\n" " # NL\n"
" \n"
" True = False # NEWLINE\n", """\ " True = False # NEWLINE\n", """\
NAME 'if' (1, 0) (1, 2) NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8) NAME 'False' (1, 3) (1, 8)
@ -46,13 +47,14 @@ class TokenizeTest(TestCase):
NEWLINE '\\n' (1, 9) (1, 10) NEWLINE '\\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8) COMMENT '# NL' (2, 4) (2, 8)
NL '\\n' (2, 8) (2, 9) NL '\\n' (2, 8) (2, 9)
INDENT ' ' (3, 0) (3, 4) NL '\\n' (3, 4) (3, 5)
NAME 'True' (3, 4) (3, 8) INDENT ' ' (4, 0) (4, 4)
OP '=' (3, 9) (3, 10) NAME 'True' (4, 4) (4, 8)
NAME 'False' (3, 11) (3, 16) OP '=' (4, 9) (4, 10)
COMMENT '# NEWLINE' (3, 17) (3, 26) NAME 'False' (4, 11) (4, 16)
NEWLINE '\\n' (3, 26) (3, 27) COMMENT '# NEWLINE' (4, 17) (4, 26)
DEDENT '' (4, 0) (4, 0) NEWLINE '\\n' (4, 26) (4, 27)
DEDENT '' (5, 0) (5, 0)
""") """)
indent_error_file = b"""\ indent_error_file = b"""\
def k(x): def k(x):

8
Lib/tokenize.py
View file

@ -560,13 +560,11 @@ def _tokenize(readline, encoding):
if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#': if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n') comment_token = line[pos:].rstrip('\r\n')
nl_pos = pos + len(comment_token)
yield TokenInfo(COMMENT, comment_token, yield TokenInfo(COMMENT, comment_token,
(lnum, pos), (lnum, pos + len(comment_token)), line) (lnum, pos), (lnum, pos + len(comment_token)), line)
yield TokenInfo(NL, line[nl_pos:], pos += len(comment_token)
(lnum, nl_pos), (lnum, len(line)), line)
else: yield TokenInfo(NL, line[pos:],
yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)), line) (lnum, pos), (lnum, len(line)), line)
continue continue