mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-30 13:38:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-44667: Treat correctly lines ending with comments and no newlines in the Python tokenizer (GH-27499)

Browse source
This commit is contained in:
Pablo Galindo Salgado 2021-07-31 02:17:09 +01:00 • committed by GitHub
parent e63e6311aa
commit b6bde9fc42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Lib/test/test_tokenize.py
View file

@ -1458,6 +1458,16 @@ class TestTokenize(TestCase):
# See http://bugs.python.org/issue16152
self.assertExactTypeEqual('@ ', token.AT)
def test_comment_at_the_end_of_the_source_without_newline(self):
# See http://bugs.python.org/issue44667
source = 'b = 1\n\n#test'
expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT]
tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline))
self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING])
for i in range(6):
self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]])
self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER])
class UntokenizeTest(TestCase):