mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 08:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

[3.12] gh-104866: Tokenize should emit NEWLINE after exiting block with comment (GH-104870) (#104872)

Browse source 
gh-104866: Tokenize should emit NEWLINE after exiting block with comment (GH-104870)
(cherry picked from commit c90a862cdc)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
This commit is contained in:
Miss Islington (bot) 2023-05-24 11:41:58 -07:00 • committed by GitHub
parent 45b88e0653
commit ddc29c8fdb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Lib/test/test_tokenize.py
View file

@ -1057,6 +1057,23 @@ async def f():
DEDENT '' (6, 12) (6, 12)
""")
def test_newline_after_parenthesized_block_with_comment(self):
self.check_tokenize('''\
[
# A comment here
1
]
''', """\
OP '[' (1, 0) (1, 1)
NL '\\n' (1, 1) (1, 2)
COMMENT '# A comment here' (2, 4) (2, 20)
NL '\\n' (2, 20) (2, 21)
NUMBER '1' (3, 4) (3, 5)
NL '\\n' (3, 5) (3, 6)
OP ']' (4, 0) (4, 1)
NEWLINE '\\n' (4, 1) (4, 2)
""")
class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected):
# Format the tokens in s in a table format.

9
Parser/tokenizer.c
View file

@ -2007,6 +2007,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
tok->atbol = 1;
if (blankline || tok->level > 0) {
if (tok->tok_extra_tokens) {
if (tok->comment_newline) {
tok->comment_newline = 0;
}
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
@ -2015,9 +2018,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
}
if (tok->comment_newline && tok->tok_extra_tokens) {
tok->comment_newline = 0;
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
}
p_start = tok->start;
p_end = tok->cur - 1; /* Leave '\n' out of the string */