mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 00:48:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-104866: Tokenize should emit NEWLINE after exiting block with comment (#104870)

Browse source
This commit is contained in:
Lysandros Nikolaou 2023-05-24 19:18:17 +03:00 • committed by GitHub
parent 9d457e1154
commit c90a862cdc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Parser/tokenizer.c
View file

@ -2007,6 +2007,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
tok->atbol = 1;
if (blankline || tok->level > 0) {
if (tok->tok_extra_tokens) {
if (tok->comment_newline) {
tok->comment_newline = 0;
}
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
@ -2015,9 +2018,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
}
if (tok->comment_newline && tok->tok_extra_tokens) {
tok->comment_newline = 0;
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
p_start = tok->start;
p_end = tok->cur;
return MAKE_TOKEN(NL);
}
p_start = tok->start;
p_end = tok->cur - 1; /* Leave '\n' out of the string */