mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-27 02:39:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

gh-103718: Correctly set f-string buffers in all cases (GH-103815)

Browse source 
Turns out we always need to remember/restore fstring buffers in all of
the stack of tokenizer modes, cause they might change to
`TOK_REGULAR_MODE` and have newlines inside the braces (which is when we
need to reallocate the buffer and restore the fstring ones).
This commit is contained in:
Lysandros Nikolaou 2023-04-24 19:31:21 -06:00 • committed by GitHub
parent 3df3b91e6a
commit 57f8f9a66d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Parser/tokenizer.c
View file

@ -371,11 +371,9 @@ remember_fstring_buffers(struct tok_state *tok)
for (index = tok->tok_mode_stack_index; index >= 0; --index) { for (index = tok->tok_mode_stack_index; index >= 0; --index) {
mode = &(tok->tok_mode_stack[index]); mode = &(tok->tok_mode_stack[index]);
if (mode->kind == TOK_FSTRING_MODE) {
mode->f_string_start_offset = mode->f_string_start - tok->buf; mode->f_string_start_offset = mode->f_string_start - tok->buf;
mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf; mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
} }
}
} }
/* Traverse and restore all f-string buffers after reallocating tok->buf */ /* Traverse and restore all f-string buffers after reallocating tok->buf */
@ -387,11 +385,9 @@ restore_fstring_buffers(struct tok_state *tok)
for (index = tok->tok_mode_stack_index; index >= 0; --index) { for (index = tok->tok_mode_stack_index; index >= 0; --index) {
mode = &(tok->tok_mode_stack[index]); mode = &(tok->tok_mode_stack[index]);
if (mode->kind == TOK_FSTRING_MODE) {
mode->f_string_start = tok->buf + mode->f_string_start_offset; mode->f_string_start = tok->buf + mode->f_string_start_offset;
mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset; mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
} }
}
} }
static int static int
@ -1081,6 +1077,7 @@ tok_underflow_interactive(struct tok_state *tok) {
restore_fstring_buffers(tok); restore_fstring_buffers(tok);
} }
else { else {
remember_fstring_buffers(tok);
ADVANCE_LINENO(); ADVANCE_LINENO();
PyMem_Free(tok->buf); PyMem_Free(tok->buf);
tok->buf = newtok; tok->buf = newtok;
@ -1088,6 +1085,7 @@ tok_underflow_interactive(struct tok_state *tok) {
tok->line_start = tok->buf; tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0'); tok->inp = strchr(tok->buf, '\0');
tok->end = tok->inp + 1; tok->end = tok->inp + 1;
restore_fstring_buffers(tok);
} }
if (tok->done != E_OK) { if (tok->done != E_OK) {
if (tok->prompt != NULL) { if (tok->prompt != NULL) {