mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-105259: Ensure we don't show newline characters for trailing NEWLINE tokens (#105364)

Browse source
This commit is contained in:
Pablo Galindo Salgado 2023-06-06 13:52:16 +02:00 • committed by GitHub
parent 0202aa002e
commit c0a6ed3934
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 17 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Parser/tokenizer.c
View file

@ -114,6 +114,7 @@ tok_new(void)
tok->report_warnings = 1;
tok->tok_extra_tokens = 0;
tok->comment_newline = 0;
tok->implicit_newline = 0;
tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
tok->tok_mode_stack_index = 0;
tok->tok_report_warnings = 1;
@ -355,10 +356,12 @@ tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
return -1;
}
strcpy(new_str + current_size, line);
tok->implicit_newline = 0;
if (last_char != '\n') {
/* Last line does not end in \n, fake one */
new_str[current_size + line_size - 1] = '\n';
new_str[current_size + line_size] = '\0';
tok->implicit_newline = 1;
}
tok->interactive_src_start = new_str;
tok->interactive_src_end = new_str + current_size + line_size;
@ -1262,11 +1265,13 @@ tok_underflow_file(struct tok_state *tok) {
tok->done = E_EOF;
return 0;
}
tok->implicit_newline = 0;
if (tok->inp[-1] != '\n') {
assert(tok->inp + 1 < tok->end);
/* Last line does not end in \n, fake one */
*tok->inp++ = '\n';
*tok->inp = '\0';
tok->implicit_newline = 1;
}
ADVANCE_LINENO();
@ -1304,11 +1309,13 @@ tok_underflow_readline(struct tok_state* tok) {
tok->done = E_EOF;
return 0;
}
tok->implicit_newline = 0;
if (tok->inp[-1] != '\n') {
assert(tok->inp + 1 < tok->end);
/* Last line does not end in \n, fake one */
*tok->inp++ = '\n';
*tok->inp = '\0';
tok->implicit_newline = 1;
}
ADVANCE_LINENO();

1
Parser/tokenizer.h
View file

@ -131,6 +131,7 @@ struct tok_state {
int tok_report_warnings;
int tok_extra_tokens;
int comment_newline;
int implicit_newline;
#ifdef Py_DEBUG
int debug;
#endif