mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 16:39:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

[3.13] gh-120343: Fix column offsets of multiline tokens in tokenize (GH-120391) (#120427)

Browse source 
(cherry picked from commit 4b5d3e0e72)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
This commit is contained in:
Miss Islington (bot) 2024-06-12 21:22:44 +02:00 • committed by GitHub
parent 0041087aa1
commit 7c6b3429b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 24 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Lib/test/test_tokenize.py
View file

@ -1210,6 +1210,20 @@ a = f'''
FSTRING_END "\'\'\'" (2, 68) (2, 71) FSTRING_END "\'\'\'" (2, 68) (2, 71)
""") """)
def test_multiline_non_ascii_fstring_with_expr(self):
self.check_tokenize("""\
f'''
🔗 This is a test {test_arg1}🔗
🔗'''""", """\
FSTRING_START "f\'\'\'" (1, 0) (1, 4)
FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21)
OP '{' (2, 21) (2, 22)
NAME 'test_arg1' (2, 22) (2, 31)
OP '}' (2, 31) (2, 32)
FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1)
FSTRING_END "\'\'\'" (3, 1) (3, 4)
""")
class GenerateTokensTest(TokenizeTest): class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected): def check_tokenize(self, s, expected):
# Format the tokens in s in a table format. # Format the tokens in s in a table format.

14
Python/Python-tokenize.c
View file

@ -215,6 +215,7 @@ tokenizeriter_next(tokenizeriterobject *it)
const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start; const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
PyObject* line = NULL; PyObject* line = NULL;
int line_changed = 1;
if (it->tok->tok_extra_tokens && is_trailing_token) { if (it->tok->tok_extra_tokens && is_trailing_token) {
line = PyUnicode_FromString(""); line = PyUnicode_FromString("");
} else { } else {
@ -229,12 +230,11 @@ tokenizeriter_next(tokenizeriterobject *it)
Py_XDECREF(it->last_line); Py_XDECREF(it->last_line);
line = PyUnicode_DecodeUTF8(line_start, size, "replace"); line = PyUnicode_DecodeUTF8(line_start, size, "replace");
it->last_line = line; it->last_line = line;
if (it->tok->lineno != it->last_end_lineno) { it->byte_col_offset_diff = 0;
it->byte_col_offset_diff = 0;
}
} else { } else {
// Line hasn't changed so we reuse the cached one. // Line hasn't changed so we reuse the cached one.
line = it->last_line; line = it->last_line;
line_changed = 0;
} }
} }
if (line == NULL) { if (line == NULL) {
@ -252,7 +252,13 @@ tokenizeriter_next(tokenizeriterobject *it)
Py_ssize_t byte_offset = -1; Py_ssize_t byte_offset = -1;
if (token.start != NULL && token.start >= line_start) { if (token.start != NULL && token.start >= line_start) {
byte_offset = token.start - line_start; byte_offset = token.start - line_start;
col_offset = byte_offset - it->byte_col_offset_diff; if (line_changed) {
col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
it->byte_col_offset_diff = byte_offset - col_offset;
}
else {
col_offset = byte_offset - it->byte_col_offset_diff;
}
} }
if (token.end != NULL && token.end >= it->tok->line_start) { if (token.end != NULL && token.end >= it->tok->line_start) {
Py_ssize_t end_byte_offset = token.end - it->tok->line_start; Py_ssize_t end_byte_offset = token.end - it->tok->line_start;