mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 00:48:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

[3.12] gh-120343: Do not reset byte_col_offset_diff after multiline tokens (GH-120352) (#120356)

Browse source 
(cherry picked from commit 1b62bcee94)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
This commit is contained in:
Miss Islington (bot) 2024-06-11 19:22:16 +02:00 • committed by GitHub
parent 92e1c136b5
commit 0315fdc24d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 18 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Python/Python-tokenize.c
View file

@ -35,6 +35,7 @@ typedef struct
/* Needed to cache line for performance */
PyObject *last_line;
Py_ssize_t last_lineno;
Py_ssize_t last_end_lineno;
Py_ssize_t byte_col_offset_diff;
} tokenizeriterobject;
@ -76,6 +77,7 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
self->last_line = NULL;
self->byte_col_offset_diff = 0;
self->last_lineno = 0;
self->last_end_lineno = 0;
return (PyObject *)self;
}
@ -226,7 +228,9 @@ tokenizeriter_next(tokenizeriterobject *it)
Py_XDECREF(it->last_line);
line = PyUnicode_DecodeUTF8(line_start, size, "replace");
it->last_line = line;
it->byte_col_offset_diff = 0;
if (it->tok->lineno != it->last_end_lineno) {
it->byte_col_offset_diff = 0;
}
} else {
// Line hasn't changed so we reuse the cached one.
line = it->last_line;
@ -240,6 +244,7 @@ tokenizeriter_next(tokenizeriterobject *it)
Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
Py_ssize_t end_lineno = it->tok->lineno;
it->last_lineno = lineno;
it->last_end_lineno = end_lineno;
Py_ssize_t col_offset = -1;
Py_ssize_t end_col_offset = -1;