mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-23 02:04:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

[3.12] gh-104972: Ensure that line attributes in tokens in the tokenize module are correct (GH-104975) (#104982)

Browse source 
gh-104972: Ensure that line attributes in tokens in the tokenize module are correct (GH-104975)
(cherry picked from commit 3fdb55c482)

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
This commit is contained in:
Miss Islington (bot) 2023-05-26 08:25:46 -07:00 • committed by GitHub
parent 01af2b0e51
commit 8ca29573a8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 21 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Python/Python-tokenize.c
View file

@ -194,15 +194,14 @@ tokenizeriter_next(tokenizeriterobject *it)
goto exit;
}
Py_ssize_t size = it->tok->inp - it->tok->buf;
assert(it->tok->buf[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
PyObject *line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
Py_ssize_t size = it->tok->inp - line_start;
PyObject *line = PyUnicode_DecodeUTF8(line_start, size, "replace");
if (line == NULL) {
Py_DECREF(str);
goto exit;
}
const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
Py_ssize_t end_lineno = it->tok->lineno;
Py_ssize_t col_offset = -1;