mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-23 10:16:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

[3.12] gh-104825: Remove implicit newline in the line attribute in tokens emitted in the tokenize module (GH-104846). (#104850)

Browse source 
(cherry picked from commit c8cf9b42eb)
This commit is contained in:
Pablo Galindo Salgado 2023-05-24 11:40:51 +01:00 • committed by GitHub
parent 2d685eca8a
commit 3d2ed8991f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 12 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Python/Python-tokenize.c
View file

@ -123,6 +123,8 @@ _tokenizer_error(struct tok_state *tok)
int result = 0;
Py_ssize_t size = tok->inp - tok->buf;
assert(tok->buf[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
if (!error_line) {
result = -1;
@ -193,6 +195,8 @@ tokenizeriter_next(tokenizeriterobject *it)
}
Py_ssize_t size = it->tok->inp - it->tok->buf;
assert(it->tok->buf[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
PyObject *line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
if (line == NULL) {
Py_DECREF(str);