mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-19 09:15:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

gh-104825: Remove implicit newline in the line attribute in tokens emitted in the tokenize module (#104846)

Browse source
This commit is contained in:
Pablo Galindo Salgado 2023-05-24 10:59:18 +01:00 • committed by GitHub
parent c45701e9ef
commit c8cf9b42eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 14 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Python/Python-tokenize.c
View file

@ -123,6 +123,8 @@ _tokenizer_error(struct tok_state *tok)
int result = 0;
Py_ssize_t size = tok->inp - tok->buf;
assert(tok->buf[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
if (!error_line) {
result = -1;
@ -193,6 +195,8 @@ tokenizeriter_next(tokenizeriterobject *it)
}
Py_ssize_t size = it->tok->inp - it->tok->buf;
assert(it->tok->buf[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
PyObject *line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
if (line == NULL) {
Py_DECREF(str);