mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

gh-112943: Correctly compute end offsets for multiline tokens in the tokenize module (#112949)

Browse source
This commit is contained in:
Pablo Galindo Salgado 2023-12-11 11:44:22 +00:00 • committed by GitHub
parent 4c5b9c107a
commit a135a6d2c6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 25 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Lib/test/test_tokenize.py
View file

@ -615,6 +615,16 @@ f'__{
OP '}' (3, 0) (3, 1) OP '}' (3, 0) (3, 1)
FSTRING_MIDDLE '__' (3, 1) (3, 3) FSTRING_MIDDLE '__' (3, 1) (3, 3)
FSTRING_END "'" (3, 3) (3, 4) FSTRING_END "'" (3, 3) (3, 4)
""")
self.check_tokenize("""\
'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
aktualni pracownicy, obecni pracownicy'''
""", """\
INDENT ' ' (1, 0) (1, 4)
STRING "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
NEWLINE '\\n' (2, 45) (2, 46)
DEDENT '' (3, 0) (3, 0)
""") """)
def test_function(self): def test_function(self):

2
Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst Normal file
View file

@ -0,0 +1,2 @@
Correctly compute end column offsets for multiline tokens in the
:mod:`tokenize` module. Patch by Pablo Galindo

16
Parser/pegen.c
View file

@ -19,12 +19,8 @@ _PyPegen_interactive_exit(Parser *p)
} }
Py_ssize_t Py_ssize_t
_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) _PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset)
{ {
const char *str = PyUnicode_AsUTF8(line);
if (!str) {
return -1;
}
Py_ssize_t len = strlen(str); Py_ssize_t len = strlen(str);
if (col_offset > len + 1) { if (col_offset > len + 1) {
col_offset = len + 1; col_offset = len + 1;
@ -39,6 +35,16 @@ _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
return size; return size;
} }
Py_ssize_t
_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
{
const char *str = PyUnicode_AsUTF8(line);
if (!str) {
return -1;
}
return _PyPegen_byte_offset_to_character_offset_raw(str, col_offset);
}
// Here, mark is the start of the node, while p->mark is the end. // Here, mark is the start of the node, while p->mark is the end.
// If node==NULL, they should be the same. // If node==NULL, they should be the same.
int int

1
Parser/pegen.h
View file

@ -149,6 +149,7 @@ expr_ty _PyPegen_name_token(Parser *p);
expr_ty _PyPegen_number_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p);
void *_PyPegen_string_token(Parser *p); void *_PyPegen_string_token(Parser *p);
Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset);
// Error handling functions and APIs // Error handling functions and APIs
typedef enum { typedef enum {

2
Python/Python-tokenize.c
View file

@ -225,7 +225,7 @@ tokenizeriter_next(tokenizeriterobject *it)
col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start); col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start);
} }
if (token.end != NULL && token.end >= it->tok->line_start) { if (token.end != NULL && token.end >= it->tok->line_start) {
end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start); end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, token.end - it->tok->line_start);
} }
if (it->tok->tok_extra_tokens) { if (it->tok->tok_extra_tokens) {