mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-07 19:35:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-97973: Return all necessary information from the tokenizer (GH-97984)

Browse source 
Right now, the tokenizer only returns type and two pointers to the start and end of the token.
This PR modifies the tokenizer to return the type and set all of the necessary information,
so that the parser does not have to this.
This commit is contained in:
Lysandros Nikolaou 2022-10-06 16:07:17 -07:00 • committed by GitHub
parent b9d2e81716
commit cbf0afd8a1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 161 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Python/Python-tokenize.c
View file

@ -60,9 +60,8 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source)
static PyObject *
tokenizeriter_next(tokenizeriterobject *it)
{
const char *start;
const char *end;
int type = _PyTokenizer_Get(it->tok, &start, &end);
struct token token;
int type = _PyTokenizer_Get(it->tok, &token);
if (type == ERRORTOKEN && PyErr_Occurred()) {
return NULL;
}
@ -71,11 +70,11 @@ tokenizeriter_next(tokenizeriterobject *it)
return NULL;
}
PyObject *str = NULL;
if (start == NULL || end == NULL) {
if (token.start == NULL || token.end == NULL) {
str = PyUnicode_FromString("");
}
else {
str = PyUnicode_FromStringAndSize(start, end - start);
str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
}
if (str == NULL) {
return NULL;
@ -92,11 +91,11 @@ tokenizeriter_next(tokenizeriterobject *it)
int end_lineno = it->tok->lineno;
int col_offset = -1;
int end_col_offset = -1;
if (start != NULL && start >= line_start) {
col_offset = (int)(start - line_start);
if (token.start != NULL && token.start >= line_start) {
col_offset = (int)(token.start - line_start);
}
if (end != NULL && end >= it->tok->line_start) {
end_col_offset = (int)(end - it->tok->line_start);
if (token.end != NULL && token.end >= it->tok->line_start) {
end_col_offset = (int)(token.end - it->tok->line_start);
}
return Py_BuildValue("(NiiiiiN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);