mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-01 18:51:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 21

bpo-25643: Refactor the C tokenizer into smaller, logical units (GH-25050)

Browse source
This commit is contained in:
Pablo Galindo 2021-03-28 23:48:05 +01:00 • committed by GitHub
parent fb1d01b963
commit 261a452a13
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 354 additions and 360 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Parser/tokenizer.h
View file

@ -15,8 +15,8 @@ extern "C" {
enum decoding_state {
STATE_INIT,
STATE_RAW,
STATE_NORMAL /* have a codec associated with input */
STATE_SEEK_CODING,
STATE_NORMAL
};
/* Tokenizer state */
@ -54,7 +54,6 @@ struct tok_state {
/* Stuff for PEP 0263 */
enum decoding_state decoding_state;
int decoding_erred; /* whether erred in decoding */
int read_coding_spec; /* whether 'coding:...' has been read */
char *encoding; /* Source encoding. */
int cont_line; /* whether we are in a continuation line. */
const char* line_start; /* pointer to start of current line */