mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-09 16:34:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

bpo-35766: Merge typed_ast back into CPython (GH-11645)

Browse source
This commit is contained in:
Guido van Rossum 2019-01-31 03:40:27 -08:00 • committed by Łukasz Langa
parent d97daebfa6
commit dcfcd146f8
30 changed files with 2043 additions and 655 deletions
Download patch file Download diff file Expand all files Collapse all files

57
Parser/tokenizer.c
View file

@ -48,6 +48,10 @@ static int tok_nextc(struct tok_state *tok);
static void tok_backup(struct tok_state *tok, int c);
/* Spaces in this constant are treated as "zero or more spaces or tabs" when
tokenizing. */
static const char* type_comment_prefix = "# type: ";
/* Create and initialize a new tok_state structure */
static struct tok_state *
@ -82,6 +86,7 @@ tok_new(void)
tok->decoding_readline = NULL;
tok->decoding_buffer = NULL;
#endif
tok->type_comments = 0;
return tok;
}
@ -1245,11 +1250,61 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
/* Set start of current token */
tok->start = tok->cur - 1;
/* Skip comment */
/* Skip comment, unless it's a type comment */
if (c == '#') {
const char *prefix, *p, *type_start;
while (c != EOF && c != '\n') {
c = tok_nextc(tok);
}
if (tok->type_comments) {
p = tok->start;
prefix = type_comment_prefix;
while (*prefix && p < tok->cur) {
if (*prefix == ' ') {
while (*p == ' ' || *p == '\t') {
p++;
}
} else if (*prefix == *p) {
p++;
} else {
break;
}
prefix++;
}
/* This is a type comment if we matched all of type_comment_prefix. */
if (!*prefix) {
int is_type_ignore = 1;
tok_backup(tok, c); /* don't eat the newline or EOF */
type_start = p;
is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
p += 6;
while (is_type_ignore && p < tok->cur) {
if (*p == '#')
break;
is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
p++;
}
if (is_type_ignore) {
/* If this type ignore is the only thing on the line, consume the newline also. */
if (blankline) {
tok_nextc(tok);
tok->atbol = 1;
}
return TYPE_IGNORE;
} else {
*p_start = (char *) type_start; /* after type_comment_prefix */
*p_end = tok->cur;
return TYPE_COMMENT;
}
}
}
}
/* Check for EOF and errors now */