mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-15 14:20:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

[3.13] gh-130587: Add hand-written docs for non-OP tokens (GH-130588) (GH-131465)

Browse source 
Co-authored-by: Blaise Pabon <blaise@gmail.com>
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
This commit is contained in:
Petr Viktorin 2025-03-24 13:34:23 +01:00 • committed by GitHub
parent b3dccc0ce2
commit 478eb73103
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 286 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

330
Doc/library/token-list.inc generated
View file

@ -1,230 +1,104 @@
.. Auto-generated by Tools/build/generate_token.py .. Auto-generated by Tools/build/generate_token.py
.. data:: ENDMARKER
.. data:: NAME .. list-table::
:align: left
.. data:: NUMBER :header-rows: 1
.. data:: STRING * - Token
- Value
.. data:: NEWLINE * - .. data:: LPAR
- ``"("``
.. data:: INDENT * - .. data:: RPAR
- ``")"``
.. data:: DEDENT * - .. data:: LSQB
- ``"["``
.. data:: LPAR * - .. data:: RSQB
- ``"]"``
Token value for ``"("``. * - .. data:: COLON
- ``":"``
.. data:: RPAR * - .. data:: COMMA
- ``","``
Token value for ``")"``. * - .. data:: SEMI
- ``";"``
.. data:: LSQB * - .. data:: PLUS
- ``"+"``
Token value for ``"["``. * - .. data:: MINUS
- ``"-"``
.. data:: RSQB * - .. data:: STAR
- ``"*"``
Token value for ``"]"``. * - .. data:: SLASH
- ``"/"``
.. data:: COLON * - .. data:: VBAR
- ``"|"``
Token value for ``":"``. * - .. data:: AMPER
- ``"&"``
.. data:: COMMA * - .. data:: LESS
- ``"<"``
Token value for ``","``. * - .. data:: GREATER
- ``">"``
.. data:: SEMI * - .. data:: EQUAL
- ``"="``
Token value for ``";"``. * - .. data:: DOT
- ``"."``
.. data:: PLUS * - .. data:: PERCENT
- ``"%"``
Token value for ``"+"``. * - .. data:: LBRACE
- ``"{"``
.. data:: MINUS * - .. data:: RBRACE
- ``"}"``
Token value for ``"-"``. * - .. data:: EQEQUAL
- ``"=="``
.. data:: STAR * - .. data:: NOTEQUAL
- ``"!="``
Token value for ``"*"``. * - .. data:: LESSEQUAL
- ``"<="``
.. data:: SLASH * - .. data:: GREATEREQUAL
- ``">="``
Token value for ``"/"``. * - .. data:: TILDE
- ``"~"``
.. data:: VBAR * - .. data:: CIRCUMFLEX
- ``"^"``
Token value for ``"|"``. * - .. data:: LEFTSHIFT
- ``"<<"``
.. data:: AMPER * - .. data:: RIGHTSHIFT
- ``">>"``
Token value for ``"&"``. * - .. data:: DOUBLESTAR
- ``"**"``
.. data:: LESS * - .. data:: PLUSEQUAL
- ``"+="``
Token value for ``"<"``. * - .. data:: MINEQUAL
- ``"-="``
.. data:: GREATER * - .. data:: STAREQUAL
- ``"*="``
Token value for ``">"``. * - .. data:: SLASHEQUAL
- ``"/="``
.. data:: EQUAL * - .. data:: PERCENTEQUAL
- ``"%="``
Token value for ``"="``. * - .. data:: AMPEREQUAL
- ``"&="``
.. data:: DOT * - .. data:: VBAREQUAL
- ``"|="``
Token value for ``"."``. * - .. data:: CIRCUMFLEXEQUAL
- ``"^="``
.. data:: PERCENT * - .. data:: LEFTSHIFTEQUAL
- ``"<<="``
Token value for ``"%"``. * - .. data:: RIGHTSHIFTEQUAL
- ``">>="``
.. data:: LBRACE * - .. data:: DOUBLESTAREQUAL
- ``"**="``
Token value for ``"{"``. * - .. data:: DOUBLESLASH
- ``"//"``
.. data:: RBRACE * - .. data:: DOUBLESLASHEQUAL
- ``"//="``
Token value for ``"}"``. * - .. data:: AT
- ``"@"``
.. data:: EQEQUAL * - .. data:: ATEQUAL
- ``"@="``
Token value for ``"=="``. * - .. data:: RARROW
- ``"->"``
.. data:: NOTEQUAL * - .. data:: ELLIPSIS
- ``"..."``
Token value for ``"!="``. * - .. data:: COLONEQUAL
- ``":="``
.. data:: LESSEQUAL * - .. data:: EXCLAMATION
- ``"!"``
Token value for ``"<="``.
.. data:: GREATEREQUAL
Token value for ``">="``.
.. data:: TILDE
Token value for ``"~"``.
.. data:: CIRCUMFLEX
Token value for ``"^"``.
.. data:: LEFTSHIFT
Token value for ``"<<"``.
.. data:: RIGHTSHIFT
Token value for ``">>"``.
.. data:: DOUBLESTAR
Token value for ``"**"``.
.. data:: PLUSEQUAL
Token value for ``"+="``.
.. data:: MINEQUAL
Token value for ``"-="``.
.. data:: STAREQUAL
Token value for ``"*="``.
.. data:: SLASHEQUAL
Token value for ``"/="``.
.. data:: PERCENTEQUAL
Token value for ``"%="``.
.. data:: AMPEREQUAL
Token value for ``"&="``.
.. data:: VBAREQUAL
Token value for ``"|="``.
.. data:: CIRCUMFLEXEQUAL
Token value for ``"^="``.
.. data:: LEFTSHIFTEQUAL
Token value for ``"<<="``.
.. data:: RIGHTSHIFTEQUAL
Token value for ``">>="``.
.. data:: DOUBLESTAREQUAL
Token value for ``"**="``.
.. data:: DOUBLESLASH
Token value for ``"//"``.
.. data:: DOUBLESLASHEQUAL
Token value for ``"//="``.
.. data:: AT
Token value for ``"@"``.
.. data:: ATEQUAL
Token value for ``"@="``.
.. data:: RARROW
Token value for ``"->"``.
.. data:: ELLIPSIS
Token value for ``"..."``.
.. data:: COLONEQUAL
Token value for ``":="``.
.. data:: EXCLAMATION
Token value for ``"!"``.
.. data:: OP
.. data:: TYPE_IGNORE
.. data:: TYPE_COMMENT
.. data:: SOFT_KEYWORD
.. data:: FSTRING_START
.. data:: FSTRING_MIDDLE
.. data:: FSTRING_END
.. data:: COMMENT
.. data:: NL
.. data:: ERRORTOKEN
.. data:: N_TOKENS
.. data:: NT_OFFSET

149
Doc/library/token.rst
View file

@ -19,6 +19,10 @@ change between Python versions.
The module also provides a mapping from numeric codes to names and some The module also provides a mapping from numeric codes to names and some
functions. The functions mirror definitions in the Python C header files. functions. The functions mirror definitions in the Python C header files.
Note that a token's value may depend on tokenizer options. For example, a
``"+"`` token may be reported as either :data:`PLUS` or :data:`OP`, or
a ``"match"`` token may be either :data:`NAME` or :data:`SOFT_KEYWORD`.
.. data:: tok_name .. data:: tok_name
@ -44,25 +48,92 @@ functions. The functions mirror definitions in the Python C header files.
The token constants are: The token constants are:
.. include:: token-list.inc .. data:: NAME
The following token type values aren't used by the C tokenizer but are needed for Token value that indicates an :ref:`identifier <identifiers>`.
the :mod:`tokenize` module. Note that keywords are also initially tokenized an ``NAME`` tokens.
.. data:: NUMBER
Token value that indicates a :ref:`numeric literal <numbers>`
.. data:: STRING
Token value that indicates a :ref:`string or byte literal <strings>`,
excluding :ref:`formatted string literals <f-strings>`.
The token string is not interpreted:
it includes the surrounding quotation marks and the prefix (if given);
backslashes are included literally, without processing escape sequences.
.. data:: OP
A generic token value that indicates an
:ref:`operator <operators>` or :ref:`delimiter <delimiters>`.
.. impl-detail::
This value is only reported by the :mod:`tokenize` module.
Internally, the tokenizer uses
:ref:`exact token types <token_operators_delimiters>` instead.
.. data:: COMMENT .. data:: COMMENT
:noindex:
Token value used to indicate a comment. Token value used to indicate a comment.
The parser ignores :data:`!COMMENT` tokens.
.. data:: NEWLINE
Token value that indicates the end of a :ref:`logical line <logical-lines>`.
.. data:: NL .. data:: NL
:noindex:
Token value used to indicate a non-terminating newline. The Token value used to indicate a non-terminating newline.
:data:`NEWLINE` token indicates the end of a logical line of Python code; :data:`!NL` tokens are generated when a logical line of code is continued
``NL`` tokens are generated when a logical line of code is continued over over multiple physical lines. The parser ignores :data:`!NL` tokens.
multiple physical lines.
.. data:: INDENT
Token value used at the beginning of a :ref:`logical line <logical-lines>`
to indicate the start of an :ref:`indented block <indentation>`.
.. data:: DEDENT
Token value used at the beginning of a :ref:`logical line <logical-lines>`
to indicate the end of an :ref:`indented block <indentation>`.
.. data:: FSTRING_START
Token value used to indicate the beginning of an
:ref:`f-string literal <f-strings>`.
.. impl-detail::
The token string includes the prefix and the opening quote(s), but none
of the contents of the literal.
.. data:: FSTRING_MIDDLE
Token value used for literal text inside an :ref:`f-string literal <f-strings>`,
including format specifications.
.. impl-detail::
Replacement fields (that is, the non-literal parts of f-strings) use
the same tokens as other expressions, and are delimited by
:data:`LBRACE`, :data:`RBRACE`, :data:`EXCLAMATION` and :data:`COLON`
tokens.
.. data:: FSTRING_END
Token value used to indicate the end of a :ref:`f-string <f-strings>`.
.. impl-detail::
The token string contains the closing quote(s).
.. data:: ENDMARKER
Token value that indicates the end of input.
.. data:: ENCODING .. data:: ENCODING
@ -70,14 +141,63 @@ the :mod:`tokenize` module.
into text. The first token returned by :func:`tokenize.tokenize` will into text. The first token returned by :func:`tokenize.tokenize` will
always be an ``ENCODING`` token. always be an ``ENCODING`` token.
.. impl-detail::
This token type isn't used by the C tokenizer but is needed for
the :mod:`tokenize` module.
The following token types are not produced by the :mod:`tokenize` module,
and are defined for special uses in the tokenizer or parser:
.. data:: TYPE_IGNORE
Token value indicating that a ``type: ignore`` comment was recognized.
Such tokens are produced instead of regular :data:`COMMENT` tokens only
with the :data:`~ast.PyCF_TYPE_COMMENTS` flag.
.. data:: TYPE_COMMENT .. data:: TYPE_COMMENT
:noindex:
Token value indicating that a type comment was recognized. Such Token value indicating that a type comment was recognized.
tokens are only produced when :func:`ast.parse` is invoked with Such tokens are produced instead of regular :data:`COMMENT` tokens only
``type_comments=True``. with the :data:`~ast.PyCF_TYPE_COMMENTS` flag.
.. data:: SOFT_KEYWORD
Token value indicating a :ref:`soft keyword <soft-keywords>`.
The tokenizer never produces this value.
To check for a soft keyword, pass a :data:`NAME` token's string to
:func:`keyword.issoftkeyword`.
.. data:: ERRORTOKEN
Token value used to indicate wrong input.
The :mod:`tokenize` module generally indicates errors by
raising exceptions instead of emitting this token.
It can also emit tokens such as :data:`OP` or :data:`NAME` with strings that
are later rejected by the parser.
.. _token_operators_delimiters:
The remaining tokens represent specific :ref:`operators <operators>` and
:ref:`delimiters <delimiters>`.
(The :mod:`tokenize` module reports these as :data:`OP`; see ``exact_type``
in the :mod:`tokenize` documentation for details.)
.. include:: token-list.inc
The following non-token constants are provided:
.. data:: N_TOKENS
The number of token types defined in this module.
.. NT_OFFSET is deliberately undocumented; if you need it you should be
reading the source
.. data:: EXACT_TOKEN_TYPES .. data:: EXACT_TOKEN_TYPES
@ -102,6 +222,9 @@ the :mod:`tokenize` module.
to support parsing older Python versions for :func:`ast.parse` with to support parsing older Python versions for :func:`ast.parse` with
``feature_version`` set to 6 or lower). ``feature_version`` set to 6 or lower).
.. versionchanged:: 3.12
Added :data:`EXCLAMATION`.
.. versionchanged:: 3.13 .. versionchanged:: 3.13
Removed :data:`!AWAIT` and :data:`!ASYNC` tokens again. Removed :data:`!AWAIT` and :data:`!ASYNC` tokens again.

53
Tools/build/generate_token.py
View file

@ -1,11 +1,18 @@
#! /usr/bin/env python3 #! /usr/bin/env python3
# This script generates token related files from Grammar/Tokens: # This script generates token related files from Grammar/Tokens:
# #
# make_rst:
# Doc/library/token-list.inc # Doc/library/token-list.inc
# Doc/library/token.rst (checked, not generated)
# make_h:
# Include/token.h # Include/token.h
# make_c:
# Parser/token.c # Parser/token.c
# make_py:
# Lib/token.py # Lib/token.py
import re
SCRIPT_NAME = 'Tools/build/generate_token.py' SCRIPT_NAME = 'Tools/build/generate_token.py'
AUTO_GENERATED_BY_SCRIPT = f'Auto-generated by {SCRIPT_NAME}' AUTO_GENERATED_BY_SCRIPT = f'Auto-generated by {SCRIPT_NAME}'
@ -199,23 +206,51 @@ def make_c(infile, outfile='Parser/token.c'):
token_inc_template = f"""\ token_inc_template = f"""\
.. {AUTO_GENERATED_BY_SCRIPT} .. {AUTO_GENERATED_BY_SCRIPT}
%s
.. data:: N_TOKENS
.. data:: NT_OFFSET .. list-table::
:align: left
:header-rows: 1
* - Token
- Value
%s
""" """
def make_rst(infile, outfile='Doc/library/token-list.inc'): def make_rst(infile, outfile='Doc/library/token-list.inc',
rstfile='Doc/library/token.rst'):
tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile) tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
tok_to_string = {value: s for s, value in string_to_tok.items()} tok_to_string = {value: s for s, value in string_to_tok.items()}
needs_handwritten_doc = set()
names = [] names = []
for value, name in enumerate(tok_names[:ERRORTOKEN + 1]): for value, name in enumerate(tok_names):
names.append('.. data:: %s' % (name,))
if value in tok_to_string: if value in tok_to_string:
names.append('') assert name.isupper()
names.append(' Token value for ``"%s"``.' % tok_to_string[value]) names.append(f' * - .. data:: {name}')
names.append('') names.append(f' - ``"{tok_to_string[value]}"``')
else:
needs_handwritten_doc.add(name)
has_handwritten_doc = set()
with open(rstfile) as fileobj:
tokendef_re = re.compile(r'.. data:: ([0-9A-Z_]+)\s*')
for line in fileobj:
if match := tokendef_re.fullmatch(line):
has_handwritten_doc.add(match[1])
# Exclude non-token constants in token.py
has_handwritten_doc -= {'N_TOKENS', 'NT_OFFSET', 'EXACT_TOKEN_TYPES'}
if needs_handwritten_doc != has_handwritten_doc:
message_parts = [f'ERROR: {rstfile} does not document all tokens!']
undocumented = needs_handwritten_doc - has_handwritten_doc
extra = has_handwritten_doc - needs_handwritten_doc
if undocumented:
message_parts.append(f'Undocumented tokens: {undocumented}')
if extra:
message_parts.append(f'Documented nonexistent tokens: {extra}')
exit('\n'.join(message_parts))
if update_file(outfile, token_inc_template % '\n'.join(names)): if update_file(outfile, token_inc_template % '\n'.join(names)):
print("%s regenerated from %s" % (outfile, infile)) print("%s regenerated from %s" % (outfile, infile))