mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-07 19:35:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-104976: Ensure trailing dedent tokens are emitted as the previous tokenizer (#104980)

Browse source 
Signed-off-by: Pablo Galindo <pablogsal@gmail.com>
This commit is contained in:
Pablo Galindo Salgado 2023-05-26 22:02:26 +01:00 • committed by GitHub
parent 402ee5a68b
commit 46b52e6e2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 34 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Lib/tokenize.py
View file

@ -447,13 +447,8 @@ def tokenize(readline):
def _tokenize(rl_gen, encoding):
source = b"".join(rl_gen).decode(encoding)
token = None
for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True):
yield token
if token is not None:
last_line, _ = token.start
yield TokenInfo(ENDMARKER, '', (last_line + 1, 0), (last_line + 1, 0), '')
def generate_tokens(readline):
"""Tokenize a source reading Python code as unicode strings.