mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

[3.12] gh-104976: Ensure trailing dedent tokens are emitted as the previous tokenizer (GH-104980) (#105000)

Browse source
This commit is contained in:
Miss Islington (bot) 2023-05-26 15:14:45 -07:00 • committed by GitHub
parent 05189f3054
commit 2c02c68867
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 34 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Lib/tokenize.py
View file

@ -447,13 +447,8 @@ def tokenize(readline):
def _tokenize(rl_gen, encoding):
source = b"".join(rl_gen).decode(encoding)
token = None
for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True):
yield token
if token is not None:
last_line, _ = token.start
yield TokenInfo(ENDMARKER, '', (last_line + 1, 0), (last_line + 1, 0), '')
def generate_tokens(readline):
"""Tokenize a source reading Python code as unicode strings.