mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-07 15:42:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 18

use some more itertools magic to make '' be yielded after readline is done

Browse source
This commit is contained in:
Benjamin Peterson 2009-11-14 18:09:17 +00:00
parent 21db77e396
commit 81dd8b9594
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Lib/tokenize.py
View file

@ -379,10 +379,11 @@ def tokenize(readline):
""" """
# This import is here to avoid problems when the itertools module is not # This import is here to avoid problems when the itertools module is not
# built yet and tokenize is imported. # built yet and tokenize is imported.
from itertools import chain from itertools import chain, repeat
encoding, consumed = detect_encoding(readline) encoding, consumed = detect_encoding(readline)
rl_iter = iter(readline, "") rl_gen = iter(readline, b"")
return _tokenize(chain(consumed, rl_iter).__next__, encoding) empty = repeat(b"")
return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
def _tokenize(readline, encoding): def _tokenize(readline, encoding):