mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

reuse tokenize.detect_encoding in linecache instead of a custom solution

Browse source 
patch by Victor Stinner #4016
This commit is contained in:
Benjamin Peterson 2009-03-24 22:30:15 +00:00
parent a8abe86331
commit 9b8d24b17d
2 changed files with 8 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Lib/tokenize.py
View file

@ -27,7 +27,6 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
import re, string, sys
from token import *
from codecs import lookup, BOM_UTF8
from itertools import chain, repeat
cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
import token
@ -327,13 +326,15 @@ def tokenize(readline):
which tells you which encoding was used to decode the bytes stream.
"""
encoding, consumed = detect_encoding(readline)
def readline_generator():
def readline_generator(consumed):
for line in consumed:
yield line
while True:
try:
yield readline()
except StopIteration:
return
chained = chain(consumed, readline_generator())
chained = readline_generator(consumed)
return _tokenize(chained.__next__, encoding)