mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 05:58:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Issue #10335: Add tokenize.open(), detect the file encoding using

Browse source 
tokenize.detect_encoding() and open it in read only mode.
This commit is contained in:
Victor Stinner 2010-11-09 01:08:59 +00:00
parent ae4836df6d
commit 58c0752a33
8 changed files with 54 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

23
Lib/test/test_tokenize.py
View file

@ -564,7 +564,8 @@ Non-ascii identifiers
from test import support
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
STRING, ENDMARKER, tok_name, detect_encoding)
STRING, ENDMARKER, tok_name, detect_encoding,
open as tokenize_open)
from io import BytesIO
from unittest import TestCase
import os, sys, glob
@ -857,6 +858,26 @@ class TestDetectEncoding(TestCase):
readline = self.get_readline((b'# coding: bad\n',))
self.assertRaises(SyntaxError, detect_encoding, readline)
def test_open(self):
filename = support.TESTFN + '.py'
self.addCleanup(support.unlink, filename)
# test coding cookie
for encoding in ('iso-8859-15', 'utf-8'):
with open(filename, 'w', encoding=encoding) as fp:
print("# coding: %s" % encoding, file=fp)
print("print('euro:\u20ac')", file=fp)
with tokenize_open(filename) as fp:
assert fp.encoding == encoding
assert fp.mode == 'r'
# test BOM (no coding cookie)
with open(filename, 'w', encoding='utf-8-sig') as fp:
print("print('euro:\u20ac')", file=fp)
with tokenize_open(filename) as fp:
assert fp.encoding == 'utf-8-sig'
assert fp.mode == 'r'
class TestTokenize(TestCase):
def test_tokenize(self):