mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 00:48:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-115154: Fix untokenize handling of unicode named literals (#115171)

Browse source
This commit is contained in:
Pablo Galindo Salgado 2024-02-19 14:54:10 +00:00 • committed by GitHub
parent d504968983
commit ecf16ee50e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 85 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

40
Lib/test/test_tokenize.py
View file

@ -1877,6 +1877,43 @@ class TestRoundtrip(TestCase):
" print('Can not import' # comment2\n)"
"else: print('Loaded')\n")
self.check_roundtrip("f'\\N{EXCLAMATION MARK}'")
self.check_roundtrip(r"f'\\N{SNAKE}'")
self.check_roundtrip(r"f'\\N{{SNAKE}}'")
self.check_roundtrip(r"f'\N{SNAKE}'")
self.check_roundtrip(r"f'\\\N{SNAKE}'")
self.check_roundtrip(r"f'\\\\\N{SNAKE}'")
self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'")
self.check_roundtrip(r"f'\\N{1}'")
self.check_roundtrip(r"f'\\\\N{2}'")
self.check_roundtrip(r"f'\\\\\\N{3}'")
self.check_roundtrip(r"f'\\\\\\\\N{4}'")
self.check_roundtrip(r"f'\\N{{'")
self.check_roundtrip(r"f'\\\\N{{'")
self.check_roundtrip(r"f'\\\\\\N{{'")
self.check_roundtrip(r"f'\\\\\\\\N{{'")
cases = [
"""
if 1:
"foo"
"bar"
""",
"""
if 1:
("foo"
"bar")
""",
"""
if 1:
"foo"
"bar"
""" ]
for case in cases:
self.check_roundtrip(case)
def test_continuation(self):
# Balancing continuation
self.check_roundtrip("a = (3,4, \n"
@ -1911,9 +1948,6 @@ class TestRoundtrip(TestCase):
tempdir = os.path.dirname(__file__) or os.curdir
testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py"))
# TODO: Remove this once we can untokenize PEP 701 syntax
testfiles.remove(os.path.join(tempdir, "test_fstring.py"))
if not support.is_resource_enabled("cpu"):
testfiles = random.sample(testfiles, 10)