mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 14:07:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

gh-120343: Fix column offsets of multiline tokens in tokenize (#120391)

Browse source
This commit is contained in:
Lysandros Nikolaou 2024-06-12 20:52:55 +02:00 • committed by GitHub
parent 127c1d2771
commit 4b5d3e0e72
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 24 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Lib/test/test_tokenize.py
View file

@ -1210,6 +1210,20 @@ a = f'''
FSTRING_END "\'\'\'" (2, 68) (2, 71)
""")
def test_multiline_non_ascii_fstring_with_expr(self):
self.check_tokenize("""\
f'''
🔗 This is a test {test_arg1}🔗
🔗'''""", """\
FSTRING_START "f\'\'\'" (1, 0) (1, 4)
FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21)
OP '{' (2, 21) (2, 22)
NAME 'test_arg1' (2, 22) (2, 31)
OP '}' (2, 31) (2, 32)
FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1)
FSTRING_END "\'\'\'" (3, 1) (3, 4)
""")
class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected):
# Format the tokens in s in a table format.