[3.12] gh-105017: Fix including additional NL token when using CRLF (GH-105022) (#105023)

Co-authored-by: Marta Gómez Macías <mgmacias@google.com>
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
This commit is contained in:
Miss Islington (bot) 2023-05-27 10:24:30 -07:00 committed by GitHub
parent edd0cb8e77
commit 2b176bc902
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 10 additions and 1 deletions

View file

@ -84,6 +84,14 @@ class TokenizeTest(TestCase):
NEWLINE '\\n' (4, 26) (4, 27)
DEDENT '' (5, 0) (5, 0)
""")
self.check_tokenize("foo='bar'\r\n", """\
NAME 'foo' (1, 0) (1, 3)
OP '=' (1, 3) (1, 4)
STRING "'bar'" (1, 4) (1, 9)
NEWLINE '\\n' (1, 9) (1, 10)
""")
indent_error_file = b"""\
def k(x):
x += 2