mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Issue #9974: When untokenizing, use row info to insert backslash+newline.

Browse source 
Original patches by A. Kuchling and G. Rees (#12691).
This commit is contained in:
Terry Jan Reedy 2014-02-23 23:33:08 -05:00
parent 938ba685dc
commit 9dc3a36c84
2 changed files with 22 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Lib/test/test_tokenize.py
View file

@ -2,7 +2,7 @@ doctests = """
Tests for the tokenize module.
The tests can be really simple. Given a small fragment of source
code, print out a table with tokens. The ENDMARK is omitted for
code, print out a table with tokens. The ENDMARKER is omitted for
brevity.
>>> dump_tokens("1 + 1")
@ -1180,6 +1180,7 @@ class TestTokenize(TestCase):
class UntokenizeTest(TestCase):
def test_bad_input_order(self):
# raise if previous row
u = Untokenizer()
u.prev_row = 2
u.prev_col = 2
@ -1187,8 +1188,22 @@ class UntokenizeTest(TestCase):
u.add_whitespace((1,3))
self.assertEqual(cm.exception.args[0],
'start (1,3) precedes previous end (2,2)')
# raise if previous column in row
self.assertRaises(ValueError, u.add_whitespace, (2,1))
def test_backslash_continuation(self):
# The problem is that <whitespace>\<newline> leaves no token
u = Untokenizer()
u.prev_row = 1
u.prev_col = 1
u.tokens = []
u.add_whitespace((2, 0))
self.assertEqual(u.tokens, ['\\\n'])
u.prev_row = 2
u.add_whitespace((4, 4))
self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', ' '])
self.assertTrue(roundtrip('a\n b\n c\n \\\n c\n'))
def test_iter_compat(self):
u = Untokenizer()
token = (NAME, 'Hello')