bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)

Lib/test/test_repl.py

@@ -58,5 +58,41 @@ class TestInteractiveInterpreter(unittest.TestCase):
         # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
         self.assertIn(p.returncode, (1, 120))
 
+    @cpython_only
+    def test_multiline_string_parsing(self):
+        # bpo-39209: Multiline string tokens need to be handled in the tokenizer
+        # in two places: the interactive path and the non-interactive path.
+        user_input = '''\
+        x = """<?xml version="1.0" encoding="iso-8859-1"?>
+        <test>
+            <Users>
+                <fun25>
+                    <limits>
+                        <total>0KiB</total>
+                        <kbps>0</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </limits>
+                    <usages>
+                        <total>16738211KiB</total>
+                        <kbps>237.15</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </usages>
+                    <time_to_refresh>never</time_to_refresh>
+                    <limit_exceeded_URL>none</limit_exceeded_URL>
+                </fun25>
+            </Users>
+        </test>"""
+        '''
+        user_input = dedent(user_input)
+        user_input = user_input.encode()
+        p = spawn_repl()
+        with SuppressCrashReport():
+            p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+
+
 if __name__ == "__main__":
     unittest.main()