Issue #16182: Merge readline locale fix from 3.5

Lib/test/test_readline.py

@@ -9,12 +9,15 @@ import subprocess
 import sys
 import tempfile
 import unittest
-from test.support import import_module, unlink
+from test.support import import_module, unlink, TESTFN
 from test.support.script_helper import assert_python_ok
 
 # Skip tests if there is no readline module
 readline = import_module('readline')
 
+@unittest.skipUnless(hasattr(readline, "clear_history"),
+                     "The history update test cannot be run because the "
+                     "clear_history method is not available.")
 class TestHistoryManipulation (unittest.TestCase):
     """
     These tests were added to check that the libedit emulation on OSX and the
@@ -22,9 +25,6 @@ class TestHistoryManipulation (unittest.TestCase):
     why the tests cover only a small subset of the interface.
     """
 
-    @unittest.skipUnless(hasattr(readline, "clear_history"),
-                         "The history update test cannot be run because the "
-                         "clear_history method is not available.")
     def testHistoryUpdates(self):
         readline.clear_history()
 
@@ -87,6 +87,21 @@ class TestHistoryManipulation (unittest.TestCase):
         # write_history_file can create the target
         readline.write_history_file(hfilename)
 
+    def test_nonascii_history(self):
+        readline.clear_history()
+        try:
+            readline.add_history("entrée 1")
+        except UnicodeEncodeError as err:
+            self.skipTest("Locale cannot encode test data: " + format(err))
+        readline.add_history("entrée 2")
+        readline.replace_history_item(1, "entrée 22")
+        readline.write_history_file(TESTFN)
+        self.addCleanup(os.remove, TESTFN)
+        readline.clear_history()
+        readline.read_history_file(TESTFN)
+        self.assertEqual(readline.get_history_item(1), "entrée 1")
+        self.assertEqual(readline.get_history_item(2), "entrée 22")
+
 
 class TestReadline(unittest.TestCase):
 
@@ -116,6 +131,69 @@ print("History length:", readline.get_current_history_length())
         output = run_pty(self.auto_history_script.format(False))
         self.assertIn(b"History length: 0\r\n", output)
 
+    def test_nonascii(self):
+        try:
+            readline.add_history("\xEB\xEF")
+        except UnicodeEncodeError as err:
+            self.skipTest("Locale cannot encode test data: " + format(err))
+
+        script = r"""import readline
+
+if readline.__doc__ and "libedit" in readline.__doc__:
+    readline.parse_and_bind(r'bind ^B ed-prev-char')
+    readline.parse_and_bind(r'bind "\t" rl_complete')
+    readline.parse_and_bind('bind -s ^A "|t\xEB[after]"')
+else:
+    readline.parse_and_bind(r'Control-b: backward-char')
+    readline.parse_and_bind(r'"\t": complete')
+    readline.parse_and_bind(r'set disable-completion off')
+    readline.parse_and_bind(r'set show-all-if-ambiguous off')
+    readline.parse_and_bind(r'set show-all-if-unmodified off')
+    readline.parse_and_bind('Control-a: "|t\xEB[after]"')
+
+def pre_input_hook():
+    readline.insert_text("[\xEFnserted]")
+    readline.redisplay()
+readline.set_pre_input_hook(pre_input_hook)
+
+def completer(text, state):
+    if text == "t\xEB":
+        if state == 0:
+            print("text", ascii(text))
+            print("line", ascii(readline.get_line_buffer()))
+            print("indexes", readline.get_begidx(), readline.get_endidx())
+            return "t\xEBnt"
+        if state == 1:
+            return "t\xEBxt"
+    if text == "t\xEBx" and state == 0:
+        return "t\xEBxt"
+    return None
+readline.set_completer(completer)
+
+def display(substitution, matches, longest_match_length):
+    print("substitution", ascii(substitution))
+    print("matches", ascii(matches))
+readline.set_completion_display_matches_hook(display)
+
+print("result", ascii(input()))
+print("history", ascii(readline.get_history_item(1)))
+"""
+
+        input = b"\x01"  # Ctrl-A, expands to "|t\xEB[after]"
+        input += b"\x02" * len("[after]")  # Move cursor back
+        input += b"\t\t"  # Display possible completions
+        input += b"x\t"  # Complete "t\xEBx" -> "t\xEBxt"
+        input += b"\r"
+        output = run_pty(script, input)
+        self.assertIn(b"text 't\\xeb'\r\n", output)
+        self.assertIn(b"line '[\\xefnserted]|t\\xeb[after]'\r\n", output)
+        self.assertIn(b"indexes 11 13\r\n", output)
+        self.assertIn(b"substitution 't\\xeb'\r\n", output)
+        self.assertIn(b"matches ['t\\xebnt', 't\\xebxt']\r\n", output)
+        expected = br"'[\xefnserted]|t\xebxt[after]'"
+        self.assertIn(b"result " + expected + b"\r\n", output)
+        self.assertIn(b"history " + expected + b"\r\n", output)
+
 
 def run_pty(script, input=b"dummy input\r"):
     pty = import_module('pty')
@@ -148,7 +226,7 @@ def run_pty(script, input=b"dummy input\r"):
                     try:
                         chunk = os.read(master, 0x10000)
                     except OSError as err:
-                        # Linux raises EIO when the slave is closed
+                        # Linux raises EIO when slave is closed (Issue 5380)
                         if err.errno != EIO:
                             raise
                         chunk = b""
@@ -156,7 +234,13 @@ def run_pty(script, input=b"dummy input\r"):
                         return output
                     output.extend(chunk)
                 if events & selectors.EVENT_WRITE:
-                    input = input[os.write(master, input):]
+                    try:
+                        input = input[os.write(master, input):]
+                    except OSError as err:
+                        # Apparently EIO means the slave was closed
+                        if err.errno != EIO:
+                            raise
+                        input = b""  # Stop writing
                     if not input:
                         sel.modify(master, selectors.EVENT_READ)