bpo-13153: Use OS native encoding for converting between Python and Tcl. (GH-16545)

On Windows use UTF-16 (or UTF-32 for 32-bit Tcl_UniChar) with the
"surrogatepass" error handler for converting to/from Tcl Unicode objects.

On Linux use UTF-8 with the "surrogateescape" error handler for converting
to/from Tcl String objects.

Converting strings from Tcl to Python and back now never fails
(except MemoryError).

Lib/idlelib/editor.py

@@ -358,21 +358,6 @@ class EditorWindow(object):
             Font(text, font=text.cget('font')).measure('0')
         self.width = pixel_width // zero_char_width
 
-    def _filename_to_unicode(self, filename):
-        """Return filename as BMP unicode so displayable in Tk."""
-        # Decode bytes to unicode.
-        if isinstance(filename, bytes):
-            try:
-                filename = filename.decode(self.filesystemencoding)
-            except UnicodeDecodeError:
-                try:
-                    filename = filename.decode(self.encoding)
-                except UnicodeDecodeError:
-                    # byte-to-byte conversion
-                    filename = filename.decode('iso8859-1')
-        # Replace non-BMP char with diamond questionmark.
-        return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename)
-
     def new_callback(self, event):
         dirname, basename = self.io.defaultfilename()
         self.flist.new(dirname)
@@ -963,10 +948,8 @@ class EditorWindow(object):
             menu.delete(0, END)  # clear, and rebuild:
             for i, file_name in enumerate(rf_list):
                 file_name = file_name.rstrip()  # zap \n
-                # make unicode string to display non-ASCII chars correctly
-                ufile_name = self._filename_to_unicode(file_name)
                 callback = instance.__recent_file_callback(file_name)
-                menu.add_command(label=ulchars[i] + " " + ufile_name,
+                menu.add_command(label=ulchars[i] + " " + file_name,
                                  command=callback,
                                  underline=0)
 
@@ -1004,16 +987,10 @@ class EditorWindow(object):
 
     def short_title(self):
         filename = self.io.filename
-        if filename:
-            filename = os.path.basename(filename)
-        else:
-            filename = "untitled"
-        # return unicode string to display non-ASCII chars correctly
-        return self._filename_to_unicode(filename)
+        return os.path.basename(filename) if filename else "untitled"
 
     def long_title(self):
-        # return unicode string to display non-ASCII chars correctly
-        return self._filename_to_unicode(self.io.filename or "")
+        return self.io.filename or ""
 
     def center_insert_event(self, event):
         self.center()

Lib/idlelib/idle_test/test_editor.py

@@ -30,18 +30,6 @@ class EditorWindowTest(unittest.TestCase):
         e._close()
 
 
-class EditorFunctionTest(unittest.TestCase):
-
-    def test_filename_to_unicode(self):
-        func = Editor._filename_to_unicode
-        class dummy():
-            filesystemencoding = 'utf-8'
-        pairs = (('abc', 'abc'), ('a\U00011111c', 'a\ufffdc'),
-                 (b'abc', 'abc'), (b'a\xf0\x91\x84\x91c', 'a\ufffdc'))
-        for inp, out in pairs:
-            self.assertEqual(func(dummy, inp), out)
-
-
 class TestGetLineIndent(unittest.TestCase):
     def test_empty_lines(self):
         for tabwidth in [1, 2, 4, 6, 8]:

Lib/idlelib/pyshell.py

@@ -679,14 +679,6 @@ class ModifiedInterpreter(InteractiveInterpreter):
         self.more = 0
         # at the moment, InteractiveInterpreter expects str
         assert isinstance(source, str)
-        #if isinstance(source, str):
-        #    from idlelib import iomenu
-        #    try:
-        #        source = source.encode(iomenu.encoding)
-        #    except UnicodeError:
-        #        self.tkconsole.resetoutput()
-        #        self.write("Unsupported characters in input\n")
-        #        return
         # InteractiveInterpreter.runsource() calls its runcode() method,
         # which is overridden (see below)
         return InteractiveInterpreter.runsource(self, source, filename)
@@ -1298,16 +1290,6 @@ class PyShell(OutputWindow):
         self.set_line_and_column()
 
     def write(self, s, tags=()):
-        if isinstance(s, str) and len(s) and max(s) > '\uffff':
-            # Tk doesn't support outputting non-BMP characters
-            # Let's assume what printed string is not very long,
-            # find first non-BMP character and construct informative
-            # UnicodeEncodeError exception.
-            for start, char in enumerate(s):
-                if char > '\uffff':
-                    break
-            raise UnicodeEncodeError("UCS-2", char, start, start+1,
-                                     'Non-BMP character not supported in Tk')
         try:
             self.text.mark_gravity("iomark", "right")
             count = OutputWindow.write(self, s, tags, "iomark")

Lib/idlelib/runscript.py

@@ -147,8 +147,7 @@ class ScriptBinding:
         interp = self.shell.interp
         if pyshell.use_subprocess and restart:
             interp.restart_subprocess(
-                    with_cwd=False, filename=
-                    self.editwin._filename_to_unicode(filename))
+                    with_cwd=False, filename=filename)
         dirname = os.path.dirname(filename)
         argv = [filename]
         if self.cli_args: