[3.14] gh-78319: Fix implementation of IMAP APPEND UTF8 (GH-9436) (GH-139406)

Make UTF8 support for the IMAP APPEND command RFC 6855 compliant.
(cherry picked from commit 408154d64a)

Co-authored-by: Gordon Messmer <gordon.messmer@gmail.com>

Lib/imaplib.py

@@ -497,8 +497,6 @@ class IMAP4:
         else:
             date_time = None
         literal = MapCRLF.sub(CRLF, message)
-        if self.utf8_enabled:
-            literal = b'UTF8 (' + literal + b')'
         self.literal = literal
         return self._simple_command(name, mailbox, flags, date_time)
 
@@ -1119,6 +1117,10 @@ class IMAP4:
                 literator = literal
             else:
                 literator = None
+                if self.utf8_enabled:
+                    data = data + bytes(' UTF8 (~{%s}' % len(literal), self._encoding)
+                    literal = literal + b')'
+                else:
                     data = data + bytes(' {%s}' % len(literal), self._encoding)
 
         if __debug__:

Lib/test/test_imaplib.py

@@ -373,7 +373,11 @@ class NewIMAPTestsMixin:
                 self._send_tagged(tag, 'OK', 'FAKEAUTH successful')
             def cmd_APPEND(self, tag, args):
                 self._send_textline('+')
-                self.server.response = yield
+                self.server.response = args
+                literal = yield
+                self.server.response.append(literal)
+                literal = yield
+                self.server.response.append(literal)
                 self._send_tagged(tag, 'OK', 'okay')
         client, server = self._setup(UTF8AppendServer)
         self.assertEqual(client._encoding, 'ascii')
@@ -384,10 +388,13 @@ class NewIMAPTestsMixin:
         self.assertEqual(code, 'OK')
         self.assertEqual(client._encoding, 'utf-8')
         msg_string = 'Subject: üñí©öðé'
-        typ, data = client.append(None, None, None, msg_string.encode('utf-8'))
+        typ, data = client.append(
+            None, None, None, (msg_string + '\n').encode('utf-8'))
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.response,
-            ('UTF8 (%s)\r\n' % msg_string).encode('utf-8'))
+            ['INBOX', 'UTF8',
+             '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'),
+             b')\r\n' ])
 
     def test_search_disallows_charset_in_utf8_mode(self):
         class UTF8Server(SimpleIMAPHandler):
@@ -887,7 +894,11 @@ class ThreadedNetworkedTests(unittest.TestCase):
         class UTF8AppendServer(self.UTF8Server):
             def cmd_APPEND(self, tag, args):
                 self._send_textline('+')
-                self.server.response = yield
+                self.server.response = args
+                literal = yield
+                self.server.response.append(literal)
+                literal = yield
+                self.server.response.append(literal)
                 self._send_tagged(tag, 'OK', 'okay')
 
         with self.reaped_pair(UTF8AppendServer) as (server, client):
@@ -901,12 +912,12 @@ class ThreadedNetworkedTests(unittest.TestCase):
             self.assertEqual(client._encoding, 'utf-8')
             msg_string = 'Subject: üñí©öðé'
             typ, data = client.append(
-                None, None, None, msg_string.encode('utf-8'))
+                None, None, None, (msg_string + '\n').encode('utf-8'))
             self.assertEqual(typ, 'OK')
-            self.assertEqual(
+            self.assertEqual(server.response,
-                server.response,
+                ['INBOX', 'UTF8',
-                ('UTF8 (%s)\r\n' % msg_string).encode('utf-8')
+                 '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'),
-            )
+                 b')\r\n' ])
 
     # XXX also need a test that makes sure that the Literal and Untagged_status
     # regexes uses unicode in UTF8 mode instead of the default ASCII.

Misc/NEWS.d/next/Library/2023-02-13-20-34-52.gh-issue-78319.V1zzed.rst

@@ -0,0 +1 @@
+UTF8 support for the IMAP APPEND command has been made RFC compliant.