Issue #1210: Fixed imaplib

Patch by Victor Stinner, reviewed by Barry Warsaw.

Doc/library/imaplib.rst

@@ -75,7 +75,7 @@ The second subclass allows for connections created by a child process:
 
    This is a subclass derived from :class:`IMAP4` that connects to the
    ``stdin/stdout`` file descriptors created by passing *command* to
-   ``os.popen2()``.
+   ``subprocess.Popen()``.
 
 
 The following utility functions are defined:
@@ -468,13 +468,6 @@ An :class:`IMAP4` instance has the following methods:
 
    Allow simple extension commands notified by server in ``CAPABILITY`` response.
 
-Instances of :class:`IMAP4_SSL` have just one additional method:
-
-
-.. method:: IMAP4_SSL.ssl()
-
-   Returns SSLObject instance used for the secure connection with the server.
-
 The following attributes are defined on instances of :class:`IMAP4`:
 
 

Lib/imaplib.py

@@ -22,14 +22,14 @@ Public functions:       Internaldate2tuple
 
 __version__ = "2.58"
 
-import binascii, os, random, re, socket, sys, time
+import binascii, random, re, socket, subprocess, sys, time
 
 __all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple",
            "Int2AP", "ParseFlags", "Time2Internaldate"]
 
 #       Globals
 
-CRLF = '\r\n'
+CRLF = b'\r\n'
 Debug = 0
 IMAP4_PORT = 143
 IMAP4_SSL_PORT = 993
@@ -81,19 +81,19 @@ Commands = {
 
 #       Patterns to match server responses
 
-Continuation = re.compile(r'\+( (?P<data>.*))?')
+Continuation = re.compile(br'\+( (?P<data>.*))?')
-Flags = re.compile(r'.*FLAGS \((?P<flags>[^\)]*)\)')
+Flags = re.compile(br'.*FLAGS \((?P<flags>[^\)]*)\)')
-InternalDate = re.compile(r'.*INTERNALDATE "'
+InternalDate = re.compile(br'.*INTERNALDATE "'
-        r'(?P<day>[ 0123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])'
+        br'(?P<day>[ 0123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])'
-        r' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])'
+        br' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])'
-        r' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])'
+        br' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])'
-        r'"')
+        br'"')
-Literal = re.compile(r'.*{(?P<size>\d+)}$', re.ASCII)
+Literal = re.compile(br'.*{(?P<size>\d+)}$', re.ASCII)
-MapCRLF = re.compile(r'\r\n|\r|\n')
+MapCRLF = re.compile(br'\r\n|\r|\n')
-Response_code = re.compile(r'\[(?P<type>[A-Z-]+)( (?P<data>[^\]]*))?\]')
+Response_code = re.compile(br'\[(?P<type>[A-Z-]+)( (?P<data>[^\]]*))?\]')
-Untagged_response = re.compile(r'\* (?P<type>[A-Z-]+)( (?P<data>.*))?')
+Untagged_response = re.compile(br'\* (?P<type>[A-Z-]+)( (?P<data>.*))?')
 Untagged_status = re.compile(
-    r'\* (?P<data>\d+) (?P<type>[A-Z-]+)( (?P<data2>.*))?', re.ASCII)
+    br'\* (?P<data>\d+) (?P<type>[A-Z-]+)( (?P<data2>.*))?', re.ASCII)
 
 
 
@@ -147,7 +147,7 @@ class IMAP4:
     class abort(error): pass        # Service errors - close and retry
     class readonly(abort): pass     # Mailbox status changed to READ-ONLY
 
-    mustquote = re.compile(r"[^\w!#$%&'*+,.:;<=>?^`|~-]", re.ASCII)
+    mustquote = re.compile(br"[^\w!#$%&'*+,.:;<=>?^`|~-]", re.ASCII)
 
     def __init__(self, host = '', port = IMAP4_PORT):
         self.debug = Debug
@@ -167,9 +167,9 @@ class IMAP4:
         # and compile tagged response matcher.
 
         self.tagpre = Int2AP(random.randint(4096, 65535))
-        self.tagre = re.compile(r'(?P<tag>'
+        self.tagre = re.compile(br'(?P<tag>'
                         + self.tagpre
-                        + r'\d+) (?P<type>[A-Z]+) (?P<data>.*)', re.ASCII)
+                        + br'\d+) (?P<type>[A-Z]+) (?P<data>.*)', re.ASCII)
 
         # Get server welcome message,
         # request and store CAPABILITY response.
@@ -193,7 +193,9 @@ class IMAP4:
         typ, dat = self.capability()
         if dat == [None]:
             raise self.error('no CAPABILITY response from server')
-        self.capabilities = tuple(dat[-1].upper().split())
+        dat = str(dat[-1], "ASCII")
+        dat = dat.upper()
+        self.capabilities = tuple(dat.split())
 
         if __debug__:
             if self.debug >= 3:
@@ -219,6 +221,11 @@ class IMAP4:
     #       Overridable methods
 
 
+    def _create_socket(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.connect((self.host, self.port))
+        return sock
+
     def open(self, host = '', port = IMAP4_PORT):
         """Setup connection to remote server on "host:port"
             (default: localhost:standard IMAP4 port).
@@ -227,14 +234,21 @@ class IMAP4:
         """
         self.host = host
         self.port = port
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.sock = self._create_socket()
-        self.sock.connect((host, port))
         self.file = self.sock.makefile('rb')
 
 
     def read(self, size):
         """Read 'size' bytes from remote."""
-        return self.file.read(size)
+        chunks = []
+        read = 0
+        while read < size:
+            data = self.file.read(min(size-read, 4096))
+            if not data:
+                break
+            read += len(data)
+            chunks.append(data)
+        return b''.join(chunks)
 
 
     def readline(self):
@@ -791,12 +805,12 @@ class IMAP4:
 
 
     def _append_untagged(self, typ, dat):
-
+        if dat is None:
-        if dat is None: dat = ''
+            dat = b''
         ur = self.untagged_responses
         if __debug__:
             if self.debug >= 5:
-                self._mesg('untagged_responses[%s] %s += ["%s"]' %
+                self._mesg('untagged_responses[%s] %s += ["%r"]' %
                         (typ, len(ur.get(typ,'')), dat))
         if typ in ur:
             ur[typ].append(dat)
@@ -828,10 +842,14 @@ class IMAP4:
             raise self.readonly('mailbox status changed to READ-ONLY')
 
         tag = self._new_tag()
-        data = '%s %s' % (tag, name)
+        name = bytes(name, 'ASCII')
+        data = tag + b' ' + name
         for arg in args:
             if arg is None: continue
-            data = '%s %s' % (data, self._checkquote(arg))
+            if isinstance(arg, str):
+                arg = bytes(arg, "ASCII")
+            #data = data + b' ' + self._checkquote(arg)
+            data = data + b' ' + arg
 
         literal = self.literal
         if literal is not None:
@@ -840,16 +858,16 @@ class IMAP4:
                 literator = literal
             else:
                 literator = None
-                data = '%s {%s}' % (data, len(literal))
+                data = data + bytes(' {%s}' % len(literal), 'ASCII')
 
         if __debug__:
             if self.debug >= 4:
-                self._mesg('> %s' % data)
+                self._mesg('> %r' % data)
             else:
-                self._log('> %s' % data)
+                self._log('> %r' % data)
 
         try:
-            self.send('%s%s' % (data, CRLF))
+            self.send(data + CRLF)
         except (socket.error, OSError) as val:
             raise self.abort('socket error: %s' % val)
 
@@ -915,6 +933,7 @@ class IMAP4:
                 raise self.abort('unexpected tagged response: %s' % resp)
 
             typ = self.mo.group('type')
+            typ = str(typ, 'ASCII')
             dat = self.mo.group('data')
             self.tagged_commands[tag] = (typ, [dat])
         else:
@@ -936,9 +955,10 @@ class IMAP4:
                 raise self.abort("unexpected response: '%s'" % resp)
 
             typ = self.mo.group('type')
+            typ = str(typ, 'ascii')
             dat = self.mo.group('data')
-            if dat is None: dat = ''        # Null untagged response
+            if dat is None: dat = b''        # Null untagged response
-            if dat2: dat = dat + ' ' + dat2
+            if dat2: dat = dat + b' ' + dat2
 
             # Is there a literal to come?
 
@@ -965,11 +985,13 @@ class IMAP4:
         # Bracketed response information?
 
         if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat):
-            self._append_untagged(self.mo.group('type'), self.mo.group('data'))
+            typ = self.mo.group('type')
+            typ = str(typ, "ASCII")
+            self._append_untagged(typ, self.mo.group('data'))
 
         if __debug__:
             if self.debug >= 1 and typ in ('NO', 'BAD', 'BYE'):
-                self._mesg('%s response: %s' % (typ, dat))
+                self._mesg('%s response: %r' % (typ, dat))
 
         return resp
 
@@ -1007,9 +1029,9 @@ class IMAP4:
         line = line[:-2]
         if __debug__:
             if self.debug >= 4:
-                self._mesg('< %s' % line)
+                self._mesg('< %r' % line)
             else:
-                self._log('< %s' % line)
+                self._log('< %r' % line)
         return line
 
 
@@ -1021,13 +1043,13 @@ class IMAP4:
         self.mo = cre.match(s)
         if __debug__:
             if self.mo is not None and self.debug >= 5:
-                self._mesg("\tmatched r'%s' => %r" % (cre.pattern, self.mo.groups()))
+                self._mesg("\tmatched r'%r' => %r" % (cre.pattern, self.mo.groups()))
         return self.mo is not None
 
 
     def _new_tag(self):
 
-        tag = '%s%s' % (self.tagpre, self.tagnum)
+        tag = self.tagpre + bytes(str(self.tagnum), 'ASCII')
         self.tagnum = self.tagnum + 1
         self.tagged_commands[tag] = None
         return tag
@@ -1038,8 +1060,6 @@ class IMAP4:
         # Must quote command args if non-alphanumeric chars present,
         # and not already quoted.
 
-        if type(arg) is not type(''):
-            return arg
         if len(arg) >= 2 and (arg[0],arg[-1]) in (('(',')'),('"','"')):
             return arg
         if arg and self.mustquote.search(arg) is None:
@@ -1049,10 +1069,10 @@ class IMAP4:
 
     def _quote(self, arg):
 
-        arg = arg.replace('\\', '\\\\')
+        arg = arg.replace(b'\\', b'\\\\')
-        arg = arg.replace('"', '\\"')
+        arg = arg.replace(b'"', b'\\"')
 
-        return '"%s"' % arg
+        return b'"' + arg + b'"'
 
 
     def _simple_command(self, name, *args):
@@ -1061,7 +1081,6 @@ class IMAP4:
 
 
     def _untagged_response(self, typ, dat, name):
-
         if typ == 'NO':
             return typ, dat
         if not name in self.untagged_responses:
@@ -1137,73 +1156,17 @@ else:
             self.certfile = certfile
             IMAP4.__init__(self, host, port)
 
+        def _create_socket(self):
+            sock = IMAP4._create_socket(self)
+            return ssl.wrap_socket(sock, self.keyfile, self.certfile)
 
-        def open(self, host = '', port = IMAP4_SSL_PORT):
+        def open(self, host='', port=IMAP4_SSL_PORT):
             """Setup connection to remote server on "host:port".
                 (default: localhost:standard IMAP4 SSL port).
             This connection will be used by the routines:
                 read, readline, send, shutdown.
             """
-            self.host = host
+            IMAP4.open(self, host, port)
-            self.port = port
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            sock.connect((host, port))
-            self.sock = ssl.wrap_socket(sock, self.keyfile, self.certfile)
-            self.file = self.sock.makefile('rb')
-
-
-        def read(self, size):
-            """Read 'size' bytes from remote."""
-            # sslobj.read() sometimes returns < size bytes
-            chunks = []
-            read = 0
-            while read < size:
-                data = self.sslobj.read(min(size-read, 16384))
-                read += len(data)
-                chunks.append(data)
-
-            return b''.join(chunks)
-
-
-        def readline(self):
-            """Read line from remote."""
-            line = []
-            while 1:
-                char = self.sslobj.read(1)
-                line.append(char)
-                if char == b"\n": return b''.join(line)
-
-
-        def send(self, data):
-            """Send data to remote."""
-            bytes = len(data)
-            while bytes > 0:
-                sent = self.sslobj.write(data)
-                if sent == bytes:
-                    break    # avoid copy
-                data = data[sent:]
-                bytes = bytes - sent
-
-
-        def shutdown(self):
-            """Close I/O established in "open"."""
-            self.sock.close()
-
-
-        def socket(self):
-            """Return socket instance used to connect to IMAP4 server.
-
-            socket = <instance>.socket()
-            """
-            return self.sock
-
-
-        def ssl(self):
-            """Return SSLObject instance used to communicate with the IMAP4 server.
-
-            ssl = ssl.wrap_socket(<instance>.socket)
-            """
-            return self.sock
 
     __all__.append("IMAP4_SSL")
 
@@ -1214,7 +1177,7 @@ class IMAP4_stream(IMAP4):
 
     Instantiate with: IMAP4_stream(command)
 
-            where "command" is a string that can be passed to os.popen2()
+            where "command" is a string that can be passed to subprocess.Popen()
 
     for more documentation see the docstring of the parent class IMAP4.
     """
@@ -1234,8 +1197,11 @@ class IMAP4_stream(IMAP4):
         self.port = None
         self.sock = None
         self.file = None
-        self.writefile, self.readfile = os.popen2(self.command)
+        self.process = subprocess.Popen(self.command,
-
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            shell=True, close_fds=True)
+        self.writefile = self.process.stdin
+        self.readfile = self.process.stdout
 
     def read(self, size):
         """Read 'size' bytes from remote."""
@@ -1257,6 +1223,7 @@ class IMAP4_stream(IMAP4):
         """Close I/O established in "open"."""
         self.readfile.close()
         self.writefile.close()
+        self.process.wait()
 
 
 
@@ -1355,11 +1322,11 @@ def Int2AP(num):
 
     """Convert integer to A-P string representation."""
 
-    val = ''; AP = 'ABCDEFGHIJKLMNOP'
+    val = b''; AP = b'ABCDEFGHIJKLMNOP'
     num = int(abs(num))
     while num:
         num, mod = divmod(num, 16)
-        val = AP[mod] + val
+        val = AP[mod:mod+1] + val
     return val
 
 

Misc/NEWS

@@ -15,6 +15,8 @@ What's New in Python 3.0 beta 5
 Core and Builtins
 -----------------
 
+- Issue #1210: Fixed imaplib and its documentation.
+
 - Issue #4233: Changed semantic of ``_fileio.FileIO``'s ``close()`` 
   method on file objects with closefd=False. The file descriptor is still
   kept open but the file object behaves like a closed file. The ``FileIO``