Issue #10883: Fix socket leaks in urllib.request.

* ftpwrapper now uses reference counting to ensure that the underlying socket
  is closed when the ftpwrapper object is no longer in use
* ftplib.FTP.ntransfercmd() now closes the socket if an error occurs

Initial patch by Victor Stinner.

Lib/ftplib.py

@@ -325,33 +325,39 @@ class FTP:
         if self.passiveserver:
             host, port = self.makepasv()
             conn = socket.create_connection((host, port), self.timeout)
-            if rest is not None:
+            try:
-                self.sendcmd("REST %s" % rest)
+                if rest is not None:
-            resp = self.sendcmd(cmd)
+                    self.sendcmd("REST %s" % rest)
-            # Some servers apparently send a 200 reply to
+                resp = self.sendcmd(cmd)
-            # a LIST or STOR command, before the 150 reply
+                # Some servers apparently send a 200 reply to
-            # (and way before the 226 reply). This seems to
+                # a LIST or STOR command, before the 150 reply
-            # be in violation of the protocol (which only allows
+                # (and way before the 226 reply). This seems to
-            # 1xx or error messages for LIST), so we just discard
+                # be in violation of the protocol (which only allows
-            # this response.
+                # 1xx or error messages for LIST), so we just discard
-            if resp[0] == '2':
+                # this response.
-                resp = self.getresp()
+                if resp[0] == '2':
-            if resp[0] != '1':
+                    resp = self.getresp()
-                raise error_reply, resp
+                if resp[0] != '1':
+                    raise error_reply, resp
+            except:
+                conn.close()
+                raise
         else:
             sock = self.makeport()
-            if rest is not None:
+            try:
-                self.sendcmd("REST %s" % rest)
+                if rest is not None:
-            resp = self.sendcmd(cmd)
+                    self.sendcmd("REST %s" % rest)
-            # See above.
+                resp = self.sendcmd(cmd)
-            if resp[0] == '2':
+                # See above.
-                resp = self.getresp()
+                if resp[0] == '2':
-            if resp[0] != '1':
+                    resp = self.getresp()
-                raise error_reply, resp
+                if resp[0] != '1':
-            conn, sockaddr = sock.accept()
+                    raise error_reply, resp
-            if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT:
+                conn, sockaddr = sock.accept()
-                conn.settimeout(self.timeout)
+                if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT:
-            sock.close()
+                    conn.settimeout(self.timeout)
+            finally:
+                sock.close()
         if resp[:3] == '150':
             # this is conditional in case we received a 125
             size = parse150(resp)

Lib/test/test_urllib2.py

@@ -611,6 +611,7 @@ class HandlerTests(unittest.TestCase):
             def retrfile(self, filename, filetype):
                 self.filename, self.filetype = filename, filetype
                 return StringIO.StringIO(self.data), len(self.data)
+            def close(self): pass
 
         class NullFTPHandler(urllib2.FTPHandler):
             def __init__(self, data): self.data = data

Lib/test/test_urllib2net.py

@@ -231,6 +231,7 @@ class OtherNetworkTests(unittest.TestCase):
         handlers = []
 
         cfh = urllib2.CacheFTPHandler()
+        self.addCleanup(cfh.clear_cache)
         cfh.setTimeout(1)
         handlers.append(cfh)
 

Lib/urllib.py

@@ -850,13 +850,16 @@ class ftpwrapper:
     """Class used by open_ftp() for cache of open FTP connections."""
 
     def __init__(self, user, passwd, host, port, dirs,
-                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                 persistent=False):
         self.user = user
         self.passwd = passwd
         self.host = host
         self.port = port
         self.dirs = dirs
         self.timeout = timeout
+        self.refcount = 0
+        self.keepalive = persistent
         self.init()
 
     def init(self):
@@ -883,7 +886,7 @@ class ftpwrapper:
             # Try to retrieve as a file
             try:
                 cmd = 'RETR ' + file
-                conn = self.ftp.ntransfercmd(cmd)
+                conn, retrlen = self.ftp.ntransfercmd(cmd)
             except ftplib.error_perm, reason:
                 if str(reason)[:3] != '550':
                     raise IOError, ('ftp error', reason), sys.exc_info()[2]
@@ -903,11 +906,14 @@ class ftpwrapper:
                 cmd = 'LIST ' + file
             else:
                 cmd = 'LIST'
-            conn = self.ftp.ntransfercmd(cmd)
+            conn, retrlen = self.ftp.ntransfercmd(cmd)
         self.busy = 1
+        ftpobj = addclosehook(conn.makefile('rb'), self.file_close)
+        self.refcount += 1
+        conn.close()
         # Pass back both a suitably decorated object and a retrieval length
-        return (addclosehook(conn[0].makefile('rb'),
+        return (ftpobj, retrlen)
-                             self.endtransfer), conn[1])
+
     def endtransfer(self):
         if not self.busy:
             return
@@ -918,6 +924,17 @@ class ftpwrapper:
             pass
 
     def close(self):
+        self.keepalive = False
+        if self.refcount <= 0:
+            self.real_close()
+
+    def file_close(self):
+        self.endtransfer()
+        self.refcount -= 1
+        if self.refcount <= 0 and not self.keepalive:
+            self.real_close()
+
+    def real_close(self):
         self.endtransfer()
         try:
             self.ftp.close()

Lib/urllib2.py

@@ -1399,7 +1399,8 @@ class FTPHandler(BaseHandler):
             raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
 
     def connect_ftp(self, user, passwd, host, port, dirs, timeout):
-        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout,
+                        persistent=False)
 ##        fw.ftp.set_debuglevel(1)
         return fw
 
@@ -1448,3 +1449,9 @@ class CacheFTPHandler(FTPHandler):
                     del self.timeout[k]
                     break
             self.soonest = min(self.timeout.values())
+
+    def clear_cache(self):
+        for conn in self.cache.values():
+            conn.close()
+        self.cache.clear()
+        self.timeout.clear()

Misc/NEWS

@@ -37,6 +37,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #10883: Fix socket leaks in urllib.request when using FTP.
+
 - Issue #12592: Make Python build on OpenBSD 5 (and future major releases).
 
 - Issue #12372: POSIX semaphores are broken on AIX: don't use them.