Merge: #10883: Fix socket leaks in urllib.request.

* ftpwrapper now uses reference counting to ensure that the underlying socket
  is closed when the ftpwrapper object is no longer in use
* ftplib.FTP.ntransfercmd() now closes the socket if an error occurs

Initial patch by Victor Stinner.

Lib/ftplib.py

@@ -343,33 +343,39 @@ class FTP:
             host, port = self.makepasv()
             conn = socket.create_connection((host, port), self.timeout,
                                             source_address=self.source_address)
-            if rest is not None:
+            try:
-                self.sendcmd("REST %s" % rest)
+                if rest is not None:
-            resp = self.sendcmd(cmd)
+                    self.sendcmd("REST %s" % rest)
-            # Some servers apparently send a 200 reply to
+                resp = self.sendcmd(cmd)
-            # a LIST or STOR command, before the 150 reply
+                # Some servers apparently send a 200 reply to
-            # (and way before the 226 reply). This seems to
+                # a LIST or STOR command, before the 150 reply
-            # be in violation of the protocol (which only allows
+                # (and way before the 226 reply). This seems to
-            # 1xx or error messages for LIST), so we just discard
+                # be in violation of the protocol (which only allows
-            # this response.
+                # 1xx or error messages for LIST), so we just discard
-            if resp[0] == '2':
+                # this response.
-                resp = self.getresp()
+                if resp[0] == '2':
-            if resp[0] != '1':
+                    resp = self.getresp()
-                raise error_reply(resp)
+                if resp[0] != '1':
+                    raise error_reply(resp)
+            except:
+                conn.close()
+                raise
         else:
             sock = self.makeport()
-            if rest is not None:
+            try:
-                self.sendcmd("REST %s" % rest)
+                if rest is not None:
-            resp = self.sendcmd(cmd)
+                    self.sendcmd("REST %s" % rest)
-            # See above.
+                resp = self.sendcmd(cmd)
-            if resp[0] == '2':
+                # See above.
-                resp = self.getresp()
+                if resp[0] == '2':
-            if resp[0] != '1':
+                    resp = self.getresp()
-                raise error_reply(resp)
+                if resp[0] != '1':
-            conn, sockaddr = sock.accept()
+                    raise error_reply(resp)
-            if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT:
+                conn, sockaddr = sock.accept()
-                conn.settimeout(self.timeout)
+                if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT:
-            sock.close()
+                    conn.settimeout(self.timeout)
+            finally:
+                sock.close()
         if resp[:3] == '150':
             # this is conditional in case we received a 125
             size = parse150(resp)

Lib/test/test_urllib2.py

@@ -623,6 +623,7 @@ class HandlerTests(unittest.TestCase):
             def retrfile(self, filename, filetype):
                 self.filename, self.filetype = filename, filetype
                 return io.StringIO(self.data), len(self.data)
+            def close(self): pass
 
         class NullFTPHandler(urllib.request.FTPHandler):
             def __init__(self, data): self.data = data

Lib/test/test_urllib2net.py

@@ -222,6 +222,7 @@ class OtherNetworkTests(unittest.TestCase):
         handlers = []
 
         cfh = urllib.request.CacheFTPHandler()
+        self.addCleanup(cfh.clear_cache)
         cfh.setTimeout(1)
         handlers.append(cfh)
 

Lib/urllib/request.py

@@ -1371,8 +1371,8 @@ class FTPHandler(BaseHandler):
             raise exc.with_traceback(sys.exc_info()[2])
 
     def connect_ftp(self, user, passwd, host, port, dirs, timeout):
-        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+        return ftpwrapper(user, passwd, host, port, dirs, timeout,
-        return fw
+                          persistent=False)
 
 class CacheFTPHandler(FTPHandler):
     # XXX would be nice to have pluggable cache strategies
@@ -1421,6 +1421,13 @@ class CacheFTPHandler(FTPHandler):
                     break
             self.soonest = min(list(self.timeout.values()))
 
+    def clear_cache(self):
+        for conn in self.cache.values():
+            conn.close()
+        self.cache.clear()
+        self.timeout.clear()
+
+
 # Code move from the old urllib module
 
 MAXFTPCACHE = 10        # Trim the ftp cache beyond this size
@@ -2144,13 +2151,16 @@ def noheaders():
 class ftpwrapper:
     """Class used by open_ftp() for cache of open FTP connections."""
 
-    def __init__(self, user, passwd, host, port, dirs, timeout=None):
+    def __init__(self, user, passwd, host, port, dirs, timeout=None,
+                 persistent=True):
         self.user = user
         self.passwd = passwd
         self.host = host
         self.port = port
         self.dirs = dirs
         self.timeout = timeout
+        self.refcount = 0
+        self.keepalive = persistent
         self.init()
 
     def init(self):
@@ -2201,7 +2211,8 @@ class ftpwrapper:
             conn, retrlen = self.ftp.ntransfercmd(cmd)
         self.busy = 1
 
-        ftpobj = addclosehook(conn.makefile('rb'), self.endtransfer)
+        ftpobj = addclosehook(conn.makefile('rb'), self.file_close)
+        self.refcount += 1
         conn.close()
         # Pass back both a suitably decorated object and a retrieval length
         return (ftpobj, retrlen)
@@ -2216,6 +2227,17 @@ class ftpwrapper:
             pass
 
     def close(self):
+        self.keepalive = False
+        if self.refcount <= 0:
+            self.real_close()
+
+    def file_close(self):
+        self.endtransfer()
+        self.refcount -= 1
+        if self.refcount <= 0 and not self.keepalive:
+            self.real_close()
+
+    def real_close(self):
         self.endtransfer()
         try:
             self.ftp.close()