urllib: socket is not closed explicitly

Lib/urllib/request.py

 """An extensible library for opening URLs using a variety of protocols
 
 The simplest way to use this module is to call the urlopen function,
 which accepts a string containing a URL or a Request object (described
 below).  It opens the URL and returns the results as file-like
 object; the returned object has some extra methods described below.
 
 The OpenerDirector manages a collection of Handler objects that do
 all the actual work.  Each Handler implements a particular protocol or
 option.  The OpenerDirector is a composite object that invokes the
@@ skipping 1353 matching lines @@
                 headers += "Content-type: %s\n" % mtype
             if retrlen is not None and retrlen >= 0:
                 headers += "Content-length: %d\n" % retrlen
             headers = email.message_from_string(headers)
             return addinfourl(fp, headers, req.full_url)
         except ftplib.all_errors as msg:
             exc = URLError('ftp error: %s' % msg)
             raise exc.with_traceback(sys.exc_info()[2])
 
     def connect_ftp(self, user, passwd, host, port, dirs, timeout):
-        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
-        return fw
+        return ftpwrapper(user, passwd, host, port, dirs, timeout,
+                          persistent=False)
 
 class CacheFTPHandler(FTPHandler):
     # XXX would be nice to have pluggable cache strategies
     # XXX this stuff is definitely not thread safe
     def __init__(self):
         self.cache = {}
         self.timeout = {}
         self.soonest = 0
         self.delay = 60
         self.max_conns = 16
@@ skipping 27 matching lines @@
         self.soonest = min(list(self.timeout.values()))
 
         # then check the size
         if len(self.cache) == self.max_conns:
             for k, v in list(self.timeout.items()):
                 if v == self.soonest:
                     del self.cache[k]
                     del self.timeout[k]
                     break
             self.soonest = min(list(self.timeout.values()))
+
+    def clear_cache(self):
+        for conn in self.cache.values():
+            conn.close()
+        self.cache.clear()
+        self.timeout.clear()
+
 
 # Code move from the old urllib module
 
 MAXFTPCACHE = 10        # Trim the ftp cache beyond this size
 
 # Helper for non-unix systems
 if os.name == 'nt':
     from nturl2path import url2pathname, pathname2url
 else:
     def url2pathname(pathname):
@@ skipping 704 matching lines @@
     if _noheaders is None:
         _noheaders = email.message_from_string("")
     return _noheaders
 
 
 # Utility classes
 
 class ftpwrapper:
     """Class used by open_ftp() for cache of open FTP connections."""
 
-    def __init__(self, user, passwd, host, port, dirs, timeout=None):
+    def __init__(self, user, passwd, host, port, dirs, timeout=None,
+                 persistent=True):
         self.user = user
         self.passwd = passwd
         self.host = host
         self.port = port
         self.dirs = dirs
         self.timeout = timeout
+        self.refcount = 0
+        self.keepalive = persistent
         self.init()
 
     def init(self):
         import ftplib
         self.busy = 0
         self.ftp = ftplib.FTP()
         self.ftp.connect(self.host, self.port, self.timeout)
         self.ftp.login(self.user, self.passwd)
         for dir in self.dirs:
             self.ftp.cwd(dir)
@@ skipping 30 matching lines @@
                     except ftplib.error_perm as reason:
                         raise URLError('ftp error', reason) from reason
                 finally:
                     self.ftp.cwd(pwd)
                 cmd = 'LIST ' + file
             else:
                 cmd = 'LIST'
             conn, retrlen = self.ftp.ntransfercmd(cmd)
         self.busy = 1
 
-        ftpobj = addclosehook(conn.makefile('rb'), self.endtransfer)
+        ftpobj = addclosehook(conn.makefile('rb'), self.file_close)
+        self.refcount += 1
         conn.close()
         # Pass back both a suitably decorated object and a retrieval length
         return (ftpobj, retrlen)
 
     def endtransfer(self):
         if not self.busy:
             return
         self.busy = 0
         try:
             self.ftp.voidresp()
         except ftperrors():
             pass
 
     def close(self):
+        self.keepalive = False
+        if self.refcount <= 0:
+            self.real_close()
+
+    def file_close(self):
+        self.endtransfer()
+        self.refcount -= 1
+        if self.refcount <= 0 and not self.keepalive:
+            self.real_close()
+
+    def real_close(self):
         self.endtransfer()
         try:
             self.ftp.close()
         except ftperrors():
             pass
 
 # Proxy handling
 def getproxies_environment():
     """Return a dictionary of scheme -> proxy server URL mappings.
 
@@ skipping 238 matching lines @@
         """
         if getproxies_environment():
             return proxy_bypass_environment(host)
         else:
             return proxy_bypass_registry(host)
 
 else:
     # By default use environment variables
     getproxies = getproxies_environment
     proxy_bypass = proxy_bypass_environment