Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(30)

Side by Side Diff: Lib/urllib/request.py

Issue 10639: reindent.py converts newlines to platform default
Patch Set: Created 8 years, 8 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Lib/unittest/test/test_case.py ('k') | Lib/urllib/response.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 """An extensible library for opening URLs using a variety of protocols 1 """An extensible library for opening URLs using a variety of protocols
2 2
3 The simplest way to use this module is to call the urlopen function, 3 The simplest way to use this module is to call the urlopen function,
4 which accepts a string containing a URL or a Request object (described 4 which accepts a string containing a URL or a Request object (described
5 below). It opens the URL and returns the results as file-like 5 below). It opens the URL and returns the results as file-like
6 object; the returned object has some extra methods described below. 6 object; the returned object has some extra methods described below.
7 7
8 The OpenerDirector manages a collection of Handler objects that do 8 The OpenerDirector manages a collection of Handler objects that do
9 all the actual work. Each Handler implements a particular protocol or 9 all the actual work. Each Handler implements a particular protocol or
10 option. The OpenerDirector is a composite object that invokes the 10 option. The OpenerDirector is a composite object that invokes the
(...skipping 1353 matching lines...) Expand 10 before | Expand all | Expand 10 after
1364 headers += "Content-type: %s\n" % mtype 1364 headers += "Content-type: %s\n" % mtype
1365 if retrlen is not None and retrlen >= 0: 1365 if retrlen is not None and retrlen >= 0:
1366 headers += "Content-length: %d\n" % retrlen 1366 headers += "Content-length: %d\n" % retrlen
1367 headers = email.message_from_string(headers) 1367 headers = email.message_from_string(headers)
1368 return addinfourl(fp, headers, req.full_url) 1368 return addinfourl(fp, headers, req.full_url)
1369 except ftplib.all_errors as msg: 1369 except ftplib.all_errors as msg:
1370 exc = URLError('ftp error: %s' % msg) 1370 exc = URLError('ftp error: %s' % msg)
1371 raise exc.with_traceback(sys.exc_info()[2]) 1371 raise exc.with_traceback(sys.exc_info()[2])
1372 1372
1373 def connect_ftp(self, user, passwd, host, port, dirs, timeout): 1373 def connect_ftp(self, user, passwd, host, port, dirs, timeout):
1374 return ftpwrapper(user, passwd, host, port, dirs, timeout, 1374 fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
1375 persistent=False) 1375 return fw
1376 1376
1377 class CacheFTPHandler(FTPHandler): 1377 class CacheFTPHandler(FTPHandler):
1378 # XXX would be nice to have pluggable cache strategies 1378 # XXX would be nice to have pluggable cache strategies
1379 # XXX this stuff is definitely not thread safe 1379 # XXX this stuff is definitely not thread safe
1380 def __init__(self): 1380 def __init__(self):
1381 self.cache = {} 1381 self.cache = {}
1382 self.timeout = {} 1382 self.timeout = {}
1383 self.soonest = 0 1383 self.soonest = 0
1384 self.delay = 60 1384 self.delay = 60
1385 self.max_conns = 16 1385 self.max_conns = 16
(...skipping 27 matching lines...) Expand all
1413 self.soonest = min(list(self.timeout.values())) 1413 self.soonest = min(list(self.timeout.values()))
1414 1414
1415 # then check the size 1415 # then check the size
1416 if len(self.cache) == self.max_conns: 1416 if len(self.cache) == self.max_conns:
1417 for k, v in list(self.timeout.items()): 1417 for k, v in list(self.timeout.items()):
1418 if v == self.soonest: 1418 if v == self.soonest:
1419 del self.cache[k] 1419 del self.cache[k]
1420 del self.timeout[k] 1420 del self.timeout[k]
1421 break 1421 break
1422 self.soonest = min(list(self.timeout.values())) 1422 self.soonest = min(list(self.timeout.values()))
1423
1424 def clear_cache(self):
1425 for conn in self.cache.values():
1426 conn.close()
1427 self.cache.clear()
1428 self.timeout.clear()
1429
1430 1423
1431 # Code move from the old urllib module 1424 # Code move from the old urllib module
1432 1425
1433 MAXFTPCACHE = 10 # Trim the ftp cache beyond this size 1426 MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
1434 1427
1435 # Helper for non-unix systems 1428 # Helper for non-unix systems
1436 if os.name == 'nt': 1429 if os.name == 'nt':
1437 from nturl2path import url2pathname, pathname2url 1430 from nturl2path import url2pathname, pathname2url
1438 else: 1431 else:
1439 def url2pathname(pathname): 1432 def url2pathname(pathname):
(...skipping 704 matching lines...) Expand 10 before | Expand all | Expand 10 after
2144 if _noheaders is None: 2137 if _noheaders is None:
2145 _noheaders = email.message_from_string("") 2138 _noheaders = email.message_from_string("")
2146 return _noheaders 2139 return _noheaders
2147 2140
2148 2141
2149 # Utility classes 2142 # Utility classes
2150 2143
2151 class ftpwrapper: 2144 class ftpwrapper:
2152 """Class used by open_ftp() for cache of open FTP connections.""" 2145 """Class used by open_ftp() for cache of open FTP connections."""
2153 2146
2154 def __init__(self, user, passwd, host, port, dirs, timeout=None, 2147 def __init__(self, user, passwd, host, port, dirs, timeout=None):
2155 persistent=True):
2156 self.user = user 2148 self.user = user
2157 self.passwd = passwd 2149 self.passwd = passwd
2158 self.host = host 2150 self.host = host
2159 self.port = port 2151 self.port = port
2160 self.dirs = dirs 2152 self.dirs = dirs
2161 self.timeout = timeout 2153 self.timeout = timeout
2162 self.refcount = 0
2163 self.keepalive = persistent
2164 self.init() 2154 self.init()
2165 2155
2166 def init(self): 2156 def init(self):
2167 import ftplib 2157 import ftplib
2168 self.busy = 0 2158 self.busy = 0
2169 self.ftp = ftplib.FTP() 2159 self.ftp = ftplib.FTP()
2170 self.ftp.connect(self.host, self.port, self.timeout) 2160 self.ftp.connect(self.host, self.port, self.timeout)
2171 self.ftp.login(self.user, self.passwd) 2161 self.ftp.login(self.user, self.passwd)
2172 for dir in self.dirs: 2162 for dir in self.dirs:
2173 self.ftp.cwd(dir) 2163 self.ftp.cwd(dir)
(...skipping 30 matching lines...) Expand all
2204 except ftplib.error_perm as reason: 2194 except ftplib.error_perm as reason:
2205 raise URLError('ftp error', reason) from reason 2195 raise URLError('ftp error', reason) from reason
2206 finally: 2196 finally:
2207 self.ftp.cwd(pwd) 2197 self.ftp.cwd(pwd)
2208 cmd = 'LIST ' + file 2198 cmd = 'LIST ' + file
2209 else: 2199 else:
2210 cmd = 'LIST' 2200 cmd = 'LIST'
2211 conn, retrlen = self.ftp.ntransfercmd(cmd) 2201 conn, retrlen = self.ftp.ntransfercmd(cmd)
2212 self.busy = 1 2202 self.busy = 1
2213 2203
2214 ftpobj = addclosehook(conn.makefile('rb'), self.file_close) 2204 ftpobj = addclosehook(conn.makefile('rb'), self.endtransfer)
2215 self.refcount += 1
2216 conn.close() 2205 conn.close()
2217 # Pass back both a suitably decorated object and a retrieval length 2206 # Pass back both a suitably decorated object and a retrieval length
2218 return (ftpobj, retrlen) 2207 return (ftpobj, retrlen)
2219 2208
2220 def endtransfer(self): 2209 def endtransfer(self):
2221 if not self.busy: 2210 if not self.busy:
2222 return 2211 return
2223 self.busy = 0 2212 self.busy = 0
2224 try: 2213 try:
2225 self.ftp.voidresp() 2214 self.ftp.voidresp()
2226 except ftperrors(): 2215 except ftperrors():
2227 pass 2216 pass
2228 2217
2229 def close(self): 2218 def close(self):
2230 self.keepalive = False
2231 if self.refcount <= 0:
2232 self.real_close()
2233
2234 def file_close(self):
2235 self.endtransfer()
2236 self.refcount -= 1
2237 if self.refcount <= 0 and not self.keepalive:
2238 self.real_close()
2239
2240 def real_close(self):
2241 self.endtransfer() 2219 self.endtransfer()
2242 try: 2220 try:
2243 self.ftp.close() 2221 self.ftp.close()
2244 except ftperrors(): 2222 except ftperrors():
2245 pass 2223 pass
2246 2224
2247 # Proxy handling 2225 # Proxy handling
2248 def getproxies_environment(): 2226 def getproxies_environment():
2249 """Return a dictionary of scheme -> proxy server URL mappings. 2227 """Return a dictionary of scheme -> proxy server URL mappings.
2250 2228
(...skipping 238 matching lines...) Expand 10 before | Expand all | Expand 10 after
2489 """ 2467 """
2490 if getproxies_environment(): 2468 if getproxies_environment():
2491 return proxy_bypass_environment(host) 2469 return proxy_bypass_environment(host)
2492 else: 2470 else:
2493 return proxy_bypass_registry(host) 2471 return proxy_bypass_registry(host)
2494 2472
2495 else: 2473 else:
2496 # By default use environment variables 2474 # By default use environment variables
2497 getproxies = getproxies_environment 2475 getproxies = getproxies_environment
2498 proxy_bypass = proxy_bypass_environment 2476 proxy_bypass = proxy_bypass_environment
OLDNEW
« no previous file with comments | « Lib/unittest/test/test_case.py ('k') | Lib/urllib/response.py » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+