mirror of https://github.com/python/cpython
Added an optional timeout parameter to function urllib2.urlopen,
with tests in test_urllib2net.py (must have network resource enabled to execute them). Also modified test_urllib2.py because testing mock classes must take it into acount. Docs are also updated.
This commit is contained in:
parent
9249312020
commit
10951d51e2
|
@ -14,7 +14,7 @@ authentication, redirections, cookies and more.
|
|||
|
||||
The \module{urllib2} module defines the following functions:
|
||||
|
||||
\begin{funcdesc}{urlopen}{url\optional{, data}}
|
||||
\begin{funcdesc}{urlopen}{url\optional{, data}\optional{, timeout}}
|
||||
Open the URL \var{url}, which can be either a string or a \class{Request}
|
||||
object.
|
||||
|
||||
|
@ -27,6 +27,11 @@ parameter is provided. \var{data} should be a buffer in the standard
|
|||
\function{urllib.urlencode()} function takes a mapping or sequence of
|
||||
2-tuples and returns a string in this format.
|
||||
|
||||
The optional \var{timeout} parameter specifies a timeout in seconds for the
|
||||
connection attempt (if not specified, or passed as None, the global default
|
||||
timeout setting will be used). This actually only work for HTTP, HTTPS, FTP
|
||||
and FTPS connections.
|
||||
|
||||
This function returns a file-like object with two additional methods:
|
||||
|
||||
\begin{itemize}
|
||||
|
@ -351,12 +356,17 @@ that HTTP errors are a special case).
|
|||
\end{itemize}
|
||||
\end{methoddesc}
|
||||
|
||||
\begin{methoddesc}[OpenerDirector]{open}{url\optional{, data}}
|
||||
\begin{methoddesc}[OpenerDirector]{open}{url\optional{, data}{\optional{, timeout}}}
|
||||
Open the given \var{url} (which can be a request object or a string),
|
||||
optionally passing the given \var{data}.
|
||||
Arguments, return values and exceptions raised are the same as those
|
||||
of \function{urlopen()} (which simply calls the \method{open()} method
|
||||
on the currently installed global \class{OpenerDirector}).
|
||||
on the currently installed global \class{OpenerDirector}). The optional
|
||||
\var{timeout} parameter specifies a timeout in seconds for the connection
|
||||
attempt (if not specified, or passed as None, the global default timeout
|
||||
setting will be used; this actually only work for HTTP, HTTPS, FTP
|
||||
and FTPS connections).
|
||||
|
||||
\end{methoddesc}
|
||||
|
||||
\begin{methoddesc}[OpenerDirector]{error}{proto\optional{,
|
||||
|
|
|
@ -545,7 +545,7 @@ class HandlerTests(unittest.TestCase):
|
|||
|
||||
class NullFTPHandler(urllib2.FTPHandler):
|
||||
def __init__(self, data): self.data = data
|
||||
def connect_ftp(self, user, passwd, host, port, dirs):
|
||||
def connect_ftp(self, user, passwd, host, port, dirs, timeout=None):
|
||||
self.user, self.passwd = user, passwd
|
||||
self.host, self.port = host, port
|
||||
self.dirs = dirs
|
||||
|
@ -568,7 +568,9 @@ class HandlerTests(unittest.TestCase):
|
|||
"localhost", ftplib.FTP_PORT, "A",
|
||||
[], "baz.gif", None), # XXX really this should guess image/gif
|
||||
]:
|
||||
r = h.ftp_open(Request(url))
|
||||
req = Request(url)
|
||||
req.timeout = None
|
||||
r = h.ftp_open(req)
|
||||
# ftp authentication not yet implemented by FTPHandler
|
||||
self.assert_(h.user == h.passwd == "")
|
||||
self.assertEqual(h.host, socket.gethostbyname(host))
|
||||
|
@ -683,8 +685,9 @@ class HandlerTests(unittest.TestCase):
|
|||
self.req_headers = []
|
||||
self.data = None
|
||||
self.raise_on_endheaders = False
|
||||
def __call__(self, host):
|
||||
def __call__(self, host, timeout=None):
|
||||
self.host = host
|
||||
self.timeout = timeout
|
||||
return self
|
||||
def set_debuglevel(self, level):
|
||||
self.level = level
|
||||
|
@ -707,6 +710,7 @@ class HandlerTests(unittest.TestCase):
|
|||
url = "http://example.com/"
|
||||
for method, data in [("GET", None), ("POST", "blah")]:
|
||||
req = Request(url, data, {"Foo": "bar"})
|
||||
req.timeout = None
|
||||
req.add_unredirected_header("Spam", "eggs")
|
||||
http = MockHTTPClass()
|
||||
r = h.do_open(http, req)
|
||||
|
|
|
@ -267,6 +267,49 @@ class OtherNetworkTests(unittest.TestCase):
|
|||
|
||||
return handlers
|
||||
|
||||
class TimeoutTest(unittest.TestCase):
|
||||
def test_http_basic(self):
|
||||
u = urllib2.urlopen("http://www.python.org")
|
||||
self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None)
|
||||
|
||||
def test_http_NoneWithdefault(self):
|
||||
prev = socket.getdefaulttimeout()
|
||||
socket.setdefaulttimeout(60)
|
||||
try:
|
||||
u = urllib2.urlopen("http://www.python.org", timeout=None)
|
||||
self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60)
|
||||
finally:
|
||||
socket.setdefaulttimeout(prev)
|
||||
|
||||
def test_http_Value(self):
|
||||
u = urllib2.urlopen("http://www.python.org", timeout=120)
|
||||
self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120)
|
||||
|
||||
def test_http_NoneNodefault(self):
|
||||
u = urllib2.urlopen("http://www.python.org", timeout=None)
|
||||
self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None)
|
||||
|
||||
def test_ftp_basic(self):
|
||||
u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/")
|
||||
self.assertTrue(u.fp.fp._sock.gettimeout() is None)
|
||||
|
||||
def test_ftp_NoneWithdefault(self):
|
||||
prev = socket.getdefaulttimeout()
|
||||
socket.setdefaulttimeout(60)
|
||||
try:
|
||||
u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None)
|
||||
self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
|
||||
finally:
|
||||
socket.setdefaulttimeout(prev)
|
||||
|
||||
def test_ftp_NoneNodefault(self):
|
||||
u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None)
|
||||
self.assertTrue(u.fp.fp._sock.gettimeout() is None)
|
||||
|
||||
def test_ftp_Value(self):
|
||||
u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=60)
|
||||
self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
|
||||
|
||||
|
||||
def test_main():
|
||||
test_support.requires("network")
|
||||
|
@ -275,6 +318,7 @@ def test_main():
|
|||
AuthTests,
|
||||
OtherNetworkTests,
|
||||
CloseSocketTest,
|
||||
TimeoutTest,
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
@ -117,11 +117,11 @@ from urllib import localhost, url2pathname, getproxies
|
|||
__version__ = sys.version[:3]
|
||||
|
||||
_opener = None
|
||||
def urlopen(url, data=None):
|
||||
def urlopen(url, data=None, timeout=None):
|
||||
global _opener
|
||||
if _opener is None:
|
||||
_opener = build_opener()
|
||||
return _opener.open(url, data)
|
||||
return _opener.open(url, data, timeout)
|
||||
|
||||
def install_opener(opener):
|
||||
global _opener
|
||||
|
@ -355,7 +355,7 @@ class OpenerDirector:
|
|||
if result is not None:
|
||||
return result
|
||||
|
||||
def open(self, fullurl, data=None):
|
||||
def open(self, fullurl, data=None, timeout=None):
|
||||
# accept a URL or a Request object
|
||||
if isinstance(fullurl, basestring):
|
||||
req = Request(fullurl, data)
|
||||
|
@ -364,6 +364,7 @@ class OpenerDirector:
|
|||
if data is not None:
|
||||
req.add_data(data)
|
||||
|
||||
req.timeout = timeout
|
||||
protocol = req.get_type()
|
||||
|
||||
# pre-process request
|
||||
|
@ -1057,7 +1058,7 @@ class AbstractHTTPHandler(BaseHandler):
|
|||
if not host:
|
||||
raise URLError('no host given')
|
||||
|
||||
h = http_class(host) # will parse host:port
|
||||
h = http_class(host, timeout=req.timeout) # will parse host:port
|
||||
h.set_debuglevel(self._debuglevel)
|
||||
|
||||
headers = dict(req.headers)
|
||||
|
@ -1269,7 +1270,7 @@ class FTPHandler(BaseHandler):
|
|||
if dirs and not dirs[0]:
|
||||
dirs = dirs[1:]
|
||||
try:
|
||||
fw = self.connect_ftp(user, passwd, host, port, dirs)
|
||||
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
|
||||
type = file and 'I' or 'D'
|
||||
for attr in attrs:
|
||||
attr, value = splitvalue(attr)
|
||||
|
@ -1289,8 +1290,8 @@ class FTPHandler(BaseHandler):
|
|||
except ftplib.all_errors, msg:
|
||||
raise IOError, ('ftp error', msg), sys.exc_info()[2]
|
||||
|
||||
def connect_ftp(self, user, passwd, host, port, dirs):
|
||||
fw = ftpwrapper(user, passwd, host, port, dirs)
|
||||
def connect_ftp(self, user, passwd, host, port, dirs, timeout):
|
||||
fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
|
||||
## fw.ftp.set_debuglevel(1)
|
||||
return fw
|
||||
|
||||
|
@ -1310,12 +1311,12 @@ class CacheFTPHandler(FTPHandler):
|
|||
def setMaxConns(self, m):
|
||||
self.max_conns = m
|
||||
|
||||
def connect_ftp(self, user, passwd, host, port, dirs):
|
||||
key = user, host, port, '/'.join(dirs)
|
||||
def connect_ftp(self, user, passwd, host, port, dirs, timeout):
|
||||
key = user, host, port, '/'.join(dirs), timeout
|
||||
if key in self.cache:
|
||||
self.timeout[key] = time.time() + self.delay
|
||||
else:
|
||||
self.cache[key] = ftpwrapper(user, passwd, host, port, dirs)
|
||||
self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
|
||||
self.timeout[key] = time.time() + self.delay
|
||||
self.check_cache()
|
||||
return self.cache[key]
|
||||
|
|
|
@ -222,6 +222,9 @@ Core and builtins
|
|||
Library
|
||||
-------
|
||||
|
||||
- The urlopen function of urllib2 now has an optional timeout parameter (note
|
||||
that it actually works with HTTP, HTTPS, FTP and FTPS connections).
|
||||
|
||||
- In ftplib, the FTP.ntransfercmd method, when in passive mode, now uses
|
||||
the socket.create_connection function, using the timeout specified at
|
||||
connection time.
|
||||
|
|
Loading…
Reference in New Issue