diff --git a/Doc/lib/liburllib2.tex b/Doc/lib/liburllib2.tex index 0df73859ba8..5547b1510fe 100644 --- a/Doc/lib/liburllib2.tex +++ b/Doc/lib/liburllib2.tex @@ -14,7 +14,7 @@ authentication, redirections, cookies and more. The \module{urllib2} module defines the following functions: -\begin{funcdesc}{urlopen}{url\optional{, data}} +\begin{funcdesc}{urlopen}{url\optional{, data}\optional{, timeout}} Open the URL \var{url}, which can be either a string or a \class{Request} object. @@ -27,6 +27,11 @@ parameter is provided. \var{data} should be a buffer in the standard \function{urllib.urlencode()} function takes a mapping or sequence of 2-tuples and returns a string in this format. +The optional \var{timeout} parameter specifies a timeout in seconds for the +connection attempt (if not specified, or passed as None, the global default +timeout setting will be used). This actually only work for HTTP, HTTPS, FTP +and FTPS connections. + This function returns a file-like object with two additional methods: \begin{itemize} @@ -351,12 +356,17 @@ that HTTP errors are a special case). \end{itemize} \end{methoddesc} -\begin{methoddesc}[OpenerDirector]{open}{url\optional{, data}} +\begin{methoddesc}[OpenerDirector]{open}{url\optional{, data}{\optional{, timeout}}} Open the given \var{url} (which can be a request object or a string), optionally passing the given \var{data}. Arguments, return values and exceptions raised are the same as those of \function{urlopen()} (which simply calls the \method{open()} method -on the currently installed global \class{OpenerDirector}). +on the currently installed global \class{OpenerDirector}). The optional +\var{timeout} parameter specifies a timeout in seconds for the connection +attempt (if not specified, or passed as None, the global default timeout +setting will be used; this actually only work for HTTP, HTTPS, FTP +and FTPS connections). + \end{methoddesc} \begin{methoddesc}[OpenerDirector]{error}{proto\optional{, diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 1a2986a5ffc..c9dcf5ef7be 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -545,7 +545,7 @@ class HandlerTests(unittest.TestCase): class NullFTPHandler(urllib2.FTPHandler): def __init__(self, data): self.data = data - def connect_ftp(self, user, passwd, host, port, dirs): + def connect_ftp(self, user, passwd, host, port, dirs, timeout=None): self.user, self.passwd = user, passwd self.host, self.port = host, port self.dirs = dirs @@ -568,7 +568,9 @@ class HandlerTests(unittest.TestCase): "localhost", ftplib.FTP_PORT, "A", [], "baz.gif", None), # XXX really this should guess image/gif ]: - r = h.ftp_open(Request(url)) + req = Request(url) + req.timeout = None + r = h.ftp_open(req) # ftp authentication not yet implemented by FTPHandler self.assert_(h.user == h.passwd == "") self.assertEqual(h.host, socket.gethostbyname(host)) @@ -683,8 +685,9 @@ class HandlerTests(unittest.TestCase): self.req_headers = [] self.data = None self.raise_on_endheaders = False - def __call__(self, host): + def __call__(self, host, timeout=None): self.host = host + self.timeout = timeout return self def set_debuglevel(self, level): self.level = level @@ -707,6 +710,7 @@ class HandlerTests(unittest.TestCase): url = "http://example.com/" for method, data in [("GET", None), ("POST", "blah")]: req = Request(url, data, {"Foo": "bar"}) + req.timeout = None req.add_unredirected_header("Spam", "eggs") http = MockHTTPClass() r = h.do_open(http, req) diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index e76301e3e76..c363140fc2d 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -267,6 +267,49 @@ class OtherNetworkTests(unittest.TestCase): return handlers +class TimeoutTest(unittest.TestCase): + def test_http_basic(self): + u = urllib2.urlopen("http://www.python.org") + self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None) + + def test_http_NoneWithdefault(self): + prev = socket.getdefaulttimeout() + socket.setdefaulttimeout(60) + try: + u = urllib2.urlopen("http://www.python.org", timeout=None) + self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60) + finally: + socket.setdefaulttimeout(prev) + + def test_http_Value(self): + u = urllib2.urlopen("http://www.python.org", timeout=120) + self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) + + def test_http_NoneNodefault(self): + u = urllib2.urlopen("http://www.python.org", timeout=None) + self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None) + + def test_ftp_basic(self): + u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/") + self.assertTrue(u.fp.fp._sock.gettimeout() is None) + + def test_ftp_NoneWithdefault(self): + prev = socket.getdefaulttimeout() + socket.setdefaulttimeout(60) + try: + u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None) + self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + finally: + socket.setdefaulttimeout(prev) + + def test_ftp_NoneNodefault(self): + u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None) + self.assertTrue(u.fp.fp._sock.gettimeout() is None) + + def test_ftp_Value(self): + u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=60) + self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + def test_main(): test_support.requires("network") @@ -275,6 +318,7 @@ def test_main(): AuthTests, OtherNetworkTests, CloseSocketTest, + TimeoutTest, ) if __name__ == "__main__": diff --git a/Lib/urllib2.py b/Lib/urllib2.py index fe32b2d6520..a1badb74eb4 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -117,11 +117,11 @@ from urllib import localhost, url2pathname, getproxies __version__ = sys.version[:3] _opener = None -def urlopen(url, data=None): +def urlopen(url, data=None, timeout=None): global _opener if _opener is None: _opener = build_opener() - return _opener.open(url, data) + return _opener.open(url, data, timeout) def install_opener(opener): global _opener @@ -355,7 +355,7 @@ class OpenerDirector: if result is not None: return result - def open(self, fullurl, data=None): + def open(self, fullurl, data=None, timeout=None): # accept a URL or a Request object if isinstance(fullurl, basestring): req = Request(fullurl, data) @@ -364,6 +364,7 @@ class OpenerDirector: if data is not None: req.add_data(data) + req.timeout = timeout protocol = req.get_type() # pre-process request @@ -1057,7 +1058,7 @@ class AbstractHTTPHandler(BaseHandler): if not host: raise URLError('no host given') - h = http_class(host) # will parse host:port + h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) headers = dict(req.headers) @@ -1269,7 +1270,7 @@ class FTPHandler(BaseHandler): if dirs and not dirs[0]: dirs = dirs[1:] try: - fw = self.connect_ftp(user, passwd, host, port, dirs) + fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' for attr in attrs: attr, value = splitvalue(attr) @@ -1289,8 +1290,8 @@ class FTPHandler(BaseHandler): except ftplib.all_errors, msg: raise IOError, ('ftp error', msg), sys.exc_info()[2] - def connect_ftp(self, user, passwd, host, port, dirs): - fw = ftpwrapper(user, passwd, host, port, dirs) + def connect_ftp(self, user, passwd, host, port, dirs, timeout): + fw = ftpwrapper(user, passwd, host, port, dirs, timeout) ## fw.ftp.set_debuglevel(1) return fw @@ -1310,12 +1311,12 @@ class CacheFTPHandler(FTPHandler): def setMaxConns(self, m): self.max_conns = m - def connect_ftp(self, user, passwd, host, port, dirs): - key = user, host, port, '/'.join(dirs) + def connect_ftp(self, user, passwd, host, port, dirs, timeout): + key = user, host, port, '/'.join(dirs), timeout if key in self.cache: self.timeout[key] = time.time() + self.delay else: - self.cache[key] = ftpwrapper(user, passwd, host, port, dirs) + self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout) self.timeout[key] = time.time() + self.delay self.check_cache() return self.cache[key] diff --git a/Misc/NEWS b/Misc/NEWS index a6277655071..b7867458cab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -222,6 +222,9 @@ Core and builtins Library ------- +- The urlopen function of urllib2 now has an optional timeout parameter (note + that it actually works with HTTP, HTTPS, FTP and FTPS connections). + - In ftplib, the FTP.ntransfercmd method, when in passive mode, now uses the socket.create_connection function, using the timeout specified at connection time.