From 60a4a90c8dd2972eb4bb977e70835be9593cbbac Mon Sep 17 00:00:00 2001 From: "guido@google.com" Date: Thu, 24 Mar 2011 08:07:45 -0700 Subject: [PATCH 1/5] Issue 22663: fix redirect vulnerability in urllib/urllib2. --- Lib/urllib.py | 13 +++++++++++-- Lib/urllib2.py | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/urllib.py b/Lib/urllib.py index 963187cfb27..09ce8c57e84 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -638,10 +638,19 @@ class FancyURLopener(URLopener): newurl = headers['uri'] else: return - void = fp.read() - fp.close() + # In case the server sent a relative URL, join with original: newurl = basejoin(self.type + ":" + url, newurl) + + # For security reasons we do not allow redirects to protocols + # other than HTTP or HTTPS. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://')): + return + + void = fp.read() + fp.close() return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 50d7aaf204d..db7ce81845a 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -555,6 +555,13 @@ class HTTPRedirectHandler(BaseHandler): return newurl = urlparse.urljoin(req.get_full_url(), newurl) + # For security reasons we do not allow redirects to protocols + # other than HTTP or HTTPS. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://')): + return + # XXX Probably want to forget about the state of the current # request, although that might interact poorly with other # handlers that also use handler-specific request attributes From 2bc23b8448394e96d5562fcc7b69aa54bb2c1a38 Mon Sep 17 00:00:00 2001 From: "guido@google.com" Date: Thu, 24 Mar 2011 10:44:17 -0700 Subject: [PATCH 2/5] Add FTP to the allowed url schemes. Add Misc/NEWS. --- Lib/urllib.py | 5 +++-- Lib/urllib2.py | 5 +++-- Misc/NEWS | 3 +++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/urllib.py b/Lib/urllib.py index 09ce8c57e84..b835f52f239 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -643,10 +643,11 @@ class FancyURLopener(URLopener): newurl = basejoin(self.type + ":" + url, newurl) # For security reasons we do not allow redirects to protocols - # other than HTTP or HTTPS. + # other than HTTP, HTTPS or FTP. newurl_lower = newurl.lower() if not (newurl_lower.startswith('http://') or - newurl_lower.startswith('https://')): + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): return void = fp.read() diff --git a/Lib/urllib2.py b/Lib/urllib2.py index db7ce81845a..0bb69a01303 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -556,10 +556,11 @@ class HTTPRedirectHandler(BaseHandler): newurl = urlparse.urljoin(req.get_full_url(), newurl) # For security reasons we do not allow redirects to protocols - # other than HTTP or HTTPS. + # other than HTTP, HTTPS or FTP. newurl_lower = newurl.lower() if not (newurl_lower.startswith('http://') or - newurl_lower.startswith('https://')): + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): return # XXX Probably want to forget about the state of the current diff --git a/Misc/NEWS b/Misc/NEWS index 3aea1f331bf..76aea178271 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ What's New in Python 2.5.6c1? Library ------- +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP. This fixes a security hole. + - Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing overflow checks in the audioop module (CVE-2010-1634). From f1509306d266a091c6c12ec3b78b8eaaa9d0aff9 Mon Sep 17 00:00:00 2001 From: "guido@google.com" Date: Mon, 28 Mar 2011 13:47:01 -0700 Subject: [PATCH 3/5] Add tests for the urllib[2] vulnerability. Change to raise exceptions. --- Lib/test/test_urllib.py | 14 ++++++++++++++ Lib/test/test_urllib2.py | 21 +++++++++++++++++++++ Lib/urllib.py | 10 ++++++---- Lib/urllib2.py | 5 ++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 294ed5e06a2..72db1ef29c1 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -122,6 +122,20 @@ class urlopen_HttpTests(unittest.TestCase): finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp("""HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file:README +Connection: close +Content-Type: text/html; charset=iso-8859-1 +""") + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): """urlopen() raises IOError if the underlying socket does not send any data. (#1680230) """ diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 96a7db9e340..8e9f1b2fef0 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -857,6 +857,27 @@ class HandlerTests(unittest.TestCase): self.assertEqual(count, urllib2.HTTPRedirectHandler.max_redirections) + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http', 'https', 'ftp'] + invalid_schemes = ['file', 'imap', 'ldap'] + schemeless_url = "example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib2.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from cookielib import CookieJar diff --git a/Lib/urllib.py b/Lib/urllib.py index b835f52f239..97597f4835b 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -638,7 +638,8 @@ class FancyURLopener(URLopener): newurl = headers['uri'] else: return - + void = fp.read() + fp.close() # In case the server sent a relative URL, join with original: newurl = basejoin(self.type + ":" + url, newurl) @@ -648,10 +649,11 @@ class FancyURLopener(URLopener): if not (newurl_lower.startswith('http://') or newurl_lower.startswith('https://') or newurl_lower.startswith('ftp://')): - return + raise IOError('redirect error', errcode, + errmsg + " - Redirection to url '%s' is not allowed" % + newurl, + headers) - void = fp.read() - fp.close() return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 0bb69a01303..a537d3630e7 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -561,7 +561,10 @@ class HTTPRedirectHandler(BaseHandler): if not (newurl_lower.startswith('http://') or newurl_lower.startswith('https://') or newurl_lower.startswith('ftp://')): - return + raise HTTPError(newurl, code, + msg + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) # XXX Probably want to forget about the state of the current # request, although that might interact poorly with other From db3080e68f6a504bda6f04f448cc65400ba53c97 Mon Sep 17 00:00:00 2001 From: "guido@google.com" Date: Mon, 28 Mar 2011 13:53:40 -0700 Subject: [PATCH 4/5] Add CVE number to urllib/urllib2 news item. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 76aea178271..551ecb73abf 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,7 +13,7 @@ Library ------- - Issue #11662: Make urllib and urllib2 ignore redirections if the - scheme is not HTTP, HTTPS or FTP. This fixes a security hole. + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). - Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing overflow checks in the audioop module (CVE-2010-1634). From 92ecb8737b9c708268c6451a01835192c181b721 Mon Sep 17 00:00:00 2001 From: "guido@google.com" Date: Tue, 29 Mar 2011 09:53:33 -0700 Subject: [PATCH 5/5] Adding .hgignore (copied from default branch). --- .hgignore | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .hgignore diff --git a/.hgignore b/.hgignore new file mode 100644 index 00000000000..70aca549360 --- /dev/null +++ b/.hgignore @@ -0,0 +1,66 @@ +.gdb_history +.purify +.svn/ +Makefile$ +Makefile.pre$ +TAGS$ +autom4te.cache$ +build/ +buildno$ +config.cache +config.log +config.status +config.status.lineno +db_home +platform$ +pyconfig.h$ +python$ +python.exe$ +reflog.txt$ +tags$ +Lib/plat-mac/errors.rsrc.df.rsrc +Doc/tools/sphinx/ +Doc/tools/docutils/ +Doc/tools/jinja/ +Doc/tools/jinja2/ +Doc/tools/pygments/ +Misc/python.pc +Modules/Setup$ +Modules/Setup.config +Modules/Setup.local +Modules/config.c +Modules/ld_so_aix$ +Parser/pgen$ +Parser/pgen.stamp$ +^core +^python-gdb.py +^python.exe-gdb.py +^pybuilddir.txt + +syntax: glob +libpython*.a +*.swp +*.o +*.pyc +*.pyo +*.pyd +*.cover +*.orig +*.rej +*~ +Lib/lib2to3/*.pickle +Lib/test/data/* +Misc/*.wpu +PC/python_nt*.h +PC/pythonnt_rc*.h +PC/*.obj +PCbuild/*.exe +PCbuild/*.dll +PCbuild/*.pdb +PCbuild/*.lib +PCbuild/*.exp +PCbuild/*.o +PCbuild/*.ncb +PCbuild/*.bsc +PCbuild/Win32-temp-* +__pycache__