From 8116364b7362b1d5a25240342aa7296e307bd6da Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 20 Dec 2009 06:32:46 +0000 Subject: [PATCH] Merged revisions 76908 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r76908 | senthil.kumaran | 2009-12-20 11:35:13 +0530 (Sun, 20 Dec 2009) | 4 lines Fix for issue 7291 - urllib2 cannot handle https with proxy requiring auth Refactored HTTPHandler tests and added testcase for proxy authorization. ........ --- Lib/httplib.py | 16 +++++- Lib/test/test_urllib2.py | 105 +++++++++++++++++++++++++++------------ Lib/urllib2.py | 9 +++- Misc/NEWS | 3 ++ 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/Lib/httplib.py b/Lib/httplib.py index 3623bc3853e..8629cc336bb 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -652,15 +652,24 @@ class HTTPConnection: self._method = None self._tunnel_host = None self._tunnel_port = None + self._tunnel_headers = {} self._set_hostport(host, port) if strict is not None: self.strict = strict def _set_tunnel(self, host, port=None): - """ Sets up the host and the port for the HTTP CONNECT Tunnelling.""" + """ Sets up the host and the port for the HTTP CONNECT Tunnelling. + + The headers argument should be a mapping of extra HTTP headers + to send with the CONNECT request. + """ self._tunnel_host = host self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() def _set_hostport(self, host, port): if port is None: @@ -684,7 +693,10 @@ class HTTPConnection: def _tunnel(self): self._set_hostport(self._tunnel_host, self._tunnel_port) - self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port)) + self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)) + for header, value in self._tunnel_headers.iteritems(): + self.send("%s: %s\r\n" % (header, value)) + self.send("\r\n") response = self.response_class(self.sock, strict = self.strict, method = self._method) (version, code, message) = response._read_status() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 67782932405..e7188e4e602 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -261,6 +261,50 @@ class FakeMethod: def __call__(self, *args): return self.handle(self.meth_name, self.action, *args) +class MockHTTPResponse: + def __init__(self, fp, msg, status, reason): + self.fp = fp + self.msg = msg + self.status = status + self.reason = reason + def read(self): + return '' + +class MockHTTPClass: + def __init__(self): + self.req_headers = [] + self.data = None + self.raise_on_endheaders = False + self._tunnel_headers = {} + + def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.timeout = timeout + return self + + def set_debuglevel(self, level): + self.level = level + + def _set_tunnel(self, host, port=None, headers=None): + self._tunnel_host = host + self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() + def request(self, method, url, body=None, headers={}): + self.method = method + self.selector = url + self.req_headers += headers.items() + self.req_headers.sort() + if body: + self.data = body + if self.raise_on_endheaders: + import socket + raise socket.error() + def getresponse(self): + return MockHTTPResponse(MockFile(), {}, 200, "OK") + class MockHandler: # useful for testing handler machinery # see add_ordered_mock_handlers() docstring @@ -368,6 +412,13 @@ class MockHTTPHandler(urllib2.BaseHandler): msg = mimetools.Message(StringIO("\r\n\r\n")) return MockResponse(200, "OK", msg, "", req.get_full_url()) +class MockHTTPSHandler(urllib2.AbstractHTTPHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon + httpconn = MockHTTPClass() + def https_open(self, req): + return self.do_open(self.httpconn, req) + class MockPasswordManager: def add_password(self, realm, uri, user, password): self.realm = realm @@ -680,37 +731,6 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.type, "ftp") def test_http(self): - class MockHTTPResponse: - def __init__(self, fp, msg, status, reason): - self.fp = fp - self.msg = msg - self.status = status - self.reason = reason - def read(self): - return '' - class MockHTTPClass: - def __init__(self): - self.req_headers = [] - self.data = None - self.raise_on_endheaders = False - def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - self.host = host - self.timeout = timeout - return self - def set_debuglevel(self, level): - self.level = level - def request(self, method, url, body=None, headers={}): - self.method = method - self.selector = url - self.req_headers += headers.items() - self.req_headers.sort() - if body: - self.data = body - if self.raise_on_endheaders: - import socket - raise socket.error() - def getresponse(self): - return MockHTTPResponse(MockFile(), {}, 200, "OK") h = urllib2.AbstractHTTPHandler() o = h.parent = MockOpener() @@ -974,6 +994,29 @@ class HandlerTests(unittest.TestCase): self.assertEqual([(handlers[0], "https_open")], [tup[0:2] for tup in o.calls]) + def test_proxy_https_proxy_authorization(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + https_handler = MockHTTPSHandler() + o.add_handler(https_handler) + req = Request("https://www.example.com/") + req.add_header("Proxy-Authorization","FooBar") + req.add_header("User-Agent","Grail") + self.assertEqual(req.get_host(), "www.example.com") + self.assertTrue(req._tunnel_host is None) + r = o.open(req) + # Verify Proxy-Authorization gets tunneled to request. + # httpsconn req_headers do not have the Proxy-Authorization header but + # the req will have. + self.assertFalse(("Proxy-Authorization","FooBar") in + https_handler.httpconn.req_headers) + self.assertTrue(("User-Agent","Grail") in + https_handler.httpconn.req_headers) + self.assertFalse(req._tunnel_host is None) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual(req.get_header("Proxy-authorization"),"FooBar") + def test_basic_auth(self, quote_char='"'): opener = OpenerDirector() password_manager = MockPasswordManager() diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 03bd5f8af2c..a976533918a 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -1120,7 +1120,14 @@ class AbstractHTTPHandler(BaseHandler): (name.title(), val) for name, val in headers.items()) if req._tunnel_host: - h._set_tunnel(req._tunnel_host) + tunnel_headers = {} + proxy_auth_hdr = "Proxy-Authorization" + if proxy_auth_hdr in headers: + tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] + # Proxy-Authorization should not be sent to origin + # server. + del headers[proxy_auth_hdr] + h._set_tunnel(req._tunnel_host, headers=tunnel_headers) try: h.request(req.get_method(), req.get_selector(), req.data, headers) diff --git a/Misc/NEWS b/Misc/NEWS index c16e6977f10..817429270f5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -35,6 +35,9 @@ Core and Builtins Library ------- +- Issue #7231: urllib2 cannot handle https with proxy requiring auth. Patch by + Tatsuhiro Tsujikawa. + - Issue #7348: StringIO.StringIO.readline(-1) now acts as if it got no argument like other file objects.