Fixes for two separate HTTP/1.1 bugs: 100 responses and HTTPS connections.
The HTTPResponse class now handles 100 continue responses, instead of choking on them. It detects them internally in the _begin() method and ignores them. Based on a patch by Bob Kline. This closes SF bugs 498149 and 551273. The FakeSocket class (for SSL) is now usable with HTTP/1.1 connections. The old version of the code could not work with persistent connections, because the makefile() implementation read until EOF before returning. If the connection is persistent, the server sends a response and leaves the connection open. A client that reads until EOF will block until the server gives up on the connection -- more than a minute in my test case. The problem was fixed by implementing a reasonable makefile(). It reads data only when it is needed by the layers above it. It's implementation uses an internal buffer with a default size of 8192. Also, rename begin() method of HTTPResponse to _begin() because it should only be called by the HTTPConnection.
This commit is contained in:
Jeremy Hylton
parent
71b63ff342
commit
be4fcf1875
134
Lib/httplib.py
134
Lib/httplib.py
|
|
@ -111,11 +111,7 @@ class HTTPResponse:
|
|||
self.length = _UNKNOWN # number of bytes left in response
|
||||
self.will_close = _UNKNOWN # conn will close at end of response
|
||||
|
||||
def begin(self):
|
||||
if self.msg is not None:
|
||||
# we've already started reading the response
|
||||
return
|
||||
|
||||
def _read_status(self):
|
||||
line = self.fp.readline()
|
||||
if self.debuglevel > 0:
|
||||
print "reply:", repr(line)
|
||||
|
|
@ -135,13 +131,33 @@ class HTTPResponse:
|
|||
|
||||
# The status code is a three-digit number
|
||||
try:
|
||||
self.status = status = int(status)
|
||||
status = int(status)
|
||||
if status < 100 or status > 999:
|
||||
raise BadStatusLine(line)
|
||||
except ValueError:
|
||||
raise BadStatusLine(line)
|
||||
self.reason = reason.strip()
|
||||
return version, status, reason
|
||||
|
||||
def _begin(self):
|
||||
if self.msg is not None:
|
||||
# we've already started reading the response
|
||||
return
|
||||
|
||||
# read until we get a non-100 response
|
||||
while 1:
|
||||
version, status, reason = self._read_status()
|
||||
if status != 100:
|
||||
break
|
||||
# skip the header from the 100 response
|
||||
while 1:
|
||||
skip = self.fp.readline().strip()
|
||||
if not skip:
|
||||
break
|
||||
if self.debuglevel > 0:
|
||||
print "header:", skip
|
||||
|
||||
self.status = status
|
||||
self.reason = reason.strip()
|
||||
if version == 'HTTP/1.0':
|
||||
self.version = 10
|
||||
elif version.startswith('HTTP/1.'):
|
||||
|
|
@ -152,6 +168,7 @@ class HTTPResponse:
|
|||
raise UnknownProtocol(version)
|
||||
|
||||
if self.version == 9:
|
||||
self.chunked = 0
|
||||
self.msg = mimetools.Message(StringIO())
|
||||
return
|
||||
|
||||
|
|
@ -233,6 +250,7 @@ class HTTPResponse:
|
|||
return ''
|
||||
|
||||
if self.chunked:
|
||||
assert self.chunked != _UNKNOWN
|
||||
chunk_left = self.chunk_left
|
||||
value = ''
|
||||
while 1:
|
||||
|
|
@ -363,7 +381,8 @@ class HTTPConnection:
|
|||
def connect(self):
|
||||
"""Connect to the host and port specified in __init__."""
|
||||
msg = "getaddrinfo returns an empty list"
|
||||
for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
|
||||
for res in socket.getaddrinfo(self.host, self.port, 0,
|
||||
socket.SOCK_STREAM):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
try:
|
||||
self.sock = socket.socket(af, socktype, proto)
|
||||
|
|
@ -595,7 +614,8 @@ class HTTPConnection:
|
|||
else:
|
||||
response = self.response_class(self.sock)
|
||||
|
||||
response.begin()
|
||||
response._begin()
|
||||
assert response.will_close != _UNKNOWN
|
||||
self.__state = _CS_IDLE
|
||||
|
||||
if response.will_close:
|
||||
|
|
@ -607,28 +627,23 @@ class HTTPConnection:
|
|||
|
||||
return response
|
||||
|
||||
class SSLFile:
|
||||
"""File-like object wrapping an SSL socket."""
|
||||
|
||||
class FakeSocket:
|
||||
def __init__(self, sock, ssl):
|
||||
self.__sock = sock
|
||||
self.__ssl = ssl
|
||||
BUFSIZE = 8192
|
||||
|
||||
def __init__(self, sock, ssl, bufsize=None):
|
||||
self._sock = sock
|
||||
self._ssl = ssl
|
||||
self._buf = ''
|
||||
self._bufsize = bufsize or self.__class__.BUFSIZE
|
||||
|
||||
def makefile(self, mode, bufsize=None):
|
||||
"""Return a readable file-like object with data from socket.
|
||||
|
||||
This method offers only partial support for the makefile
|
||||
interface of a real socket. It only supports modes 'r' and
|
||||
'rb' and the bufsize argument is ignored.
|
||||
|
||||
The returned object contains *all* of the file data
|
||||
"""
|
||||
if mode != 'r' and mode != 'rb':
|
||||
raise UnimplementedFileMode()
|
||||
|
||||
msgbuf = []
|
||||
def _read(self):
|
||||
buf = ''
|
||||
# put in a loop so that we retry on transient errors
|
||||
while 1:
|
||||
try:
|
||||
buf = self.__ssl.read()
|
||||
buf = self._ssl.read(self._bufsize)
|
||||
except socket.sslerror, err:
|
||||
if (err[0] == socket.SSL_ERROR_WANT_READ
|
||||
or err[0] == socket.SSL_ERROR_WANT_WRITE):
|
||||
|
|
@ -640,11 +655,65 @@ class FakeSocket:
|
|||
except socket.error, err:
|
||||
if err[0] == errno.EINTR:
|
||||
continue
|
||||
if err[0] == errno.EBADF:
|
||||
# XXX socket was closed?
|
||||
break
|
||||
raise
|
||||
if buf == '':
|
||||
else:
|
||||
break
|
||||
msgbuf.append(buf)
|
||||
return StringIO("".join(msgbuf))
|
||||
return buf
|
||||
|
||||
def read(self, size=None):
|
||||
L = [self._buf]
|
||||
avail = len(self._buf)
|
||||
while size is None or avail < size:
|
||||
s = self._read()
|
||||
if s == '':
|
||||
break
|
||||
L.append(s)
|
||||
avail += len(s)
|
||||
all = "".join(L)
|
||||
if size is None:
|
||||
self._buf = ''
|
||||
return all
|
||||
else:
|
||||
self._buf = all[size:]
|
||||
return all[:size]
|
||||
|
||||
def readline(self):
|
||||
L = [self._buf]
|
||||
self._buf = ''
|
||||
while 1:
|
||||
i = L[-1].find("\n")
|
||||
if i >= 0:
|
||||
break
|
||||
s = self._read()
|
||||
if s == '':
|
||||
break
|
||||
L.append(s)
|
||||
if i == -1:
|
||||
# loop exited because there is no more data
|
||||
return "".join(L)
|
||||
else:
|
||||
all = "".join(L)
|
||||
# XXX could do enough bookkeeping not to do a 2nd search
|
||||
i = all.find("\n") + 1
|
||||
line = all[:i]
|
||||
self._buf = all[i:]
|
||||
return line
|
||||
|
||||
def close(self):
|
||||
self._sock.close()
|
||||
|
||||
class FakeSocket:
|
||||
def __init__(self, sock, ssl):
|
||||
self.__sock = sock
|
||||
self.__ssl = ssl
|
||||
|
||||
def makefile(self, mode, bufsize=None):
|
||||
if mode != 'r' and mode != 'rb':
|
||||
raise UnimplementedFileMode()
|
||||
return SSLFile(self.__sock, self.__ssl, bufsize)
|
||||
|
||||
def send(self, stuff, flags = 0):
|
||||
return self.__ssl.write(stuff)
|
||||
|
|
@ -885,7 +954,7 @@ def test():
|
|||
if headers:
|
||||
for header in headers.headers: print header.strip()
|
||||
print
|
||||
print h.getfile().read()
|
||||
print "read", len(h.getfile().read())
|
||||
|
||||
# minimal test that code to extract host from url works
|
||||
class HTTP11(HTTP):
|
||||
|
|
@ -906,13 +975,14 @@ def test():
|
|||
hs.putrequest('GET', selector)
|
||||
hs.endheaders()
|
||||
status, reason, headers = hs.getreply()
|
||||
# XXX why does this give a 302 response?
|
||||
print 'status =', status
|
||||
print 'reason =', reason
|
||||
print
|
||||
if headers:
|
||||
for header in headers.headers: print header.strip()
|
||||
print
|
||||
print hs.getfile().read()
|
||||
print "read", len(hs.getfile().read())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Reference in New Issue