Issue 4879: Allow buffering for HTTPResponse

Lib/httplib.py

@@ -325,13 +325,18 @@ class HTTPResponse:
 
     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 
-    def __init__(self, sock, debuglevel=0, strict=0, method=None):
-        # The buffer size is specified as zero, because the headers of
-        # the response are read with readline().  If the reads were
-        # buffered the readline() calls could consume some of the
-        # response, which make be read via a recv() on the underlying
-        # socket.
-        self.fp = sock.makefile('rb', 0)
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommendef for performance
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
         self.debuglevel = debuglevel
         self.strict = strict
         self._method = method
@@ -935,7 +940,7 @@ class HTTPConnection:
             self.putheader(hdr, value)
         self.endheaders(body)
 
-    def getresponse(self):
+    def getresponse(self, buffering=False):
         "Get the response from the server."
 
         # if a prior response has been completed, then forget about it.
@@ -961,13 +966,15 @@ class HTTPConnection:
         if self.__state != _CS_REQ_SENT or self.__response:
             raise ResponseNotReady()
 
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
         if self.debuglevel > 0:
-            response = self.response_class(self.sock, self.debuglevel,
-                                           strict=self.strict,
-                                           method=self._method)
-        else:
-            response = self.response_class(self.sock, strict=self.strict,
-                                           method=self._method)
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
 
         response.begin()
         assert response.will_close != _UNKNOWN
@@ -1031,7 +1038,7 @@ class HTTP:
         "Provide a getfile, since the superclass' does not use this concept."
         return self.file
 
-    def getreply(self):
+    def getreply(self, buffering=False):
         """Compat definition since superclass does not define it.
 
         Returns a tuple consisting of:
@@ -1040,7 +1047,12 @@ class HTTP:
         - any RFC822 headers in the response from the server
         """
         try:
-            response = self._conn.getresponse()
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
         except BadStatusLine, e:
             ### hmm. if getresponse() ever closes the socket on a bad request,
             ### then we are going to have problems with self.sock

Lib/test/test_xmlrpc.py

@@ -655,7 +655,7 @@ class FakeSocket:
     def getvalue(self):
         return self.data.getvalue()
 
-    def makefile(self, x, y):
+    def makefile(self, x='r', y=-1):
         raise RuntimeError
 
 class FakeTransport(xmlrpclib.Transport):

Lib/urllib2.py

@@ -1100,7 +1100,10 @@ class AbstractHTTPHandler(BaseHandler):
             (name.title(), val) for name, val in headers.items())
         try:
             h.request(req.get_method(), req.get_selector(), req.data, headers)
-            r = h.getresponse()
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
         except socket.error, err: # XXX what error?
             raise URLError(err)
 

Lib/xmlrpclib.py

@@ -1234,7 +1234,7 @@ class Transport:
         self.send_user_agent(h)
         self.send_content(h, request_body)
 
-        errcode, errmsg, headers = h.getreply()
+        errcode, errmsg, headers = h.getreply(buffering=True)
 
         if errcode != 200:
             raise ProtocolError(
@@ -1245,12 +1245,7 @@ class Transport:
 
         self.verbose = verbose
 
-        try:
-            sock = h._conn.sock
-        except AttributeError:
-            sock = None
-
-        return self._parse_response(h.getfile(), sock)
+        return self._parse_response(h.getfile())
 
     ##
     # Create parser.
@@ -1355,29 +1350,12 @@ class Transport:
     # @return Response tuple and target method.
 
     def parse_response(self, file):
-        # compatibility interface
-        return self._parse_response(file, None)
-
-    ##
-    # Parse response (alternate interface).  This is similar to the
-    # parse_response method, but also provides direct access to the
-    # underlying socket object (where available).
-    #
-    # @param file Stream.
-    # @param sock Socket handle (or None, if the socket object
-    #    could not be accessed).
-    # @return Response tuple and target method.
-
-    def _parse_response(self, file, sock):
         # read response from input file/socket, and parse it
 
         p, u = self.getparser()
 
         while 1:
-            if sock:
-                response = sock.recv(1024)
-            else:
-                response = file.read(1024)
+            response = file.read(1024)
             if not response:
                 break
             if self.verbose: