Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

http://bugs.python.org/issue6267 

Cumulative patch to http and xmlrpc
This commit is contained in:
Kristján Valur Jónsson 2009-06-28 21:04:17 +00:00
parent 552e7a7e2f
commit e007860b8b
7 changed files with 406 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Doc/library/simplexmlrpcserver.rst
View File

@ -133,6 +133,15 @@ alone XML-RPC servers.
.. versionadded:: 2.5
.. attribute:: SimpleXMLRPCRequestHandler.encode_threshold
If this attribute is not ``None``, responses larger than this value
will be encoded using the *gzip* transfer encoding, if permitted by
the client. The default is ``1400`` which corresponds roughly
to a single TCP packet.
.. versionadded:: 2.7
.. _simplexmlrpcserver-example:
SimpleXMLRPCServer Example

28
Lib/BaseHTTPServer.py
View File

@ -309,18 +309,26 @@ class BaseHTTPRequestHandler(SocketServer.StreamRequestHandler):
commands such as GET and POST.
"""
self.raw_requestline = self.rfile.readline()
if not self.raw_requestline:
try:
self.raw_requestline = self.rfile.readline()
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
# An error code has been sent, just exit
return
mname = 'do_' + self.command
if not hasattr(self, mname):
self.send_error(501, "Unsupported method (%r)" % self.command)
return
method = getattr(self, mname)
method()
self.wfile.flush() #actually send the response if not already done.
except socket.timeout, e:
#a read or a write timed out. Discard this connection
self.log_error("Request timed out: %r", e)
self.close_connection = 1
return
if not self.parse_request(): # An error code has been sent, just exit
return
mname = 'do_' + self.command
if not hasattr(self, mname):
self.send_error(501, "Unsupported method (%r)" % self.command)
return
method = getattr(self, mname)
method()
def handle(self):
"""Handle multiple requests if necessary."""

4
Lib/DocXMLRPCServer.py
View File

@ -240,10 +240,6 @@ class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
class DocXMLRPCServer( SimpleXMLRPCServer,
XMLRPCDocGenerator):
"""XML-RPC and HTML documentation server.

57
Lib/SimpleXMLRPCServer.py
View File

@ -106,6 +106,7 @@ import BaseHTTPServer
import sys
import os
import traceback
import re
try:
import fcntl
except ImportError:
@ -430,6 +431,31 @@ class SimpleXMLRPCRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
# paths not on this list will result in a 404 error.
rpc_paths = ('/', '/RPC2')
#if not None, encode responses larger than this, if possible
encode_threshold = 1400 #a common MTU
#Override form StreamRequestHandler: full buffering of output
#and no Nagle.
wbufsize = -1
disable_nagle_algorithm = True
# a re to match a gzip Accept-Encoding
aepattern = re.compile(r"""
\s* ([^\s;]+) \s* #content-coding
(;\s* q \s*=\s* ([0-9\.]+))? #q
""", re.VERBOSE | re.IGNORECASE)
def accept_encodings(self):
r = {}
ae = self.headers.get("Accept-Encoding", "")
for e in ae.split(","):
match = self.aepattern.match(e)
if match:
v = match.group(3)
v = float(v) if v else 1.0
r[match.group(1)] = v
return r
def is_rpc_path_valid(self):
if self.rpc_paths:
return self.path in self.rpc_paths
@ -463,6 +489,10 @@ class SimpleXMLRPCRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
size_remaining -= len(L[-1])
data = ''.join(L)
data = self.decode_request_content(data)
if data is None:
return #response has been sent
# In previous versions of SimpleXMLRPCServer, _dispatch
# could be overridden in this class, instead of in
# SimpleXMLRPCDispatcher. To maintain backwards compatibility,
@ -481,18 +511,36 @@ class SimpleXMLRPCRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
self.send_header("X-exception", str(e))
self.send_header("X-traceback", traceback.format_exc())
self.send_header("Content-length", "0")
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
if self.encode_threshold is not None:
if len(response) > self.encode_threshold:
q = self.accept_encodings().get("gzip", 0)
if q:
response = xmlrpclib.gzip_encode(response)
self.send_header("Content-Encoding", "gzip")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
def decode_request_content(self, data):
#support gzip encoding of request
encoding = self.headers.get("content-encoding", "identity").lower()
if encoding == "identity":
return data
if encoding == "gzip":
try:
return xmlrpclib.gzip_decode(data)
except ValueError:
self.send_response(400, "error decoding gzip content")
else:
self.send_response(501, "encoding %r not supported" % encoding)
self.send_header("Content-length", "0")
self.end_headers()
def report_404 (self):
# Report a 404 error
@ -502,9 +550,6 @@ class SimpleXMLRPCRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
def log_request(self, code='-', size='-'):
"""Selectively log an accepted request."""

11
Lib/SocketServer.py
View File

@ -445,6 +445,7 @@ class TCPServer(BaseServer):
def close_request(self, request):
"""Called to clean up an individual request."""
request.shutdown(socket.SHUT_WR)
request.close()
@ -610,12 +611,11 @@ class BaseRequestHandler:
self.request = request
self.client_address = client_address
self.server = server
self.setup()
try:
self.setup()
self.handle()
self.finish()
finally:
sys.exc_traceback = None # Help garbage collection
self.finish()
def setup(self):
pass
@ -649,12 +649,17 @@ class StreamRequestHandler(BaseRequestHandler):
rbufsize = -1
wbufsize = 0
# A timeout to apply to the request socket, if not None.
timeout = None
# Disable nagle algoritm for this socket, if True.
# Use only when wbufsize != 0, to avoid small packets.
disable_nagle_algorithm = False
def setup(self):
self.connection = self.request
if self.timeout is not None:
self.connection.settimeout(self.timeout)
if self.disable_nagle_algorithm:
self.connection.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, True)

142
Lib/test/test_xmlrpc.py
View File

@ -273,7 +273,7 @@ ADDR = PORT = URL = None
# The evt is set twice. First when the server is ready to serve.
# Second when the server has been shutdown. The user must clear
# the event after it has been set the first time to catch the second set.
def http_server(evt, numrequests):
def http_server(evt, numrequests, requestHandler=None):
class TestInstanceClass:
def div(self, x, y):
return x // y
@ -294,7 +294,9 @@ def http_server(evt, numrequests):
s.setblocking(True)
return s, port
serv = MyXMLRPCServer(("localhost", 0),
if not requestHandler:
requestHandler = SimpleXMLRPCServer.SimpleXMLRPCRequestHandler
serv = MyXMLRPCServer(("localhost", 0), requestHandler,
logRequests=False, bind_and_activate=False)
try:
serv.socket.settimeout(3)
@ -348,34 +350,36 @@ def is_unavailable_exception(e):
return False
# NOTE: The tests in SimpleServerTestCase will ignore failures caused by
# "temporarily unavailable" exceptions raised in SimpleXMLRPCServer. This
# condition occurs infrequently on some platforms, frequently on others, and
# is apparently caused by using SimpleXMLRPCServer with a non-blocking socket.
# If the server class is updated at some point in the future to handle this
# situation more gracefully, these tests should be modified appropriately.
class SimpleServerTestCase(unittest.TestCase):
class BaseServerTestCase(unittest.TestCase):
requestHandler = None
def setUp(self):
# enable traceback reporting
SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = True
self.evt = threading.Event()
# start server thread to handle requests
serv_args = (self.evt, 1)
serv_args = (self.evt, 1, self.requestHandler)
threading.Thread(target=http_server, args=serv_args).start()
# wait for the server to be ready
self.evt.wait()
self.evt.wait(10)
self.evt.clear()
def tearDown(self):
# wait on the server thread to terminate
self.evt.wait()
self.evt.wait(10)
# disable traceback reporting
SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = False
# NOTE: The tests in SimpleServerTestCase will ignore failures caused by
# "temporarily unavailable" exceptions raised in SimpleXMLRPCServer. This
# condition occurs infrequently on some platforms, frequently on others, and
# is apparently caused by using SimpleXMLRPCServer with a non-blocking socket
# If the server class is updated at some point in the future to handle this
# situation more gracefully, these tests should be modified appropriately.
class SimpleServerTestCase(BaseServerTestCase):
def test_simple1(self):
try:
p = xmlrpclib.ServerProxy(URL)
@ -512,6 +516,110 @@ class SimpleServerTestCase(unittest.TestCase):
# This avoids waiting for the socket timeout.
self.test_simple1()
#A test case that verifies that a server using the HTTP/1.1 keep-alive mechanism
#does indeed serve subsequent requests on the same connection
class KeepaliveServerTestCase(BaseServerTestCase):
#a request handler that supports keep-alive and logs requests into a
#class variable
class RequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
parentClass = SimpleXMLRPCServer.SimpleXMLRPCRequestHandler
protocol_version = 'HTTP/1.1'
myRequests = []
def handle(self):
self.myRequests.append([])
return self.parentClass.handle(self)
def handle_one_request(self):
result = self.parentClass.handle_one_request(self)
self.myRequests[-1].append(self.raw_requestline)
return result
requestHandler = RequestHandler
def setUp(self):
#clear request log
self.RequestHandler.myRequests = []
return BaseServerTestCase.setUp(self)
def test_two(self):
p = xmlrpclib.ServerProxy(URL)
self.assertEqual(p.pow(6,8), 6**8)
self.assertEqual(p.pow(6,8), 6**8)
self.assertEqual(len(self.RequestHandler.myRequests), 1)
#we may or may not catch the final "append" with the empty line
self.assertTrue(len(self.RequestHandler.myRequests[-1]) >= 2)
#A test case that verifies that gzip encoding works in both directions
#(for a request and the response)
class GzipServerTestCase(BaseServerTestCase):
#a request handler that supports keep-alive and logs requests into a
#class variable
class RequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
parentClass = SimpleXMLRPCServer.SimpleXMLRPCRequestHandler
protocol_version = 'HTTP/1.1'
def do_POST(self):
#store content of last request in class
self.__class__.content_length = int(self.headers["content-length"])
return self.parentClass.do_POST(self)
requestHandler = RequestHandler
class Transport(xmlrpclib.Transport):
#custom transport, stores the response length for our perusal
fake_gzip = False
def parse_response(self, response):
self.response_length=int(response.getheader("content-length", 0))
return xmlrpclib.Transport.parse_response(self, response)
def send_content(self, connection, body):
if self.fake_gzip:
#add a lone gzip header to induce decode error remotely
connection.putheader("Content-Encoding", "gzip")
return xmlrpclib.Transport.send_content(self, connection, body)
def test_gzip_request(self):
t = self.Transport()
t.encode_threshold = None
p = xmlrpclib.ServerProxy(URL, transport=t)
self.assertEqual(p.pow(6,8), 6**8)
a = self.RequestHandler.content_length
t.encode_threshold = 0 #turn on request encoding
self.assertEqual(p.pow(6,8), 6**8)
b = self.RequestHandler.content_length
self.assertTrue(a>b)
def test_bad_gzip_request(self):
t = self.Transport()
t.encode_threshold = None
t.fake_gzip = True
p = xmlrpclib.ServerProxy(URL, transport=t)
cm = self.assertRaisesRegexp(xmlrpclib.ProtocolError,
re.compile(r"\b400\b"))
with cm:
p.pow(6, 8)
def test_gsip_response(self):
t = self.Transport()
p = xmlrpclib.ServerProxy(URL, transport=t)
old = self.requestHandler.encode_threshold
self.requestHandler.encode_threshold = None #no encoding
self.assertEqual(p.pow(6,8), 6**8)
a = t.response_length
self.requestHandler.encode_threshold = 0 #always encode
self.assertEqual(p.pow(6,8), 6**8)
b = t.response_length
self.requestHandler.encode_threshold = old
self.assertTrue(a>b)
#Test special attributes of the ServerProxy object
class ServerProxyTestCase(unittest.TestCase):
def test_close(self):
p = xmlrpclib.ServerProxy(URL)
self.assertEqual(p('close')(), None)
def test_transport(self):
t = xmlrpclib.Transport()
p = xmlrpclib.ServerProxy(URL, transport=t)
self.assertEqual(p('transport'), t)
# This is a contrived way to make a failure occur on the server side
# in order to test the _send_traceback_header flag on the server
class FailingMessageClass(mimetools.Message):
@ -693,6 +801,9 @@ class FakeSocket:
def makefile(self, x='r', y=-1):
raise RuntimeError
def close(self):
pass
class FakeTransport(xmlrpclib.Transport):
"""A Transport instance that records instead of sending a request.
@ -703,7 +814,7 @@ class FakeTransport(xmlrpclib.Transport):
def make_connection(self, host):
conn = xmlrpclib.Transport.make_connection(self, host)
conn._conn.sock = self.fake_socket = FakeSocket()
conn.sock = self.fake_socket = FakeSocket()
return conn
class TransportSubclassTestCase(unittest.TestCase):
@ -763,6 +874,9 @@ def test_main():
xmlrpc_tests = [XMLRPCTestCase, HelperTestCase, DateTimeTestCase,
BinaryTestCase, FaultTestCase, TransportSubclassTestCase]
xmlrpc_tests.append(SimpleServerTestCase)
xmlrpc_tests.append(KeepaliveServerTestCase)
xmlrpc_tests.append(GzipServerTestCase)
xmlrpc_tests.append(ServerProxyTestCase)
xmlrpc_tests.append(FailingServerTestCase)
xmlrpc_tests.append(CGIHandlerTestCase)

225
Lib/xmlrpclib.py
View File

@ -139,6 +139,10 @@ Exported functions:
import re, string, time, operator
from types import *
import gzip
import socket
import errno
import httplib
# --------------------------------------------------------------------
# Internal stuff
@ -1129,6 +1133,72 @@ def loads(data, use_datetime=0):
p.close()
return u.close(), u.getmethodname()
##
# Encode a string using the gzip content encoding such as specified by the
# Content-Encoding: gzip
# in the HTTP header, as described in RFC 1952
#
# @param data the unencoded data
# @return the encoded data
def gzip_encode(data):
"""data -> gzip encoded data
Encode data using the gzip content encoding as described in RFC 1952
"""
f = StringIO.StringIO()
gzf = gzip.GzipFile(mode="wb", fileobj=f, compresslevel=1)
gzf.write(data)
gzf.close()
encoded = f.getvalue()
f.close()
return encoded
##
# Decode a string using the gzip content encoding such as specified by the
# Content-Encoding: gzip
# in the HTTP header, as described in RFC 1952
#
# @param data The encoded data
# @return the unencoded data
# @raises ValueError if data is not correctly coded.
def gzip_decode(data):
"""gzip encoded data -> unencoded data
Decode data using the gzip content encoding as described in RFC 1952
"""
f = StringIO.StringIO(data)
gzf = gzip.GzipFile(mode="rb", fileobj=f)
try:
decoded = gzf.read()
except IOError:
raise ValueError("invalid data")
f.close()
gzf.close()
return decoded
##
# Return a decoded file-like object for the gzip encoding
# as described in RFC 1952.
#
# @param response A stream supporting a read() method
# @return a file-like object that the decoded data can be read() from
class GzipDecodedResponse(gzip.GzipFile):
"""a file-like object to decode a response encoded with the gzip
method, as described in RFC 1952.
"""
def __init__(self, response):
#response doesn't support tell() and read(), required by
#GzipFile
self.stringio = StringIO.StringIO(response.read())
gzip.GzipFile.__init__(self, mode="rb", fileobj=self.stringio)
def close(self):
gzip.GzipFile.close(self)
self.stringio.close()
# --------------------------------------------------------------------
# request dispatcher
@ -1156,8 +1226,39 @@ class Transport:
# client identifier (may be overridden)
user_agent = "xmlrpclib.py/%s (by www.pythonware.com)" % __version__
#if true, we'll request gzip encoding
accept_gzip_encoding = True
# if positive, encode request using gzip if it exceeds this threshold
# note that many server will get confused, so only use it if you know
# that they can decode such a request
encode_threshold = None #None = don't encode
def __init__(self, use_datetime=0):
self._use_datetime = use_datetime
self._connection = (None, None)
self._extra_headers = []
##
# Send a complete request, and parse the response.
# Retry request if a cached connection has disconnected.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except (socket.error, httplib.HTTPException), e:
retry = (errno.ECONNRESET,
errno.ECONNABORTED,
httplib.BadStatusLine) #close after we sent request
if i or e[0] not in retry:
raise
##
# Send a complete request, and parse the response.
@ -1168,30 +1269,39 @@ class Transport:
# @param verbose Debugging flag.
# @return Parsed response.
def request(self, host, handler, request_body, verbose=0):
def single_request(self, host, handler, request_body, verbose=0):
# issue XML-RPC request
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_user_agent(h)
self.send_content(h, request_body)
try:
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply(buffering=True)
response = h.getresponse(buffering=True)
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
except Fault:
raise
except Exception:
# All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
if errcode != 200:
raise ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
return self.parse_response(h.getfile())
#discard any response data and raise exception
if (response.getheader("content-length", 0)):
response.read()
raise ProtocolError(
host + handler,
response.status, response.reason,
response.msg,
)
##
# Create parser.
@ -1240,10 +1350,25 @@ class Transport:
# @return A connection handle.
def make_connection(self, host):
#return an existing connection if possible. This allows
#HTTP/1.1 keep-alive.
if self._connection and host == self._connection[0]:
return self._connection[1]
# create a HTTP connection object from a host descriptor
import httplib
host, extra_headers, x509 = self.get_host_info(host)
return httplib.HTTP(host)
chost, self._extra_headers, x509 = self.get_host_info(host)
#store the host argument along with the connection object
self._connection = host, httplib.HTTPConnection(chost)
return self._connection[1]
##
# Clear any cached connection object.
# Used in the event of socket errors.
#
def close(self):
if self._connection[1]:
self._connection[1].close()
self._connection = (None, None)
##
# Send request header.
@ -1253,17 +1378,24 @@ class Transport:
# @param request_body XML-RPC body.
def send_request(self, connection, handler, request_body):
connection.putrequest("POST", handler)
if (self.accept_gzip_encoding):
connection.putrequest("POST", handler, skip_accept_encoding=True)
connection.putheader("Accept-Encoding", "gzip")
else:
connection.putrequest("POST", handler)
##
# Send host name.
#
# @param connection Connection handle.
# @param host Host name.
#
# Note: This function doesn't actually add the "Host"
# header anymore, it is done as part of the connection.putrequest() in
# send_request() above.
def send_host(self, connection, host):
host, extra_headers, x509 = self.get_host_info(host)
connection.putheader("Host", host)
extra_headers = self._extra_headers
if extra_headers:
if isinstance(extra_headers, DictType):
extra_headers = extra_headers.items()
@ -1286,6 +1418,13 @@ class Transport:
def send_content(self, connection, request_body):
connection.putheader("Content-Type", "text/xml")
#optionally encode the request
if (self.encode_threshold is not None and
self.encode_threshold < len(request_body)):
connection.putheader("Content-Encoding", "gzip")
request_body = gzip_encode(request_body)
connection.putheader("Content-Length", str(len(request_body)))
connection.endheaders(request_body)
@ -1295,20 +1434,25 @@ class Transport:
# @param file Stream.
# @return Response tuple and target method.
def parse_response(self, file):
# read response from input file/socket, and parse it
def parse_response(self, response):
# read response data from httpresponse, and parse it
if response.getheader("Content-Encoding", "") == "gzip":
stream = GzipDecodedResponse(response)
else:
stream = response
p, u = self.getparser()
while 1:
response = file.read(1024)
if not response:
data = stream.read(1024)
if not data:
break
if self.verbose:
print "body:", repr(response)
p.feed(response)
print "body:", repr(data)
p.feed(data)
file.close()
if stream is not response:
stream.close()
p.close()
return u.close()
@ -1322,18 +1466,20 @@ class SafeTransport(Transport):
# FIXME: mostly untested
def make_connection(self, host):
if self._connection and host == self._connection[0]:
return self._connection[1]
# create a HTTPS connection object from a host descriptor
# host may be a string, or a (host, x509-dict) tuple
import httplib
host, extra_headers, x509 = self.get_host_info(host)
try:
HTTPS = httplib.HTTPS
HTTPS = httplib.HTTPSConnection
except AttributeError:
raise NotImplementedError(
"your version of httplib doesn't support HTTPS"
)
else:
return HTTPS(host, None, **(x509 or {}))
chost, self._extra_headers, x509 = self.get_host_info(host)
self._connection = host, HTTPSConnection(chost, None, **(x509 or {}))
return self._connection[1]
##
# Standard server proxy. This class establishes a virtual connection
@ -1398,6 +1544,9 @@ class ServerProxy:
self.__verbose = verbose
self.__allow_none = allow_none
def __close(self):
self.__transport.close()
def __request(self, methodname, params):
# call a method on the remote server
@ -1431,6 +1580,16 @@ class ServerProxy:
# note: to call a remote object with an non-standard name, use
# result getattr(server, "strange-python-name")(args)
def __call__(self, attr):
"""A workaround to get special attributes on the ServerProxy
without interfering with the magic __getattr__
"""
if attr == "close":
return self.__close
elif attr == "transport":
return self.__transport
raise AttributeError("Attribute %r not found" % (attr,))
# compatibility
Server = ServerProxy