From 6166519d2bfb0b6fd7ff304da89c59b10557d4a3 Mon Sep 17 00:00:00 2001 From: Florent Xicluna Date: Tue, 15 Nov 2011 20:53:25 +0100 Subject: [PATCH] Closes #13297: use bytes type to send and receive binary data through XMLRPC. --- Doc/library/xmlrpc.client.rst | 53 ++++++++++++++++--------- Lib/test/test_xmlrpc.py | 73 ++++++++++++++++++++++++++++------- Lib/xmlrpc/client.py | 52 +++++++++++++++++-------- Misc/NEWS | 2 + 4 files changed, 132 insertions(+), 48 deletions(-) diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index e72770ae935..1871c998ec8 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -8,7 +8,7 @@ .. XXX Not everything is documented yet. It might be good to describe - Marshaller, Unmarshaller, getparser, dumps, loads, and Transport. + Marshaller, Unmarshaller, getparser and Transport. **Source code:** :source:`Lib/xmlrpc/client.py` @@ -21,7 +21,12 @@ supports writing XML-RPC client code; it handles all the details of translating between conformable Python objects and XML on the wire. -.. class:: ServerProxy(uri, transport=None, encoding=None, verbose=False, allow_none=False, use_datetime=False) +.. class:: ServerProxy(uri, transport=None, encoding=None, verbose=False, \ + allow_none=False, use_datetime=False, \ + use_builtin_types=False) + + .. versionchanged:: 3.3 + The *use_builtin_types* flag was added. A :class:`ServerProxy` instance is an object that manages communication with a remote XML-RPC server. The required first argument is a URI (Uniform Resource @@ -34,9 +39,13 @@ between conformable Python objects and XML on the wire. XML; the default behaviour is for ``None`` to raise a :exc:`TypeError`. This is a commonly-used extension to the XML-RPC specification, but isn't supported by all clients and servers; see http://ontosys.com/xml-rpc/extensions.php for a - description. The *use_datetime* flag can be used to cause date/time values to - be presented as :class:`datetime.datetime` objects; this is false by default. - :class:`datetime.datetime` objects may be passed to calls. + description. The *use_builtin_types* flag can be used to cause date/time values + to be presented as :class:`datetime.datetime` objects and binary data to be + presented as :class:`bytes` objects; this flag is false by default. + :class:`datetime.datetime` and :class:`bytes` objects may be passed to calls. + + The obsolete *use_datetime* flag is similar to *use_builtin_types* but it + applies only to date/time values. Both the HTTP and HTTPS transports support the URL syntax extension for HTTP Basic Authentication: ``http://user:pass@host:port/path``. The ``user:pass`` @@ -78,12 +87,12 @@ between conformable Python objects and XML on the wire. | | only their *__dict__* attribute is | | | transmitted. | +---------------------------------+---------------------------------------------+ - | :const:`dates` | in seconds since the epoch (pass in an | - | | instance of the :class:`DateTime` class) or | + | :const:`dates` | In seconds since the epoch. Pass in an | + | | instance of the :class:`DateTime` class or | | | a :class:`datetime.datetime` instance. | +---------------------------------+---------------------------------------------+ - | :const:`binary data` | pass in an instance of the :class:`Binary` | - | | wrapper class | + | :const:`binary data` | Pass in an instance of the :class:`Binary` | + | | wrapper class or a :class:`bytes` instance. | +---------------------------------+---------------------------------------------+ This is the full set of data types supported by XML-RPC. Method calls may also @@ -98,8 +107,9 @@ between conformable Python objects and XML on the wire. ensure that the string is free of characters that aren't allowed in XML, such as the control characters with ASCII values between 0 and 31 (except, of course, tab, newline and carriage return); failing to do this will result in an XML-RPC - request that isn't well-formed XML. If you have to pass arbitrary strings via - XML-RPC, use the :class:`Binary` wrapper class described below. + request that isn't well-formed XML. If you have to pass arbitrary bytes + via XML-RPC, use the :class:`bytes` class or the class:`Binary` wrapper class + described below. :class:`Server` is retained as an alias for :class:`ServerProxy` for backwards compatibility. New code should use :class:`ServerProxy`. @@ -249,7 +259,7 @@ The client code for the preceding server:: Binary Objects -------------- -This class may be initialized from string data (which may include NULs). The +This class may be initialized from bytes data (which may include NULs). The primary access to the content of a :class:`Binary` object is provided by an attribute: @@ -257,15 +267,15 @@ attribute: .. attribute:: Binary.data The binary data encapsulated by the :class:`Binary` instance. The data is - provided as an 8-bit string. + provided as a :class:`bytes` object. :class:`Binary` objects have the following methods, supported mainly for internal use by the marshalling/unmarshalling code: -.. method:: Binary.decode(string) +.. method:: Binary.decode(bytes) - Accept a base64 string and decode it as the instance's new data. + Accept a base64 :class:`bytes` object and decode it as the instance's new data. .. method:: Binary.encode(out) @@ -471,14 +481,21 @@ Convenience Functions it via an extension, provide a true value for *allow_none*. -.. function:: loads(data, use_datetime=False) +.. function:: loads(data, use_datetime=False, use_builtin_types=False) Convert an XML-RPC request or response into Python objects, a ``(params, methodname)``. *params* is a tuple of argument; *methodname* is a string, or ``None`` if no method name is present in the packet. If the XML-RPC packet represents a fault condition, this function will raise a :exc:`Fault` exception. - The *use_datetime* flag can be used to cause date/time values to be presented as - :class:`datetime.datetime` objects; this is false by default. + The *use_builtin_types* flag can be used to cause date/time values to be + presented as :class:`datetime.datetime` objects and binary data to be + presented as :class:`bytes` objects; this flag is false by default. + + The obsolete *use_datetime* flag is similar to *use_builtin_types* but it + applies only to date/time values. + + .. versionchanged:: 3.3 + The *use_builtin_types* flag was added. .. _xmlrpc-client-example: diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index e97420b8a8d..76723fad013 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -24,6 +24,8 @@ alist = [{'astring': 'foo@bar.baz.spam', 'ashortlong': 2, 'anotherlist': ['.zyx.41'], 'abase64': xmlrpclib.Binary(b"my dog has fleas"), + 'b64bytes': b"my dog has fleas", + 'b64bytearray': bytearray(b"my dog has fleas"), 'boolean': False, 'unicode': '\u4000\u6000\u8000', 'ukey\u4000': 'regular value', @@ -44,27 +46,54 @@ class XMLRPCTestCase(unittest.TestCase): def test_dump_bare_datetime(self): # This checks that an unwrapped datetime.date object can be handled # by the marshalling code. This can't be done via test_dump_load() - # since with use_datetime set to 1 the unmarshaller would create + # since with use_builtin_types set to 1 the unmarshaller would create # datetime objects for the 'datetime[123]' keys as well dt = datetime.datetime(2005, 2, 10, 11, 41, 23) + self.assertEqual(dt, xmlrpclib.DateTime('20050210T11:41:23')) s = xmlrpclib.dumps((dt,)) - (newdt,), m = xmlrpclib.loads(s, use_datetime=1) - self.assertEqual(newdt, dt) - self.assertEqual(m, None) - (newdt,), m = xmlrpclib.loads(s, use_datetime=0) - self.assertEqual(newdt, xmlrpclib.DateTime('20050210T11:41:23')) + result, m = xmlrpclib.loads(s, use_builtin_types=True) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), datetime.datetime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_builtin_types=False) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), xmlrpclib.DateTime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_datetime=True) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), datetime.datetime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_datetime=False) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), xmlrpclib.DateTime) + self.assertIsNone(m) + def test_datetime_before_1900(self): # same as before but with a date before 1900 dt = datetime.datetime(1, 2, 10, 11, 41, 23) + self.assertEqual(dt, xmlrpclib.DateTime('00010210T11:41:23')) s = xmlrpclib.dumps((dt,)) - (newdt,), m = xmlrpclib.loads(s, use_datetime=1) - self.assertEqual(newdt, dt) - self.assertEqual(m, None) - (newdt,), m = xmlrpclib.loads(s, use_datetime=0) - self.assertEqual(newdt, xmlrpclib.DateTime('00010210T11:41:23')) + result, m = xmlrpclib.loads(s, use_builtin_types=True) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), datetime.datetime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_builtin_types=False) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), xmlrpclib.DateTime) + self.assertIsNone(m) def test_bug_1164912 (self): d = xmlrpclib.DateTime() @@ -133,6 +162,25 @@ class XMLRPCTestCase(unittest.TestCase): xmlrpclib.loads(strg)[0][0]) self.assertRaises(TypeError, xmlrpclib.dumps, (arg1,)) + def test_dump_bytes(self): + sample = b"my dog has fleas" + self.assertEqual(sample, xmlrpclib.Binary(sample)) + for type_ in bytes, bytearray, xmlrpclib.Binary: + value = type_(sample) + s = xmlrpclib.dumps((value,)) + + result, m = xmlrpclib.loads(s, use_builtin_types=True) + (newvalue,) = result + self.assertEqual(newvalue, sample) + self.assertIs(type(newvalue), bytes) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_builtin_types=False) + (newvalue,) = result + self.assertEqual(newvalue, sample) + self.assertIs(type(newvalue), xmlrpclib.Binary) + self.assertIsNone(m) + def test_get_host_info(self): # see bug #3613, this raised a TypeError transp = xmlrpc.client.Transport() @@ -140,9 +188,6 @@ class XMLRPCTestCase(unittest.TestCase): ('host.tld', [('Authorization', 'Basic dXNlcg==')], {})) - def test_dump_bytes(self): - self.assertRaises(TypeError, xmlrpclib.dumps, (b"my dog has fleas",)) - def test_ssl_presence(self): try: import ssl diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index ff931e75879..59f75462ff3 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -386,8 +386,8 @@ class Binary: if data is None: data = b"" else: - if not isinstance(data, bytes): - raise TypeError("expected bytes, not %s" % + if not isinstance(data, (bytes, bytearray)): + raise TypeError("expected bytes or bytearray, not %s" % data.__class__.__name__) data = bytes(data) # Make a copy of the bytes! self.data = data @@ -559,6 +559,14 @@ class Marshaller: write("\n") dispatch[str] = dump_unicode + def dump_bytes(self, value, write): + write("\n") + encoded = base64.encodebytes(value) + write(encoded.decode('ascii')) + write("\n") + dispatch[bytes] = dump_bytes + dispatch[bytearray] = dump_bytes + def dump_array(self, value, write): i = id(value) if i in self.memo: @@ -629,7 +637,7 @@ class Unmarshaller: # and again, if you don't understand what's going on in here, # that's perfectly ok. - def __init__(self, use_datetime=False): + def __init__(self, use_datetime=False, use_builtin_types=False): self._type = None self._stack = [] self._marks = [] @@ -637,7 +645,8 @@ class Unmarshaller: self._methodname = None self._encoding = "utf-8" self.append = self._stack.append - self._use_datetime = use_datetime + self._use_datetime = use_builtin_types or use_datetime + self._use_bytes = use_builtin_types def close(self): # return response tuple and target method @@ -749,6 +758,8 @@ class Unmarshaller: def end_base64(self, data): value = Binary() value.decode(data.encode("ascii")) + if self._use_bytes: + value = value.data self.append(value) self._value = 0 dispatch["base64"] = end_base64 @@ -860,21 +871,26 @@ FastMarshaller = FastParser = FastUnmarshaller = None # # return A (parser, unmarshaller) tuple. -def getparser(use_datetime=False): +def getparser(use_datetime=False, use_builtin_types=False): """getparser() -> parser, unmarshaller Create an instance of the fastest available parser, and attach it to an unmarshalling object. Return both objects. """ if FastParser and FastUnmarshaller: - if use_datetime: + if use_builtin_types: mkdatetime = _datetime_type + mkbytes = base64.decodebytes + elif use_datetime: + mkdatetime = _datetime_type + mkbytes = _binary else: mkdatetime = _datetime - target = FastUnmarshaller(True, False, _binary, mkdatetime, Fault) + mkbytes = _binary + target = FastUnmarshaller(True, False, mkbytes, mkdatetime, Fault) parser = FastParser(target) else: - target = Unmarshaller(use_datetime=use_datetime) + target = Unmarshaller(use_datetime=use_datetime, use_builtin_types=use_builtin_types) if FastParser: parser = FastParser(target) else: @@ -912,7 +928,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None, encoding: the packet encoding (default is UTF-8) - All 8-bit strings in the data structure are assumed to use the + All byte strings in the data structure are assumed to use the packet encoding. Unicode strings are automatically converted, where necessary. """ @@ -971,7 +987,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None, # (None if not present). # @see Fault -def loads(data, use_datetime=False): +def loads(data, use_datetime=False, use_builtin_types=False): """data -> unmarshalled data, method name Convert an XML-RPC packet to unmarshalled data plus a method @@ -980,7 +996,7 @@ def loads(data, use_datetime=False): If the XML-RPC packet represents a fault condition, this function raises a Fault exception. """ - p, u = getparser(use_datetime=use_datetime) + p, u = getparser(use_datetime=use_datetime, use_builtin_types=use_builtin_types) p.feed(data) p.close() return u.close(), u.getmethodname() @@ -1092,8 +1108,9 @@ class Transport: # that they can decode such a request encode_threshold = None #None = don't encode - def __init__(self, use_datetime=False): + def __init__(self, use_datetime=False, use_builtin_types=False): self._use_datetime = use_datetime + self._use_builtin_types = use_builtin_types self._connection = (None, None) self._extra_headers = [] @@ -1154,7 +1171,8 @@ class Transport: def getparser(self): # get parser and unmarshaller - return getparser(use_datetime=self._use_datetime) + return getparser(use_datetime=self._use_datetime, + use_builtin_types=self._use_builtin_types) ## # Get authorization info from host parameter @@ -1361,7 +1379,7 @@ class ServerProxy: """ def __init__(self, uri, transport=None, encoding=None, verbose=False, - allow_none=False, use_datetime=False): + allow_none=False, use_datetime=False, use_builtin_types=False): # establish a "logical" server connection # get the url @@ -1375,9 +1393,11 @@ class ServerProxy: if transport is None: if type == "https": - transport = SafeTransport(use_datetime=use_datetime) + handler = SafeTransport else: - transport = Transport(use_datetime=use_datetime) + handler = Transport + transport = handler(use_datetime=use_datetime, + use_builtin_types=use_builtin_types) self.__transport = transport self.__encoding = encoding or 'utf-8' diff --git a/Misc/NEWS b/Misc/NEWS index ed713e35b29..3ed7e7ab162 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -374,6 +374,8 @@ Core and Builtins Library ------- +- Issue #13297: Use bytes type to send and receive binary data through XMLRPC. + - Issue #6397: Support "/dev/poll" polling objects in select module, under Solaris & derivatives.