From c6a7bdb356835c9d7513b1ea6846683d446fe6c3 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Mon, 25 Nov 2019 16:36:25 +0100 Subject: [PATCH] bpo-20928: support base-URL and recursive includes in etree.ElementInclude (#5723) * bpo-20928: bring elementtree's XInclude support en-par with the implementation in lxml by adding support for recursive includes and a base-URL. * bpo-20928: Support xincluding the same file multiple times, just not recursively. * bpo-20928: Add 'max_depth' parameter to xinclude that limits the maximum recursion depth to 6 by default. * Add news entry for updated ElementInclude support --- Lib/test/test_xml_etree.py | 82 +++++++++++++++++++ Lib/xml/etree/ElementInclude.py | 56 +++++++++++-- .../2018-03-30-16-18-12.bpo-20928.ieXu6I.rst | 1 + 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-03-30-16-18-12.bpo-20928.ieXu6I.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 9fbb4ac1813..09c234ca689 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1668,6 +1668,17 @@ XINCLUDE["default.xml"] = """\ """.format(html.escape(SIMPLE_XMLFILE, True)) +XINCLUDE["include_c1_repeated.xml"] = """\ + + +

The following is the source code of Recursive1.xml:

+ + + + + +""" + # # badly formatted xi:include tags @@ -1688,6 +1699,31 @@ XINCLUDE_BAD["B2.xml"] = """\ """ +XINCLUDE["Recursive1.xml"] = """\ + + +

The following is the source code of Recursive2.xml:

+ + +""" + +XINCLUDE["Recursive2.xml"] = """\ + + +

The following is the source code of Recursive3.xml:

+ + +""" + +XINCLUDE["Recursive3.xml"] = """\ + + +

The following is the source code of Recursive1.xml:

+ + +""" + + class XIncludeTest(unittest.TestCase): def xinclude_loader(self, href, parse="xml", encoding=None): @@ -1789,6 +1825,13 @@ class XIncludeTest(unittest.TestCase): ' \n' '') # C5 + def test_xinclude_repeated(self): + from xml.etree import ElementInclude + + document = self.xinclude_loader("include_c1_repeated.xml") + ElementInclude.include(document, self.xinclude_loader) + self.assertEqual(1+4*2, len(document.findall(".//p"))) + def test_xinclude_failures(self): from xml.etree import ElementInclude @@ -1821,6 +1864,45 @@ class XIncludeTest(unittest.TestCase): "xi:fallback tag must be child of xi:include " "('{http://www.w3.org/2001/XInclude}fallback')") + # Test infinitely recursive includes. + document = self.xinclude_loader("Recursive1.xml") + with self.assertRaises(ElementInclude.FatalIncludeError) as cm: + ElementInclude.include(document, self.xinclude_loader) + self.assertEqual(str(cm.exception), + "recursive include of Recursive2.xml") + + # Test 'max_depth' limitation. + document = self.xinclude_loader("Recursive1.xml") + with self.assertRaises(ElementInclude.FatalIncludeError) as cm: + ElementInclude.include(document, self.xinclude_loader, max_depth=None) + self.assertEqual(str(cm.exception), + "recursive include of Recursive2.xml") + + document = self.xinclude_loader("Recursive1.xml") + with self.assertRaises(ElementInclude.LimitedRecursiveIncludeError) as cm: + ElementInclude.include(document, self.xinclude_loader, max_depth=0) + self.assertEqual(str(cm.exception), + "maximum xinclude depth reached when including file Recursive2.xml") + + document = self.xinclude_loader("Recursive1.xml") + with self.assertRaises(ElementInclude.LimitedRecursiveIncludeError) as cm: + ElementInclude.include(document, self.xinclude_loader, max_depth=1) + self.assertEqual(str(cm.exception), + "maximum xinclude depth reached when including file Recursive3.xml") + + document = self.xinclude_loader("Recursive1.xml") + with self.assertRaises(ElementInclude.LimitedRecursiveIncludeError) as cm: + ElementInclude.include(document, self.xinclude_loader, max_depth=2) + self.assertEqual(str(cm.exception), + "maximum xinclude depth reached when including file Recursive1.xml") + + document = self.xinclude_loader("Recursive1.xml") + with self.assertRaises(ElementInclude.FatalIncludeError) as cm: + ElementInclude.include(document, self.xinclude_loader, max_depth=3) + self.assertEqual(str(cm.exception), + "recursive include of Recursive2.xml") + + # -------------------------------------------------------------------- # reported bugs diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 963470e3b15..5303062716c 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -50,18 +50,28 @@ import copy from . import ElementTree +from urllib.parse import urljoin XINCLUDE = "{http://www.w3.org/2001/XInclude}" XINCLUDE_INCLUDE = XINCLUDE + "include" XINCLUDE_FALLBACK = XINCLUDE + "fallback" +# For security reasons, the inclusion depth is limited to this read-only value by default. +DEFAULT_MAX_INCLUSION_DEPTH = 6 + + ## # Fatal include error. class FatalIncludeError(SyntaxError): pass + +class LimitedRecursiveIncludeError(FatalIncludeError): + pass + + ## # Default loader. This loader reads an included resource from disk. # @@ -92,13 +102,33 @@ def default_loader(href, parse, encoding=None): # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as default_loader. +# @param base_url The base URL of the original file, to resolve +# relative include file references. +# @param max_depth The maximum number of recursive inclusions. +# Limited to reduce the risk of malicious content explosion. +# Pass a negative value to disable the limitation. +# @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws OSError If the function fails to load a given resource. +# @throws IOError If the function fails to load a given resource. +# @returns the node or its replacement if it was an XInclude node -def include(elem, loader=None): +def include(elem, loader=None, base_url=None, + max_depth=DEFAULT_MAX_INCLUSION_DEPTH): + if max_depth is None: + max_depth = -1 + elif max_depth < 0: + raise ValueError("expected non-negative depth or None for 'max_depth', got %r" % max_depth) + + if hasattr(elem, 'getroot'): + elem = elem.getroot() if loader is None: loader = default_loader + + _include(elem, loader, base_url, max_depth, set()) + + +def _include(elem, loader, base_url, max_depth, _parent_hrefs): # look for xinclude elements i = 0 while i < len(elem): @@ -106,14 +136,24 @@ def include(elem, loader=None): if e.tag == XINCLUDE_INCLUDE: # process xinclude directive href = e.get("href") + if base_url: + href = urljoin(base_url, href) parse = e.get("parse", "xml") if parse == "xml": + if href in _parent_hrefs: + raise FatalIncludeError("recursive include of %s" % href) + if max_depth == 0: + raise LimitedRecursiveIncludeError( + "maximum xinclude depth reached when including file %s" % href) + _parent_hrefs.add(href) node = loader(href, parse) if node is None: raise FatalIncludeError( "cannot load %r as %r" % (href, parse) ) - node = copy.copy(node) + node = copy.copy(node) # FIXME: this makes little sense with recursive includes + _include(node, loader, href, max_depth - 1, _parent_hrefs) + _parent_hrefs.remove(href) if e.tail: node.tail = (node.tail or "") + e.tail elem[i] = node @@ -123,11 +163,13 @@ def include(elem, loader=None): raise FatalIncludeError( "cannot load %r as %r" % (href, parse) ) + if e.tail: + text += e.tail if i: node = elem[i-1] - node.tail = (node.tail or "") + text + (e.tail or "") + node.tail = (node.tail or "") + text else: - elem.text = (elem.text or "") + text + (e.tail or "") + elem.text = (elem.text or "") + text del elem[i] continue else: @@ -139,5 +181,5 @@ def include(elem, loader=None): "xi:fallback tag must be child of xi:include (%r)" % e.tag ) else: - include(e, loader) - i = i + 1 + _include(e, loader, base_url, max_depth, _parent_hrefs) + i += 1 diff --git a/Misc/NEWS.d/next/Library/2018-03-30-16-18-12.bpo-20928.ieXu6I.rst b/Misc/NEWS.d/next/Library/2018-03-30-16-18-12.bpo-20928.ieXu6I.rst new file mode 100644 index 00000000000..25854009077 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-30-16-18-12.bpo-20928.ieXu6I.rst @@ -0,0 +1 @@ +ElementTree supports recursive XInclude processing. Patch by Stefan Behnel.