From 7de56f6a04ab60010b3e2185752b3ddf681d068d Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 1 Nov 2011 14:12:22 +0200 Subject: [PATCH] #670664: Fix HTMLParser to correctly handle the content of ```` and ````. --- Doc/library/html.parser.rst | 3 ++- Lib/html/parser.py | 22 +++++++++++++++---- Lib/test/test_htmlparser.py | 42 ++++++++++++++++++++++++++----------- Misc/NEWS | 7 +++++-- 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 0dc81a5b0b4..7c44bec73d8 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -115,7 +115,8 @@ An exception is defined as well: .. method:: HTMLParser.handle_data(data) - This method is called to process arbitrary data. It is intended to be + This method is called to process arbitrary data (e.g. the content of + ```` and ````). It is intended to be overridden by a derived class; the base class implementation does nothing. diff --git a/Lib/html/parser.py b/Lib/html/parser.py index affaf7344fa..afdb305d08c 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -62,6 +62,8 @@ locatestarttagend_tolerant = re.compile(r""" \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') +# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between +# ') @@ -121,6 +123,7 @@ class HTMLParser(_markupbase.ParserBase): self.rawdata = '' self.lasttag = '???' self.interesting = interesting_normal + self.cdata_elem = None _markupbase.ParserBase.reset(self) def feed(self, data): @@ -145,11 +148,13 @@ class HTMLParser(_markupbase.ParserBase): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self): + def set_cdata_mode(self, elem): self.interesting = interesting_cdata + self.cdata_elem = elem.lower() def clear_cdata_mode(self): self.interesting = interesting_normal + self.cdata_elem = None # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -314,7 +319,7 @@ class HTMLParser(_markupbase.ParserBase): else: self.handle_starttag(tag, attrs) if tag in self.CDATA_CONTENT_ELEMENTS: - self.set_cdata_mode() + self.set_cdata_mode(tag) return endpos # Internal -- check to see if we have a complete starttag; return end @@ -371,6 +376,9 @@ class HTMLParser(_markupbase.ParserBase): j = match.end() match = endtagfind.match(rawdata, i) # if not match: + if self.cdata_elem is not None: + self.handle_data(rawdata[i:j]) + return j if self.strict: self.error("bad end tag: %r" % (rawdata[i:j],)) k = rawdata.find('<', i + 1, j) @@ -380,8 +388,14 @@ class HTMLParser(_markupbase.ParserBase): j = i + 1 self.handle_data(rawdata[i:j]) return j - tag = match.group(1) - self.handle_endtag(tag.lower()) + + elem = match.group(1).lower() # script or style + if self.cdata_elem is not None: + if elem != self.cdata_elem: + self.handle_data(rawdata[i:j]) + return j + + self.handle_endtag(elem.lower()) self.clear_cdata_mode() return j diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 96644855131..fd8447c91d1 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -321,18 +321,36 @@ DOCTYPE html [ ("starttag_text", s)]) def test_cdata_content(self): - s = """""" - self._run_check(s, [ - ("starttag", "script", []), - ("data", " ¬-an-entity-ref; "), - ("endtag", "script"), - ]) - s = """""" - self._run_check(s, [ - ("starttag", "script", []), - ("data", " "), - ("endtag", "script"), - ]) + contents = [ + ' ¬-an-entity-ref;', + "", + '

', + 'foo = "";', + 'foo = "";', + 'foo = <\n/script> ', + '', + ('\n//<\\/s\'+\'cript>\');\n//]]>'), + '\n\n', + 'foo = "";', + '', + # these two should be invalid according to the HTML 5 spec, + # section 8.1.2.2 + #'foo = ', + #'foo = ', + ] + elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] + for content in contents: + for element in elements: + element_lower = element.lower() + s = '<{element}>{content}'.format(element=element, + content=content) + self._run_check(s, [("starttag", element_lower, []), + ("data", content), + ("endtag", element_lower)]) + def test_entityrefs_in_attributes(self): self._run_check("", [ diff --git a/Misc/NEWS b/Misc/NEWS index 63a3790183f..657c80290d1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,10 +66,13 @@ Core and Builtins Library ------- -- Issue 10817: Fix urlretrieve function to raise ContentTooShortError even +- Issue #670664: Fix HTMLParser to correctly handle the content of + ```` and ````. + +- Issue #10817: Fix urlretrieve function to raise ContentTooShortError even when reporthook is None. Patch by Jyrki Pulliainen. -- Issue 13296: Fix IDLE to clear compile __future__ flags on shell restart. +- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) - Issue #13293: Better error message when trying to marshal bytes using