#13993: HTMLParser is now able to handle broken end tags when strict=False.
This commit is contained in:
parent
efc66f9e36
commit
5211ffe4df
|
@ -23,6 +23,9 @@ starttagopen = re.compile('<[a-zA-Z]')
|
|||
piclose = re.compile('>')
|
||||
commentclose = re.compile(r'--\s*>')
|
||||
tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*')
|
||||
# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
|
||||
# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
|
||||
tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*')
|
||||
# Note, the strict one of this pair isn't really strict, but we can't
|
||||
# make it correctly strict without breaking backward compatibility.
|
||||
attrfind = re.compile(
|
||||
|
@ -270,7 +273,7 @@ class HTMLParser(_markupbase.ParserBase):
|
|||
# see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
|
||||
def parse_bogus_comment(self, i, report=1):
|
||||
rawdata = self.rawdata
|
||||
if rawdata[i:i+2] != '<!':
|
||||
if rawdata[i:i+2] not in ('<!', '</'):
|
||||
self.error('unexpected call to parse_comment()')
|
||||
pos = rawdata.find('>', i+2)
|
||||
if pos == -1:
|
||||
|
@ -398,31 +401,40 @@ class HTMLParser(_markupbase.ParserBase):
|
|||
match = endendtag.search(rawdata, i+1) # >
|
||||
if not match:
|
||||
return -1
|
||||
j = match.end()
|
||||
gtpos = match.end()
|
||||
match = endtagfind.match(rawdata, i) # </ + tag + >
|
||||
if not match:
|
||||
if self.cdata_elem is not None:
|
||||
self.handle_data(rawdata[i:j])
|
||||
return j
|
||||
self.handle_data(rawdata[i:gtpos])
|
||||
return gtpos
|
||||
if self.strict:
|
||||
self.error("bad end tag: %r" % (rawdata[i:j],))
|
||||
k = rawdata.find('<', i + 1, j)
|
||||
if k > i:
|
||||
j = k
|
||||
if j <= i:
|
||||
j = i + 1
|
||||
self.handle_data(rawdata[i:j])
|
||||
return j
|
||||
self.error("bad end tag: %r" % (rawdata[i:gtpos],))
|
||||
# find the name: w3.org/TR/html5/tokenization.html#tag-name-state
|
||||
namematch = tagfind_tolerant.match(rawdata, i+2)
|
||||
if not namematch:
|
||||
# w3.org/TR/html5/tokenization.html#end-tag-open-state
|
||||
if rawdata[i:i+3] == '</>':
|
||||
return i+3
|
||||
else:
|
||||
return self.parse_bogus_comment(i)
|
||||
tagname = namematch.group().lower()
|
||||
# consume and ignore other stuff between the name and the >
|
||||
# Note: this is not 100% correct, since we might have things like
|
||||
# </tag attr=">">, but looking for > after tha name should cover
|
||||
# most of the cases and is much simpler
|
||||
gtpos = rawdata.find('>', namematch.end())
|
||||
self.handle_endtag(tagname)
|
||||
return gtpos+1
|
||||
|
||||
elem = match.group(1).lower() # script or style
|
||||
if self.cdata_elem is not None:
|
||||
if elem != self.cdata_elem:
|
||||
self.handle_data(rawdata[i:j])
|
||||
return j
|
||||
self.handle_data(rawdata[i:gtpos])
|
||||
return gtpos
|
||||
|
||||
self.handle_endtag(elem.lower())
|
||||
self.clear_cdata_mode()
|
||||
return j
|
||||
return gtpos
|
||||
|
||||
# Overridable -- finish processing of start+end tag: <tag.../>
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
|
|
|
@ -364,8 +364,9 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
|
|||
('data', '<<bc'),
|
||||
('endtag', 'a'),
|
||||
('endtag', 'html'),
|
||||
('data', '\n<img src="URL><//img></html'),
|
||||
('endtag', 'html')])
|
||||
('data', '\n<img src="URL>'),
|
||||
('comment', '/img'),
|
||||
('endtag', 'html<')])
|
||||
|
||||
def test_with_unquoted_attributes(self):
|
||||
# see #12008
|
||||
|
@ -403,6 +404,44 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
|
|||
('starttag', 'form',
|
||||
[('action', 'bogus|&#()value')])])
|
||||
|
||||
def test_invalid_end_tags(self):
|
||||
# A collection of broken end tags. <br> is used as separator.
|
||||
# see http://www.w3.org/TR/html5/tokenization.html#end-tag-open-state
|
||||
# and #13993
|
||||
html = ('<br></label</p><br></div end tmAd-leaderBoard><br></<h4><br>'
|
||||
'</li class="unit"><br></li\r\n\t\t\t\t\t\t</ul><br></><br>')
|
||||
expected = [('starttag', 'br', []),
|
||||
# < is part of the name, / is discarded, p is an attribute
|
||||
('endtag', 'label<'),
|
||||
('starttag', 'br', []),
|
||||
# text and attributes are discarded
|
||||
('endtag', 'div'),
|
||||
('starttag', 'br', []),
|
||||
# comment because the first char after </ is not a-zA-Z
|
||||
('comment', '<h4'),
|
||||
('starttag', 'br', []),
|
||||
# attributes are discarded
|
||||
('endtag', 'li'),
|
||||
('starttag', 'br', []),
|
||||
# everything till ul (included) is discarded
|
||||
('endtag', 'li'),
|
||||
('starttag', 'br', []),
|
||||
# </> is ignored
|
||||
('starttag', 'br', [])]
|
||||
self._run_check(html, expected)
|
||||
|
||||
def test_broken_invalid_end_tag(self):
|
||||
# This is technically wrong (the "> shouldn't be included in the 'data')
|
||||
# but is probably not worth fixing it (in addition to all the cases of
|
||||
# the previous test, it would require a full attribute parsing).
|
||||
# see #13993
|
||||
html = '<b>This</b attr=">"> confuses the parser'
|
||||
expected = [('starttag', 'b', []),
|
||||
('data', 'This'),
|
||||
('endtag', 'b'),
|
||||
('data', '"> confuses the parser')]
|
||||
self._run_check(html, expected)
|
||||
|
||||
def test_correct_detection_of_start_tags(self):
|
||||
# see #13273
|
||||
html = ('<div style="" ><b>The <a href="some_url">rain</a> '
|
||||
|
|
|
@ -113,6 +113,9 @@ Core and Builtins
|
|||
Library
|
||||
-------
|
||||
|
||||
- Issue #13993: HTMLParser is now able to handle broken end tags when
|
||||
strict=False.
|
||||
|
||||
- Issue #9750: Fix sqlite3.Connection.iterdump on tables and fields
|
||||
with a name that is a keyword or contains quotes. Patch by Marko
|
||||
Kohtala.
|
||||
|
|
Loading…
Reference in New Issue