From 6f2bb98966853edcf1855b9dd310529d071241a2 Mon Sep 17 00:00:00 2001
From: Ezio Melotti <ezio.melotti@gmail.com>
Date: Sun, 6 Sep 2015 21:38:06 +0300
Subject: [PATCH] #23144: Make sure that HTMLParser.feed() returns all the
 data, even when convert_charrefs is True.

---
 Lib/html/parser.py          | 10 +++++++++-
 Lib/test/test_htmlparser.py | 15 ++++++++++++---
 Misc/NEWS                   |  5 ++++-
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/Lib/html/parser.py b/Lib/html/parser.py
index a650d5eeded..9ae31b9128d 100644
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -198,7 +198,15 @@ class HTMLParser(_markupbase.ParserBase):
             if self.convert_charrefs and not self.cdata_elem:
                 j = rawdata.find('<', i)
                 if j < 0:
-                    if not end:
+                    # if we can't find the next <, either we are at the end
+                    # or there's more text incoming.  If the latter is True,
+                    # we can't pass the text to handle_data in case we have
+                    # a charref cut in half at end.  Try to determine if
+                    # this is the case before proceding by looking for an
+                    # & near the end and see if it's followed by a space or ;.
+                    amppos = rawdata.rfind('&', max(i, n-34))
+                    if (amppos >= 0 and
+                        not re.compile(r'[\s;]').search(rawdata, amppos)):
                         break  # wait till we get all the text
                     j = n
             else:
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
index 2d771a2a974..144f820af29 100644
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -72,9 +72,6 @@ class EventCollectorExtra(EventCollector):
 
 class EventCollectorCharrefs(EventCollector):
 
-    def get_events(self):
-        return self.events
-
     def handle_charref(self, data):
         self.fail('This should never be called with convert_charrefs=True')
 
@@ -685,6 +682,18 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
         ]
         self._run_check(html, expected)
 
+    def test_convert_charrefs_dropped_text(self):
+        # #23144: make sure that all the events are triggered when
+        # convert_charrefs is True, even if we don't call .close()
+        parser = EventCollector(convert_charrefs=True)
+        # before the fix, bar & baz was missing
+        parser.feed("foo <a>link</a> bar &amp; baz")
+        self.assertEqual(
+            parser.get_events(),
+            [('data', 'foo '), ('starttag', 'a', []), ('data', 'link'),
+             ('endtag', 'a'), ('data', ' bar & baz')]
+        )
+
 
 class AttributesStrictTestCase(TestCaseBase):
 
diff --git a/Misc/NEWS b/Misc/NEWS
index dd175fead46..7a3c22c4832 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -1,4 +1,4 @@
-+++++++++++
++++++++++++
 Python News
 +++++++++++
 
@@ -81,6 +81,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #23144: Make sure that HTMLParser.feed() returns all the data, even
+  when convert_charrefs is True.
+
 - Issue #16180: Exit pdb if file has syntax error, instead of trapping user
   in an infinite loop.  Patch by Xavier de Gaye.