Fix quadratic search behavior

changelog.d/20260731_163747_kurtmckee_rm_quadratic_behavior.rst

@@ -0,0 +1,7 @@
+Fixed
+-----
+
+*   Fix quadratic search behavior in ``SGMLParser.feed()``.
+
+    This can occur when a large document with an unfinished token
+    (like ``<a href="...``) is fed into the parser in very small increments.

src/feedparser_sgmllib/__init__.py

@@ -65,18 +65,57 @@ class SGMLParser(_markupbase.ParserBase):
     def __init__(self, verbose: bool = False) -> None:
         """Initialize and reset this instance."""
         self.verbose = verbose
+
+        # These variables help guard against quadratic search behavior.
+        # Previously, each call to `.feed()` resulted in string-concatenation
+        # and a regular expression matching from the start of the raw data.
+        # For large unclosed tokens (like `<a href="...`) fed in small chunks,
+        # this could become quadratically expensive.
+        #
+        # This quadratic behavior is addressed by:
+        #
+        # * Gating failing calls to `.goahead()` behind an exponential backoff
+        # * Avoiding string concatenation until it's needed
+        #
+        self._rawdata = ""
+        self._pending: list[str] = []
+        self._pending_content_length = 0
+        self._retry_at = 0
+
         self.reset()
 
     def reset(self) -> None:
         """Reset this instance. Loses all unprocessed data."""
         self.__starttag_text: str | None = None
-        self.rawdata = ""
+        self._rawdata = ""
+        self._pending = []
+        self._pending_content_length = 0
+        self._retry_at = 0
         self.stack: list[str] = []
         self.lasttag = "???"
         self.nomoretags = 0
         self.literal = 0
         super().reset()
 
+    @property
+    def rawdata(self) -> str:
+        self._flush_pending()
+        return self._rawdata
+
+    @rawdata.setter
+    def rawdata(self, value: str) -> None:
+        self._rawdata = value
+        self._pending = []
+        self._pending_content_length = 0
+
+    def _flush_pending(self) -> None:
+        """Merge any chunks queued by `feed()` into `self._rawdata`."""
+
+        if self._pending:
+            self._rawdata += "".join(self._pending)
+            self._pending = []
+            self._pending_content_length = 0
+
     def setnomoretags(self) -> None:
         """Enter literal mode (CDATA) till EOF.
 
@@ -99,11 +138,20 @@ def feed(self, data: str) -> None:
         all the processing is done by goahead().)
         """
 
-        self.rawdata = self.rawdata + data
+        self._pending.append(data)
+        self._pending_content_length += len(data)
+        if len(self._rawdata) + self._pending_content_length < self._retry_at:
+            # The last attempt to resolve the pending token failed.
+            # Skip re-scanning until enough new data has arrived
+            # to make another attempt worthwhile.
+            return
+        self._flush_pending()
         self.goahead(False)
 
     def close(self) -> None:
         """Handle the remaining data."""
+
+        self._flush_pending()
         self.goahead(True)
 
     def error(self, message: str) -> t.NoReturn:
@@ -113,6 +161,7 @@ def error(self, message: str) -> t.NoReturn:
     # and data to be processed by a subsequent call.  If 'end' is
     # true, force handling all data as if followed by EOF marker.
     def goahead(self, end: bool) -> None:
+        # Accessing `self.rawdata` flushes pending data.
         rawdata = self.rawdata
         i = 0
         n = len(rawdata)
@@ -225,6 +274,15 @@ def goahead(self, end: bool) -> None:
             self.handle_data(rawdata[i:n])
             i = n
         self.rawdata = rawdata[i:]
+        if i == 0 and n > 0:
+            # No progress was made this call.
+            # Use exponential back off to set how much new data must arrive
+            # before the next scan retry.
+            # This ensures a persistently-incomplete token costs O(n) overall
+            # instead of O(n**2).
+            self._retry_at = n * 2
+        else:
+            self._retry_at = 0
         # XXX if end: check for empty stack
 
     # Extensions for the DOCTYPE scanner:

tests/test_sgmllib.py

@@ -1,5 +1,6 @@
 import io
 import pathlib
+import time
 
 import pytest
 
@@ -523,6 +524,62 @@ def test_endtag_with_no_valid_tag_name_does_not_close_open_tag(nested_tag_collec
     assert nested_tag_collector.stack == ["a"]
 
 
+@pytest.mark.parametrize(
+    "prefix",
+    (
+        pytest.param('<a href="', id="starttag-unterminated-attribute"),
+        pytest.param("</", id="endtag-missing-close-bracket"),
+        pytest.param("<?", id="processing-instruction-missing-close"),
+        pytest.param("<!--", id="comment-missing-close"),
+        pytest.param("<!DOCTYPE ", id="declaration-missing-close"),
+        pytest.param("<![CDATA[", id="marked-section-missing-close"),
+    ),
+)
+def test_feed_unresolved_token_scales_linearly(prefix):
+    """
+    Verify that `.feed()` doesn't exhibit quadratic search behavior.
+
+    This is verified by creating a very large, incomplete token
+    (like an unclosed start tag) and feeding it to the parser
+    one byte at a time. The amount of CPU time required for this
+    suggests whether the parser has redeveloped quadratic behavior.
+
+    The first and last quarters of the data feed times are measured
+    because linear behavior should keep their ratio close to 1.
+    Having two quarters' worth of data between the measured quarters
+    helps make quadratic behavior much more noticeable.
+    """
+
+    n = 60_000
+    length = n - len(prefix)
+    payload = prefix + ("x" * length)
+    parser = sgmllib.SGMLParser()
+
+    quarter = n // 4
+    checkpoints = []
+    start = time.process_time()
+    for i, character in enumerate(payload, start=1):
+        parser.feed(character)
+        # At the end of each content quarter,
+        # capture the processing time and restart the timer.
+        if i % quarter == 0:
+            checkpoints.append(time.process_time() - start)
+            start = time.process_time()
+
+    assert len(checkpoints) == 4
+    first_quarter, last_quarter = checkpoints[0], checkpoints[-1]
+
+    max_allowed_growth = 2.5
+    # The numbers are scaled up to try to avoid close-to-zero float math problems.
+    budget = (first_quarter * 10_000) * max_allowed_growth
+    assert last_quarter * 10_000 <= budget, (
+        f"The cost of the last quarter of feed() calls ({last_quarter:.3f}s) "
+        f"exceeded {max_allowed_growth}x the cost of the first quarter "
+        f"({first_quarter:.3f}s) for prefix {prefix!r}. "
+        f"This suggests quadratic behavior."
+    )
+
+
 # XXX These tests have been disabled by prefixing their names with
 # an underscore.  The first two exercise outstanding bugs in the
 # sgmllib module, and the third exhibits questionable behavior