diff --git a/changelog.d/20260731_163747_kurtmckee_rm_quadratic_behavior.rst b/changelog.d/20260731_163747_kurtmckee_rm_quadratic_behavior.rst new file mode 100644 index 0000000..a2cd390 --- /dev/null +++ b/changelog.d/20260731_163747_kurtmckee_rm_quadratic_behavior.rst @@ -0,0 +1,7 @@ +Fixed +----- + +* Fix quadratic search behavior in ``SGMLParser.feed()``. + + This can occur when a large document with an unfinished token + (like `` None: """Reset this instance. Loses all unprocessed data.""" self.__starttag_text: str | None = None - self.rawdata = "" + self._rawdata = "" + self._pending = [] + self._pending_content_length = 0 + self._retry_at = 0 self.stack: list[str] = [] self.lasttag = "???" self.nomoretags = 0 self.literal = 0 super().reset() + @property + def rawdata(self) -> str: + self._flush_pending() + return self._rawdata + + @rawdata.setter + def rawdata(self, value: str) -> None: + self._rawdata = value + self._pending = [] + self._pending_content_length = 0 + + def _flush_pending(self) -> None: + """Merge any chunks queued by `feed()` into `self._rawdata`.""" + + if self._pending: + self._rawdata += "".join(self._pending) + self._pending = [] + self._pending_content_length = 0 + def setnomoretags(self) -> None: """Enter literal mode (CDATA) till EOF. @@ -99,11 +138,20 @@ def feed(self, data: str) -> None: all the processing is done by goahead().) """ - self.rawdata = self.rawdata + data + self._pending.append(data) + self._pending_content_length += len(data) + if len(self._rawdata) + self._pending_content_length < self._retry_at: + # The last attempt to resolve the pending token failed. + # Skip re-scanning until enough new data has arrived + # to make another attempt worthwhile. + return + self._flush_pending() self.goahead(False) def close(self) -> None: """Handle the remaining data.""" + + self._flush_pending() self.goahead(True) def error(self, message: str) -> t.NoReturn: @@ -113,6 +161,7 @@ def error(self, message: str) -> t.NoReturn: # and data to be processed by a subsequent call. If 'end' is # true, force handling all data as if followed by EOF marker. def goahead(self, end: bool) -> None: + # Accessing `self.rawdata` flushes pending data. rawdata = self.rawdata i = 0 n = len(rawdata) @@ -225,6 +274,15 @@ def goahead(self, end: bool) -> None: self.handle_data(rawdata[i:n]) i = n self.rawdata = rawdata[i:] + if i == 0 and n > 0: + # No progress was made this call. + # Use exponential back off to set how much new data must arrive + # before the next scan retry. + # This ensures a persistently-incomplete token costs O(n) overall + # instead of O(n**2). + self._retry_at = n * 2 + else: + self._retry_at = 0 # XXX if end: check for empty stack # Extensions for the DOCTYPE scanner: diff --git a/tests/test_sgmllib.py b/tests/test_sgmllib.py index 8e327cf..0db0563 100644 --- a/tests/test_sgmllib.py +++ b/tests/test_sgmllib.py @@ -1,5 +1,6 @@ import io import pathlib +import time import pytest @@ -523,6 +524,62 @@ def test_endtag_with_no_valid_tag_name_does_not_close_open_tag(nested_tag_collec assert nested_tag_collector.stack == ["a"] +@pytest.mark.parametrize( + "prefix", + ( + pytest.param('