From f329a8336ac46293ac3ca1b10b407c00270f5cbf Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Tue, 11 Aug 2026 07:49:05 -0500 Subject: [PATCH 01/13] Remove the `verbose` init parameter and its only usage site --- changelog.d/20260811_074802_kurtmckee_cleanups.rst | 4 ++++ src/feedparser_sgmllib/__init__.py | 8 ++------ 2 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20260811_074802_kurtmckee_cleanups.rst diff --git a/changelog.d/20260811_074802_kurtmckee_cleanups.rst b/changelog.d/20260811_074802_kurtmckee_cleanups.rst new file mode 100644 index 0000000..aafaf7d --- /dev/null +++ b/changelog.d/20260811_074802_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Removed +------- + +* Remove the ``verbose`` init parameter and its only usage site. diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index 96c0d05..aaf54e5 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -64,9 +64,8 @@ class SGMLParser(_markupbase.ParserBase): # Definition of entities -- derived classes may override entity_or_charref = re.compile("&(?:([a-zA-Z][-.a-zA-Z0-9]*)|#([0-9]+))(;?)") - def __init__(self, verbose: bool = False) -> None: + def __init__(self) -> None: """Initialize and reset this instance.""" - self.verbose = verbose # These variables help guard against quadratic CPU usage behavior # without significantly increasing memory usage. @@ -462,11 +461,8 @@ def handle_starttag( def handle_endtag(self, tag: str, method: t.Callable[[], t.Any]) -> None: method() - # Example -- report an unbalanced tag. def report_unbalanced(self, tag: str) -> None: - if self.verbose: - print("*** Unbalanced ") - print("*** Stack:", self.stack) + pass def convert_charref(self, name: str) -> str | None: """Convert character reference, may be overridden.""" From 2ce8d0c2a3a6f6e41865b3a19f5feeb77e1ce28c Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Tue, 11 Aug 2026 07:51:28 -0500 Subject: [PATCH 02/13] Fix the `convert_charref()` `ref` parameter range constraint --- changelog.d/20260811_075136_kurtmckee_cleanups.rst | 4 ++++ docs/sgmllib.rst | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20260811_075136_kurtmckee_cleanups.rst diff --git a/changelog.d/20260811_075136_kurtmckee_cleanups.rst b/changelog.d/20260811_075136_kurtmckee_cleanups.rst new file mode 100644 index 0000000..e2ff7fd --- /dev/null +++ b/changelog.d/20260811_075136_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Documentation +------------- + +* Fix the ``convert_charref()`` ``ref`` parameter range constraint. diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index d386412..bae40ad 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -130,7 +130,7 @@ A single exception is defined as well: Convert a character reference to a string, or ``None``. *ref* is the reference passed in as a string. In the base implementation, *ref* must be a decimal - number in the range 0--255. It converts the code point found using the + number in the range 0--127. It converts the code point found using the :meth:`convert_codepoint` method. If *ref* is invalid or out of range, this method returns ``None``. This method is called by the default :meth:`handle_charref` implementation and by the attribute value parser. From 0eabf90a2b38ae6abaefab5ee152654eecc0db41 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Tue, 11 Aug 2026 08:38:13 -0500 Subject: [PATCH 03/13] Clarify when the `report_unbalanced` method will be called --- changelog.d/20260811_083432_kurtmckee_cleanups.rst | 6 ++++++ docs/sgmllib.rst | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20260811_083432_kurtmckee_cleanups.rst diff --git a/changelog.d/20260811_083432_kurtmckee_cleanups.rst b/changelog.d/20260811_083432_kurtmckee_cleanups.rst new file mode 100644 index 0000000..7dda243 --- /dev/null +++ b/changelog.d/20260811_083432_kurtmckee_cleanups.rst @@ -0,0 +1,6 @@ +Documentation +------------- + +* Clarify when the ``report_unbalanced`` method will be called. + + Note that feedparser does not currently override this method. diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index bae40ad..37ed69c 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -185,7 +185,19 @@ A single exception is defined as well: .. method:: SGMLParser.report_unbalanced(tag) This method is called when an end tag is found which does not correspond to any - open element. + open element and has an ``end_{tag}()`` method defined. + + For example, given this input text: + + .. code-block:: xml + + + + then if the parser has an ``end_i()`` method defined, + ``report_unbalanced("i")`` will be called. + + Note that the ``end_{tag}()`` method is not called in this situation; + only ``report_unbalanced(tag)`` will be called. .. method:: SGMLParser.unknown_starttag(tag, attributes) From 88815533a45701b83198dae4d4eab48b3fa25c32 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Tue, 11 Aug 2026 08:40:06 -0500 Subject: [PATCH 04/13] Initialize instance variables exclusively in `__init__()` --- src/feedparser_sgmllib/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index aaf54e5..2ea2266 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -88,18 +88,25 @@ def __init__(self) -> None: self._pending = StringIO() self._retry_at = 0 - self.reset() + self.__starttag_text: str | None = None + self.stack: list[str] = [] + self.lasttag = "???" + self.nomoretags = 0 + self.literal = 0 def reset(self) -> None: """Reset this instance. Loses all unprocessed data.""" - self.__starttag_text: str | None = None + self._rawdata = "" self._pending = StringIO() self._retry_at = 0 - self.stack: list[str] = [] + + self.__starttag_text = None + self.stack = [] self.lasttag = "???" self.nomoretags = 0 self.literal = 0 + super().reset() @property From 5e20d081238e6b1cc4624386e5916679cb16c296 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Tue, 11 Aug 2026 09:10:55 -0500 Subject: [PATCH 05/13] Document the `unknown_decl` method --- .../20260811_090639_kurtmckee_cleanups.rst | 9 ++++++++ docs/conf.py | 1 + docs/sgmllib.rst | 3 +++ src/feedparser_sgmllib/__init__.py | 22 +++++++++++++++++++ tox.ini | 1 - 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20260811_090639_kurtmckee_cleanups.rst diff --git a/changelog.d/20260811_090639_kurtmckee_cleanups.rst b/changelog.d/20260811_090639_kurtmckee_cleanups.rst new file mode 100644 index 0000000..1f931d3 --- /dev/null +++ b/changelog.d/20260811_090639_kurtmckee_cleanups.rst @@ -0,0 +1,9 @@ +Documentation +------------- + +* Document the ``unknown_decl`` method. + + It exists and is called by the base implementation in ``_markupbase`` + but was not previously documented here. + + This method is not currently overridden by feedparser. diff --git a/docs/conf.py b/docs/conf.py index f264837..d664895 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,6 +23,7 @@ pygments_style = "sphinx" extensions = [ + "sphinx.ext.autodoc", "sphinx_issues", ] diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index 37ed69c..e5b8608 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -220,6 +220,9 @@ A single exception is defined as well: nothing. +.. automethod:: SGMLParser.unknown_decl + + .. method:: SGMLParser.unknown_entityref(ref) This method is called to process an unknown entity reference. It is intended to diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index 2ea2266..c4d4a56 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -527,6 +527,28 @@ def handle_comment(self, data: str) -> t.Any: def handle_decl(self, decl: t.Any) -> t.Any: pass + def unknown_decl(self, data: str) -> t.Any: + """ + Handle a non-DOCTYPE declaration. + + Given an input text like this: + + .. code-block:: xml + + + + + This method would be called only for the ```` declaration: + + .. code-block:: python + + self.unknown_decl("SOMETHING bogus") + + The default implementation does nothing. + """ + + pass + # Example -- handle processing instruction, could be overridden def handle_pi(self, data: str) -> t.Any: pass diff --git a/tox.ini b/tox.ini index 148be05..ac16476 100644 --- a/tox.ini +++ b/tox.ini @@ -60,7 +60,6 @@ commands = [testenv:docs] description = Test building the documentation base_python = py3.14 -skip_install = true deps = -r requirements/docs/requirements.txt commands = sphinx-build -aWEnb html docs/ build/docs/ From fb522ecda4a354c258d5e8f0d7a11579e09eff42 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Wed, 12 Aug 2026 10:50:59 -0500 Subject: [PATCH 06/13] Document more SGML constructs; document the `handle_pi` method --- .../20260811_091912_kurtmckee_cleanups.rst | 5 +++++ docs/sgmllib.rst | 14 ++++++++++++- src/feedparser_sgmllib/__init__.py | 20 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20260811_091912_kurtmckee_cleanups.rst diff --git a/changelog.d/20260811_091912_kurtmckee_cleanups.rst b/changelog.d/20260811_091912_kurtmckee_cleanups.rst new file mode 100644 index 0000000..a0c8d69 --- /dev/null +++ b/changelog.d/20260811_091912_kurtmckee_cleanups.rst @@ -0,0 +1,5 @@ +Documentation +------------- + +* Expand and clarify the SGML constructs that are handled. +* Document the ``handle_pi`` method. diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index e5b8608..46de1e2 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -20,7 +20,7 @@ as it is used by XML feeds and HTML. * Opening and closing tags of the form ```` and ````, respectively. - * Numeric character references of the form ``&#name;``. + * Numeric character references of the form ``&#ref;``. * Entity references of the form ``&name;``. @@ -28,6 +28,15 @@ as it is used by XML feeds and HTML. newlines are allowed between the trailing ``>`` and the immediately preceding ``--``. + * Processing instructions of the form ````. + + These are handled by :meth:`handle_pi`. + + * Declarations of the form ````. + + ```` declarations are handled by :meth:`handle_decl`. + All other declarations are handled by :meth:`unknown_decl`. + A single exception is defined as well: @@ -182,6 +191,9 @@ A single exception is defined as well: default implementation does nothing. +.. automethod:: SGMLParser.handle_pi + + .. method:: SGMLParser.report_unbalanced(tag) This method is called when an end tag is found which does not correspond to any diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index c4d4a56..201cb9f 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -551,6 +551,26 @@ def unknown_decl(self, data: str) -> t.Any: # Example -- handle processing instruction, could be overridden def handle_pi(self, data: str) -> t.Any: + """ + Handle processing instructions. + + Given an input text like this: + + .. code-block:: xml + + + + This method will be called like: + + .. code-block:: python + + self.handle_pi('xml version="1.0" encoding="UTF-8" ?') + + Note that the closing question mark is not stripped. + + The default implementation does nothing. + """ + pass # To be overridden -- handlers for unknown objects From fa60cf56cebec609a27254abff82ea53d8dec214 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Wed, 12 Aug 2026 10:52:57 -0500 Subject: [PATCH 07/13] Fix an `Any` type annotation, which is always a `str` instance --- src/feedparser_sgmllib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index 201cb9f..8ba86e2 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -524,7 +524,7 @@ def handle_comment(self, data: str) -> t.Any: pass # Example -- handle declaration, could be overridden - def handle_decl(self, decl: t.Any) -> t.Any: + def handle_decl(self, decl: str) -> t.Any: pass def unknown_decl(self, data: str) -> t.Any: From edbc2a74bb69e73a3a019db18e496a9439098e61 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Wed, 12 Aug 2026 11:00:00 -0500 Subject: [PATCH 08/13] Clarify that `super().close()`, not `self.close()`, must be called --- docs/sgmllib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index 46de1e2..fc8da37 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -78,7 +78,7 @@ A single exception is defined as well: Force processing of all buffered data as if it were followed by an end-of-file mark. This method may be redefined by a derived class to define additional processing at the end of the input, but the redefined version should always call - :meth:`close`. + ``super().close()``. .. method:: SGMLParser.get_starttag_text() From d5400911a67ae55cbc76d14164b3f18260836f33 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Thu, 13 Aug 2026 07:57:33 -0500 Subject: [PATCH 09/13] Fix inaccuracies in the `handle_starttag` documentation --- .../20260813_075633_kurtmckee_cleanups.rst | 4 +++ docs/sgmllib.rst | 26 +++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20260813_075633_kurtmckee_cleanups.rst diff --git a/changelog.d/20260813_075633_kurtmckee_cleanups.rst b/changelog.d/20260813_075633_kurtmckee_cleanups.rst new file mode 100644 index 0000000..e78507c --- /dev/null +++ b/changelog.d/20260813_075633_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Documentation +------------- + +* Fix inaccuracies in the ``handle_starttag`` documentation. diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index fc8da37..e0179e3 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -98,17 +98,27 @@ A single exception is defined as well: *attributes* argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. - The *name* has been translated to lower case. Double quotes and backslashes in - the *value* have been interpreted, as well as known character references and - known entity references terminated by a semicolon (normally, entity references - can be terminated by any non-alphanumerical character, but this would break the + The *name* has been translated to lower case. + Known character references and known entity references terminated by a semicolon + have been converted in the *value* (normally, entity references can be terminated + by any non-alphanumerical character, but this would break the very common case of ```` when ``eggs`` is a valid entity name). - For instance, for the tag ````, this method would - be called as ``unknown_starttag('a', [('href', 'http://www.cwi.nl/')])``. The - base implementation simply calls *method* with *attributes* as the only - argument. + For instance, given this input text: + + .. code-block:: xml + + + + and given that a subclass has defined a ``start_a`` method, + ``handle_starttag`` would be called with these arguments: + + .. code-block:: python + + self.handle_starttag("a", self.start_a, [("href", "https://domain.example/")]) + + The base implementation simply calls *method* with *attributes* as the only argument. .. method:: SGMLParser.handle_endtag(tag, method) From 9d9d58ca0a7701b2ae52dd73014c89d6e8432934 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Thu, 13 Aug 2026 07:59:06 -0500 Subject: [PATCH 10/13] Clarify when `handle_decl` is called and provide an example --- .../20260813_075745_kurtmckee_cleanups.rst | 4 ++++ docs/sgmllib.rst | 24 ++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20260813_075745_kurtmckee_cleanups.rst diff --git a/changelog.d/20260813_075745_kurtmckee_cleanups.rst b/changelog.d/20260813_075745_kurtmckee_cleanups.rst new file mode 100644 index 0000000..73f74ee --- /dev/null +++ b/changelog.d/20260813_075745_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Documentation +------------- + +* Clarify when ``handle_decl`` is called and provide an example. diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index e0179e3..7d0f056 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -191,14 +191,22 @@ A single exception is defined as well: does nothing. -.. method:: SGMLParser.handle_decl(data) - - Method called when an SGML declaration is read by the parser. In practice, the - ``DOCTYPE`` declaration is the only thing observed in HTML, but the parser does - not discriminate among different (or broken) declarations. Internal subsets in - a ``DOCTYPE`` declaration are not supported. The *data* parameter will be the - entire contents of the declaration inside the ```` markup. The - default implementation does nothing. +.. method:: SGMLParser.handle_decl(data) + + Method called when a ``DOCTYPE`` declaration is read by the parser. + + Internal subsets in a ``DOCTYPE`` declaration are not supported. + The *data* parameter will be the entire contents of the declaration + inside the ```` markup. + For example, given this input text: + + .. code-block:: xml + + + + Then this method will be called with the text ``"DOCTYPE html"``. + + The default implementation does nothing. .. automethod:: SGMLParser.handle_pi From b00a9f1623573d8cedfeec4c098c6579c58b09ce Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Thu, 13 Aug 2026 08:23:04 -0500 Subject: [PATCH 11/13] Enable Sphinx nitpicky mode in `conf.py` --- docs/conf.py | 3 +++ tox.ini | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index d664895..5a3553f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,6 +22,9 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +# Enable nitpicky mode. +nitpicky = True + extensions = [ "sphinx.ext.autodoc", "sphinx_issues", diff --git a/tox.ini b/tox.ini index ac16476..4304a82 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,7 @@ description = Test building the documentation base_python = py3.14 deps = -r requirements/docs/requirements.txt commands = - sphinx-build -aWEnb html docs/ build/docs/ + sphinx-build -aWEb html docs/ build/docs/ [testenv:build{, -minimum}] description = From 450473ad8314f5ffb51bdf2431136da46d3f6e77 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Thu, 13 Aug 2026 08:42:23 -0500 Subject: [PATCH 12/13] Ensure that critical instance variables exist --- src/feedparser_sgmllib/__init__.py | 7 +++++-- tests/test_sgmllib.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index 8ba86e2..eab8b70 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -67,6 +67,9 @@ class SGMLParser(_markupbase.ParserBase): def __init__(self) -> None: """Initialize and reset this instance.""" + super().__init__() + super().reset() + # These variables help guard against quadratic CPU usage behavior # without significantly increasing memory usage. # @@ -97,6 +100,8 @@ def __init__(self) -> None: def reset(self) -> None: """Reset this instance. Loses all unprocessed data.""" + super().reset() + self._rawdata = "" self._pending = StringIO() self._retry_at = 0 @@ -107,8 +112,6 @@ def reset(self) -> None: self.nomoretags = 0 self.literal = 0 - super().reset() - @property def rawdata(self) -> str: self._flush_pending() diff --git a/tests/test_sgmllib.py b/tests/test_sgmllib.py index 348903a..eb7b335 100644 --- a/tests/test_sgmllib.py +++ b/tests/test_sgmllib.py @@ -641,6 +641,19 @@ def test_feed_unresolved_token_peak_memory_scales_nicely(): ) +def test_init_and_reset_functionality(): + """ + Verify that init and reset functionality work as intended. + + The `_markupbase.ParserBase.__init__()` method doesn't set any instance variables. + It's incumbent on `SGMLParser` to do extra work to ensure these are set. + The way by which this happens is unimportant; it's only important that it happens. + """ + + instance = sgmllib.SGMLParser() + assert instance.getpos() == (1, 0) + + # XXX These tests have been disabled by prefixing their names with # an underscore. The first two exercise outstanding bugs in the # sgmllib module, and the third exhibits questionable behavior From 60abc18db697709c6769e268f537ff4844e29b3f Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Thu, 13 Aug 2026 09:53:01 -0500 Subject: [PATCH 13/13] Ensure Read the Docs installs the project for autodoc use --- .readthedocs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dbb18e0..575ac65 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,3 +13,5 @@ sphinx: python: install: - requirements: 'requirements/docs/requirements.txt' + - method: 'pip' + path: '.'