diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dbb18e0..575ac65 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,3 +13,5 @@ sphinx: python: install: - requirements: 'requirements/docs/requirements.txt' + - method: 'pip' + path: '.' diff --git a/changelog.d/20260811_074802_kurtmckee_cleanups.rst b/changelog.d/20260811_074802_kurtmckee_cleanups.rst new file mode 100644 index 0000000..aafaf7d --- /dev/null +++ b/changelog.d/20260811_074802_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Removed +------- + +* Remove the ``verbose`` init parameter and its only usage site. diff --git a/changelog.d/20260811_075136_kurtmckee_cleanups.rst b/changelog.d/20260811_075136_kurtmckee_cleanups.rst new file mode 100644 index 0000000..e2ff7fd --- /dev/null +++ b/changelog.d/20260811_075136_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Documentation +------------- + +* Fix the ``convert_charref()`` ``ref`` parameter range constraint. diff --git a/changelog.d/20260811_083432_kurtmckee_cleanups.rst b/changelog.d/20260811_083432_kurtmckee_cleanups.rst new file mode 100644 index 0000000..7dda243 --- /dev/null +++ b/changelog.d/20260811_083432_kurtmckee_cleanups.rst @@ -0,0 +1,6 @@ +Documentation +------------- + +* Clarify when the ``report_unbalanced`` method will be called. + + Note that feedparser does not currently override this method. diff --git a/changelog.d/20260811_090639_kurtmckee_cleanups.rst b/changelog.d/20260811_090639_kurtmckee_cleanups.rst new file mode 100644 index 0000000..1f931d3 --- /dev/null +++ b/changelog.d/20260811_090639_kurtmckee_cleanups.rst @@ -0,0 +1,9 @@ +Documentation +------------- + +* Document the ``unknown_decl`` method. + + It exists and is called by the base implementation in ``_markupbase`` + but was not previously documented here. + + This method is not currently overridden by feedparser. diff --git a/changelog.d/20260811_091912_kurtmckee_cleanups.rst b/changelog.d/20260811_091912_kurtmckee_cleanups.rst new file mode 100644 index 0000000..a0c8d69 --- /dev/null +++ b/changelog.d/20260811_091912_kurtmckee_cleanups.rst @@ -0,0 +1,5 @@ +Documentation +------------- + +* Expand and clarify the SGML constructs that are handled. +* Document the ``handle_pi`` method. diff --git a/changelog.d/20260813_075633_kurtmckee_cleanups.rst b/changelog.d/20260813_075633_kurtmckee_cleanups.rst new file mode 100644 index 0000000..e78507c --- /dev/null +++ b/changelog.d/20260813_075633_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Documentation +------------- + +* Fix inaccuracies in the ``handle_starttag`` documentation. diff --git a/changelog.d/20260813_075745_kurtmckee_cleanups.rst b/changelog.d/20260813_075745_kurtmckee_cleanups.rst new file mode 100644 index 0000000..73f74ee --- /dev/null +++ b/changelog.d/20260813_075745_kurtmckee_cleanups.rst @@ -0,0 +1,4 @@ +Documentation +------------- + +* Clarify when ``handle_decl`` is called and provide an example. diff --git a/docs/conf.py b/docs/conf.py index f264837..5a3553f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,11 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +# Enable nitpicky mode. +nitpicky = True + extensions = [ + "sphinx.ext.autodoc", "sphinx_issues", ] diff --git a/docs/sgmllib.rst b/docs/sgmllib.rst index d386412..7d0f056 100644 --- a/docs/sgmllib.rst +++ b/docs/sgmllib.rst @@ -20,7 +20,7 @@ as it is used by XML feeds and HTML. * Opening and closing tags of the form ```` and ````, respectively. - * Numeric character references of the form ``&#name;``. + * Numeric character references of the form ``&#ref;``. * Entity references of the form ``&name;``. @@ -28,6 +28,15 @@ as it is used by XML feeds and HTML. newlines are allowed between the trailing ``>`` and the immediately preceding ``--``. + * Processing instructions of the form ````. + + These are handled by :meth:`handle_pi`. + + * Declarations of the form ````. + + ```` declarations are handled by :meth:`handle_decl`. + All other declarations are handled by :meth:`unknown_decl`. + A single exception is defined as well: @@ -69,7 +78,7 @@ A single exception is defined as well: Force processing of all buffered data as if it were followed by an end-of-file mark. This method may be redefined by a derived class to define additional processing at the end of the input, but the redefined version should always call - :meth:`close`. + ``super().close()``. .. method:: SGMLParser.get_starttag_text() @@ -89,17 +98,27 @@ A single exception is defined as well: *attributes* argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. - The *name* has been translated to lower case. Double quotes and backslashes in - the *value* have been interpreted, as well as known character references and - known entity references terminated by a semicolon (normally, entity references - can be terminated by any non-alphanumerical character, but this would break the + The *name* has been translated to lower case. + Known character references and known entity references terminated by a semicolon + have been converted in the *value* (normally, entity references can be terminated + by any non-alphanumerical character, but this would break the very common case of ```` when ``eggs`` is a valid entity name). - For instance, for the tag ````, this method would - be called as ``unknown_starttag('a', [('href', 'http://www.cwi.nl/')])``. The - base implementation simply calls *method* with *attributes* as the only - argument. + For instance, given this input text: + + .. code-block:: xml + + + + and given that a subclass has defined a ``start_a`` method, + ``handle_starttag`` would be called with these arguments: + + .. code-block:: python + + self.handle_starttag("a", self.start_a, [("href", "https://domain.example/")]) + + The base implementation simply calls *method* with *attributes* as the only argument. .. method:: SGMLParser.handle_endtag(tag, method) @@ -130,7 +149,7 @@ A single exception is defined as well: Convert a character reference to a string, or ``None``. *ref* is the reference passed in as a string. In the base implementation, *ref* must be a decimal - number in the range 0--255. It converts the code point found using the + number in the range 0--127. It converts the code point found using the :meth:`convert_codepoint` method. If *ref* is invalid or out of range, this method returns ``None``. This method is called by the default :meth:`handle_charref` implementation and by the attribute value parser. @@ -172,20 +191,43 @@ A single exception is defined as well: does nothing. -.. method:: SGMLParser.handle_decl(data) +.. method:: SGMLParser.handle_decl(data) + + Method called when a ``DOCTYPE`` declaration is read by the parser. + + Internal subsets in a ``DOCTYPE`` declaration are not supported. + The *data* parameter will be the entire contents of the declaration + inside the ```` markup. + For example, given this input text: - Method called when an SGML declaration is read by the parser. In practice, the - ``DOCTYPE`` declaration is the only thing observed in HTML, but the parser does - not discriminate among different (or broken) declarations. Internal subsets in - a ``DOCTYPE`` declaration are not supported. The *data* parameter will be the - entire contents of the declaration inside the ```` markup. The - default implementation does nothing. + .. code-block:: xml + + + + Then this method will be called with the text ``"DOCTYPE html"``. + + The default implementation does nothing. + + +.. automethod:: SGMLParser.handle_pi .. method:: SGMLParser.report_unbalanced(tag) This method is called when an end tag is found which does not correspond to any - open element. + open element and has an ``end_{tag}()`` method defined. + + For example, given this input text: + + .. code-block:: xml + + + + then if the parser has an ``end_i()`` method defined, + ``report_unbalanced("i")`` will be called. + + Note that the ``end_{tag}()`` method is not called in this situation; + only ``report_unbalanced(tag)`` will be called. .. method:: SGMLParser.unknown_starttag(tag, attributes) @@ -208,6 +250,9 @@ A single exception is defined as well: nothing. +.. automethod:: SGMLParser.unknown_decl + + .. method:: SGMLParser.unknown_entityref(ref) This method is called to process an unknown entity reference. It is intended to diff --git a/src/feedparser_sgmllib/__init__.py b/src/feedparser_sgmllib/__init__.py index 96c0d05..eab8b70 100644 --- a/src/feedparser_sgmllib/__init__.py +++ b/src/feedparser_sgmllib/__init__.py @@ -64,9 +64,11 @@ class SGMLParser(_markupbase.ParserBase): # Definition of entities -- derived classes may override entity_or_charref = re.compile("&(?:([a-zA-Z][-.a-zA-Z0-9]*)|#([0-9]+))(;?)") - def __init__(self, verbose: bool = False) -> None: + def __init__(self) -> None: """Initialize and reset this instance.""" - self.verbose = verbose + + super().__init__() + super().reset() # These variables help guard against quadratic CPU usage behavior # without significantly increasing memory usage. @@ -89,19 +91,26 @@ def __init__(self, verbose: bool = False) -> None: self._pending = StringIO() self._retry_at = 0 - self.reset() + self.__starttag_text: str | None = None + self.stack: list[str] = [] + self.lasttag = "???" + self.nomoretags = 0 + self.literal = 0 def reset(self) -> None: """Reset this instance. Loses all unprocessed data.""" - self.__starttag_text: str | None = None + + super().reset() + self._rawdata = "" self._pending = StringIO() self._retry_at = 0 - self.stack: list[str] = [] + + self.__starttag_text = None + self.stack = [] self.lasttag = "???" self.nomoretags = 0 self.literal = 0 - super().reset() @property def rawdata(self) -> str: @@ -462,11 +471,8 @@ def handle_starttag( def handle_endtag(self, tag: str, method: t.Callable[[], t.Any]) -> None: method() - # Example -- report an unbalanced tag. def report_unbalanced(self, tag: str) -> None: - if self.verbose: - print("*** Unbalanced ") - print("*** Stack:", self.stack) + pass def convert_charref(self, name: str) -> str | None: """Convert character reference, may be overridden.""" @@ -521,11 +527,53 @@ def handle_comment(self, data: str) -> t.Any: pass # Example -- handle declaration, could be overridden - def handle_decl(self, decl: t.Any) -> t.Any: + def handle_decl(self, decl: str) -> t.Any: + pass + + def unknown_decl(self, data: str) -> t.Any: + """ + Handle a non-DOCTYPE declaration. + + Given an input text like this: + + .. code-block:: xml + + + + + This method would be called only for the ```` declaration: + + .. code-block:: python + + self.unknown_decl("SOMETHING bogus") + + The default implementation does nothing. + """ + pass # Example -- handle processing instruction, could be overridden def handle_pi(self, data: str) -> t.Any: + """ + Handle processing instructions. + + Given an input text like this: + + .. code-block:: xml + + + + This method will be called like: + + .. code-block:: python + + self.handle_pi('xml version="1.0" encoding="UTF-8" ?') + + Note that the closing question mark is not stripped. + + The default implementation does nothing. + """ + pass # To be overridden -- handlers for unknown objects diff --git a/tests/test_sgmllib.py b/tests/test_sgmllib.py index 348903a..eb7b335 100644 --- a/tests/test_sgmllib.py +++ b/tests/test_sgmllib.py @@ -641,6 +641,19 @@ def test_feed_unresolved_token_peak_memory_scales_nicely(): ) +def test_init_and_reset_functionality(): + """ + Verify that init and reset functionality work as intended. + + The `_markupbase.ParserBase.__init__()` method doesn't set any instance variables. + It's incumbent on `SGMLParser` to do extra work to ensure these are set. + The way by which this happens is unimportant; it's only important that it happens. + """ + + instance = sgmllib.SGMLParser() + assert instance.getpos() == (1, 0) + + # XXX These tests have been disabled by prefixing their names with # an underscore. The first two exercise outstanding bugs in the # sgmllib module, and the third exhibits questionable behavior diff --git a/tox.ini b/tox.ini index 148be05..4304a82 100644 --- a/tox.ini +++ b/tox.ini @@ -60,10 +60,9 @@ commands = [testenv:docs] description = Test building the documentation base_python = py3.14 -skip_install = true deps = -r requirements/docs/requirements.txt commands = - sphinx-build -aWEnb html docs/ build/docs/ + sphinx-build -aWEb html docs/ build/docs/ [testenv:build{, -minimum}] description =