docs: document each released version against the Clawpack source it shipped

.github/actions/setup-doc-build/action.yml

@@ -22,6 +22,12 @@ description: >-
 inputs:
   python-version:
     default: '3.12'
+  releases:
+    description: >-
+      Also materialise one Clawpack tree per released doc version
+      (doc/tools/clawpack-release-refs.txt) and export CLAW_VERSION_TREES, so
+      `make versions` documents each release against the code it shipped.
+    default: 'false'
 
 outputs:
   claw:
@@ -55,6 +61,16 @@ runs:
         echo "path=$RUNNER_TEMP/claw" >> "$GITHUB_OUTPUT"
         echo "CLAW=$RUNNER_TEMP/claw" >> "$GITHUB_ENV"
 
+    # --reference borrows from the dev tree just cloned: the release commits
+    # are in its history, so eight more trees cost little more than one.
+    - name: Materialise the release trees
+      if: inputs.releases == 'true'
+      shell: bash
+      run: |
+        python doc/tools/fetch_clawpack_src.py "$RUNNER_TEMP/claw-releases" \
+          --releases --reference "$RUNNER_TEMP/claw" --quiet
+        echo "CLAW_VERSION_TREES=$RUNNER_TEMP/claw-releases" >> "$GITHUB_ENV"
+
     # Fails loudly on the class of problem that used to surface only as
     # warnings nobody read.
     - name: Check the doc build environment

.github/workflows/docs-publish.yml

@@ -38,11 +38,14 @@ name: docs-publish
 #     `make versions` on a PR would show reviewers the site *without* the PR's
 #     changes.
 #
-#   * The Clawpack source tree autodoc documents is pinned by commit in
-#     doc/tools/clawpack-ref.txt and set up by .github/actions/setup-doc-build.
-#     Every version directory is therefore built against the same source; the
-#     historical ones get API docs from newer code, which is also what the
-#     manual `make versions` + rsync_doc.sh procedure always did.
+#   * Each version is documented against the Clawpack source it shipped.
+#     dev uses the tree pinned in doc/tools/clawpack-ref.txt; each released
+#     version uses its release commit from doc/tools/clawpack-release-refs.txt
+#     (doc/ext/clawpack_source.py swaps it in).  Building every version
+#     against one current tree, as the manual procedure did, documented code
+#     the releases never had and emptied pages for renamed modules.  The
+#     build fails on any autodoc import failure, and `stage` reports every
+#     page that loses API entries before anyone approves.
 
 on:
   push:
@@ -118,6 +121,8 @@ jobs:
           persist-credentials: false
 
       - uses: ./.github/actions/setup-doc-build
+        with:
+          releases: 'true'
 
       # actions/checkout leaves only the checked-out ref as a local branch,
       # and sphinx-multiversion ignores remote-tracking refs, so v5.14.x
@@ -243,6 +248,12 @@ jobs:
           echo "changed=true" >> "$GITHUB_OUTPUT"
           echo "publish_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
+      # An API page emptied by an import problem is just "M" in the file list
+      # above; this is how a reviewer sees it before approving.
+      - name: Report pages that lose API entries
+        if: steps.commit.outputs.changed == 'true'
+        run: python3 doc/tools/compare_autodoc.py site "$PREV_SHA" HEAD >> "$GITHUB_STEP_SUMMARY"
+
       - name: Report the diff
         if: always()
         run: |

.gitignore

@@ -16,6 +16,9 @@ api/generated/
 # docs can be built against exactly what CI builds against.  Deliberately
 # outside doc/, which is the Sphinx source dir.
 .claw-pin/
+# `make claw-releases`: one tree per row of doc/tools/clawpack-release-refs.txt,
+# which `make versions` builds each released version against.
+.claw-releases/
 
 # -----------------------------------------------------------------------------
 # Python

doc/Makefile

@@ -22,6 +22,10 @@ VERSIONSDIR   = _build
 # it.  Outside the sphinx source dir on purpose -- Sphinx would otherwise walk
 # it looking for .rst files.
 CLAWPIN       = ../.claw-pin
+# One tree per released version (tools/clawpack-release-refs.txt), which
+# `make versions` documents each release against; `make claw-releases` fills
+# it.  $CLAW_VERSION_TREES points the build elsewhere (CI does).
+CLAWRELEASES  = ../.claw-releases
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
@@ -35,7 +39,7 @@ else
 	LAYOUT = _themes/flask_local/layout.html
 endif
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest checkenv checkwarnings checkwarnings-update checkwarnings-strict check-versions versions clean-versions versions-promote versions-publish claw-pin claw-pin-check tools-tests
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest checkenv checkwarnings checkwarnings-update checkwarnings-strict check-versions versions clean-versions versions-promote versions-publish claw-pin claw-pin-check claw-releases claw-releases-check tools-tests
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -61,6 +65,8 @@ help:
 	@echo "  clean-versions    to remove $(VERSIONSDIR)"
 	@echo "  claw-pin             to clone the Clawpack source tree pinned in tools/clawpack-ref.txt"
 	@echo "                       into $(CLAWPIN); build against it with CLAW=\$$(cd $(CLAWPIN) && pwd)"
+	@echo "  claw-releases        to clone one tree per released version (tools/clawpack-release-refs.txt)"
+	@echo "                       into $(CLAWRELEASES); \`make versions\` documents each release against its own"
 	@echo "  checkenv             to check that every autodoc target imports from \$$CLAW"
 	@echo "  checkwarnings        to fail on any NEW reST/docstring warning (vs the baseline)"
 	@echo "  checkwarnings-update to regenerate the warning baseline (tools/doc_warnings_baseline.txt)"
@@ -209,6 +215,15 @@ claw-pin:
 claw-pin-check:
 	python tools/fetch_clawpack_src.py $(CLAWPIN) --check
 
+# The release trees for `make versions`.  The release commits are all in the
+# history of ordinary clones, so borrowing from $CLAW next door makes this
+# cheap; check-versions refuses to build if they are missing or stale.
+claw-releases:
+	python tools/fetch_clawpack_src.py $(CLAWRELEASES) --releases --reference ../.. --quiet
+
+claw-releases-check:
+	python tools/fetch_clawpack_src.py $(CLAWRELEASES) --releases --check
+
 checkenv:
 	python tools/check_doc_env.py
 
@@ -232,5 +247,6 @@ tools-tests:
 	python tools/test_check_doc_env.py
 	python tools/test_check_doc_warnings.py
 	python tools/test_promote_latest.py
+	python tools/test_compare_autodoc.py
 	bash tools/test_check_published_tree.sh
 

doc/conf.py

@@ -44,17 +44,18 @@ def _safe_fileConfig(*args, **kwargs):
 # is has to be one decision shared with tools/check_doc_warnings.py, or the
 # warning baseline stops being comparable -- hence tools/clawroot.py.
 # Set CLAW to build against the pinned tree (`make claw-pin`).
-from clawroot import claw_root
+#
+# That is the tree for dev and for single-version builds.  In a multiversion
+# build, ext/clawpack_source.py swaps in each released version's own tree
+# (tools/clawpack-release-refs.txt) before autodoc imports anything.
+from clawroot import CLAW_SUBPATHS, claw_root
 clawpack_root = claw_root()
 # Ahead of site-packages, so a pip-installed or editable clawpack cannot
 # shadow the tree we mean to document.
 sys.path.insert(0, clawpack_root)
 print("clawpack_root = %s" % clawpack_root)
-sys.path.append(os.path.join(clawpack_root,'amrclaw/doc'))
-sys.path.append(os.path.join(clawpack_root,'visclaw/doc'))
-sys.path.append(os.path.join(clawpack_root,'geoclaw/doc'))
-sys.path.append(os.path.join(clawpack_root,'pyclaw/src'))
-sys.path.append(os.path.join(clawpack_root,'riemann/src/python/riemann'))
+for _sub in CLAW_SUBPATHS:
+    sys.path.append(os.path.join(clawpack_root, _sub))
 
 
 # General configuration
@@ -70,7 +71,8 @@ def _safe_fileConfig(*args, **kwargs):
               'sphinx.ext.inheritance_diagram',
               'sphinx_multiversion',
               'sphinx.ext.mathjax',
-              'srclinks']
+              'srclinks',
+              'clawpack_source']
 
 
 # autodoc imports the documented modules at build time, so anything they

doc/ext/clawpack_source.py

@@ -0,0 +1,116 @@
+"""
+Document each version against the Clawpack source it shipped, and fail the
+build when autodoc cannot import something.
+
+sphinx-multiversion builds every version with *this* checkout's conf.py, and
+conf.py puts one tree on sys.path: $CLAW, pinned in tools/clawpack-ref.txt to
+current master.  A released version's .rst comes from its own tag, but without
+this extension its API pages would be generated from master: renamed modules
+fail to import (surge.storm -> met.storm emptied storm_module.html on every
+release), and everything else silently documents code the release never had.
+
+So, on ``config-inited``, a version listed in tools/clawpack-release-refs.txt
+gets its own tree (``tools/fetch_clawpack_src.py --releases``) swapped onto
+sys.path in place of $CLAW.  ``dev`` and plain single-version builds are left
+alone.
+
+autodoc reports an import failure as an ordinary warning, and a build that
+emits one still succeeds -- with an empty page.  Every page here is expected to
+import (tools/check_doc_env.py enforces that for dev before the build), so any
+``autodoc: failed to import`` in any version fails the build.
+"""
+
+import importlib
+import importlib.util
+import logging
+import os
+import sys
+import types
+
+from sphinx.errors import ExtensionError
+
+from clawroot import CLAW_SUBPATHS, claw_root, release_root
+
+
+def _claw_paths(root):
+    return [root] + [os.path.join(root, sub) for sub in CLAW_SUBPATHS]
+
+
+def _install_imp_shim():
+    """Python 3.12 removed ``imp``; v5.7.x's pyclaw does ``from imp import reload``.
+
+    Only ``reload`` is provided, which is all any release uses.
+    """
+    if importlib.util.find_spec('imp') is None and 'imp' not in sys.modules:
+        imp = types.ModuleType('imp')
+        imp.reload = importlib.reload
+        sys.modules['imp'] = imp
+
+
+def _use_version_tree(app, config):
+    version = getattr(config, 'smv_current_version', '') or ''
+    root = release_root(version) if version else None
+    if root is None:
+        app.clawpack_root = claw_root()
+        return
+
+    root = os.path.realpath(root)
+    stale = set(_claw_paths(claw_root()))
+    sys.path[:] = [p for p in sys.path if p not in stale]
+    sys.path[:0] = [root]
+    sys.path.extend(_claw_paths(root)[1:])
+    for name in [m for m in sys.modules
+                 if m == 'clawpack' or m.startswith('clawpack.')]:
+        del sys.modules[name]
+    _install_imp_shim()
+    app.clawpack_root = root
+    print(f'clawpack_root ({version}) = {root}')
+
+
+class _ImportFailures(logging.Handler):
+    """Collects autodoc's import-failure warnings, wherever they are emitted."""
+
+    def __init__(self):
+        super().__init__(level=logging.WARNING)
+        self.failures = set()
+
+    def emit(self, record):
+        try:
+            message = record.getMessage()
+        except Exception:
+            return
+        if 'failed to import' in message:
+            self.failures.add(message.splitlines()[0])
+
+
+def _check(app, exception, handler):
+    if exception is not None:
+        return
+    root = app.clawpack_root
+    clawpack = sys.modules.get('clawpack')
+    if clawpack is not None and getattr(clawpack, '__file__', None):
+        where = os.path.realpath(clawpack.__file__)
+        if os.path.commonpath([where, os.path.realpath(root)]) != os.path.realpath(root):
+            raise ExtensionError(
+                f'clawpack was imported from {where}, not from the tree this '
+                f'version documents ({root}).  An installed clawpack is '
+                'shadowing it; build in a virtualenv without one.')
+    if handler.failures:
+        raise ExtensionError(
+            f'autodoc could not import {len(handler.failures)} target(s) '
+            f'from {root}; their API pages would be empty:\n  '
+            + '\n  '.join(sorted(handler.failures)))
+
+
+def setup(app):
+    app.connect('config-inited', _use_version_tree, priority=100)
+    handler = _ImportFailures()
+    # Sphinx's loggers are all children of "sphinx"; a handler there sees
+    # autodoc's warnings, including ones replayed from parallel workers.
+    logging.getLogger('sphinx').addHandler(handler)
+
+    def check_imports(app, exception):
+        _check(app, exception, handler)
+
+    app.connect('build-finished', check_imports)
+    return {'parallel_read_safe': True, 'parallel_write_safe': True}

doc/howto_doc.rst

@@ -186,6 +186,49 @@ able to import what the docs reference::
 To bump the pinned Clawpack, edit `tools/clawpack-ref.txt` and repeat the
 regeneration above in the same commit.
 
+.. _howto_doc_release_trees:
+
+Released versions: one source tree each
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+`tools/clawpack-ref.txt` is the tree for `dev`.  A multiversion build documents
+each **released** version against the code that release shipped instead:
+`tools/clawpack-release-refs.txt` maps every built release (`v5.7.x`,
+`v5.8.x`, ...) to the `clawpack/clawpack` commit of its newest patch tag, and
+the subrepos come from that commit's submodule pointers.
+
+This matters because `sphinx-multiversion` builds every version with *this*
+checkout's `conf.py`.  Against one current tree, a release's API pages
+document code it never had, and a module renamed since the release documents
+nothing at all: `storm_module.html` came out empty on every release once
+`surge.storm` became a re-export of `met.storm`, with no warning.
+
+`make claw-releases` clones one tree per row into `$CLAW/doc/.claw-releases`
+(borrowing objects from your checkouts, so all of them take about a minute),
+and `ext/clawpack_source.py` swaps the right one onto `sys.path` as each
+version starts.  `$CLAW_VERSION_TREES` points the build at trees somewhere
+else.  So a full local multiversion build is::
+
+    make claw-pin
+    make claw-releases
+    CLAW=$(cd ../.claw-pin && pwd) make versions-publish
+
+Three checks keep this honest:
+
+- `make check-versions` fails if a built release has no row in
+  `tools/clawpack-release-refs.txt`, a row names a version that is not built,
+  or a release tree is missing or at the wrong commit.
+- The build fails if autodoc cannot import anything, in any version.  (That
+  catches modules that are gone; it cannot catch the re-export case above,
+  which imports fine and documents nothing.)
+- The publish workflow's `stage` summary lists every page that loses API
+  entries compared with the published site, before anyone approves a publish.
+
+When cutting a release, add its row to `tools/clawpack-release-refs.txt` (see
+:ref:`howto_release_doc`).  Releases before 5.8 import the `imp` module, which
+Python 3.12 removed; `ext/clawpack_source.py` provides the one function they
+use.
+
 .. note::
 
    One warning class is never baselined, however long it has been present:

doc/howto_release.rst

@@ -218,6 +218,12 @@ When making changes for a new release, the following pages in the directory
    `installing_pip.rst`, `installing_fortcodes.rst`,
    `contents.rst`, `docker_image.rst`.
 
+ - Add a row for the new version to `tools/clawpack-release-refs.txt` naming
+   the `clawpack/clawpack` commit of the release tag (and update the row when
+   a patch release follows), so its API pages are built from the code it
+   shipped.  `make check-versions` fails until the row is there.  See
+   :ref:`howto_doc_release_trees`.
+
  - Modify the main landing page `_templates/index.html` to cite the
    proper version number and DOI. 
    (No longer necessary -- This page has been changed to be more generic.)

doc/tools/check_versions.py

@@ -55,6 +55,17 @@
 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
 DOC_DIR = os.path.dirname(TOOLS_DIR)
 
+sys.path.insert(0, TOOLS_DIR)
+from clawroot import (RELEASE_MANIFEST, read_release_manifest,  # noqa: E402
+                      release_trees_dir)
+import fetch_clawpack_src  # noqa: E402
+
+# Built versions that document the dev pin (tools/clawpack-ref.txt) rather
+# than a release tree.  Every other built version needs a row in
+# tools/clawpack-release-refs.txt, or ext/clawpack_source.py would document it
+# against master.
+PINNED_VERSIONS = {"dev"}
+
 # Refs that match the whitelists but whose conf.py cannot be loaded by a
 # supported Sphinx.  See "The second trap" above.  Their content is already
 # published and the sync never deletes it, so dropping them is not data loss --
@@ -256,6 +267,27 @@ def main(argv=None):
             % ", ".join(unexpected)
         )
 
+    releases = set(read_release_manifest())
+    unpinned = sorted(set(built) - PINNED_VERSIONS - releases)
+    stale = sorted(releases - set(built))
+    if unpinned:
+        sys.exit(
+            "ERROR: built version(s) with no Clawpack release tree: %s\n"
+            "Add a row to %s naming the release commit to document them "
+            "against." % (", ".join(unpinned), os.path.relpath(RELEASE_MANIFEST))
+        )
+    if stale:
+        sys.exit(
+            "ERROR: %s lists version(s) that are not built: %s"
+            % (os.path.relpath(RELEASE_MANIFEST), ", ".join(stale))
+        )
+    print("\nrelease trees pinned for %d version(s); %s document(s) the dev pin"
+          % (len(releases), ", ".join(sorted(PINNED_VERSIONS & set(built)))))
+    # A tree left at an old commit after the manifest moved would document the
+    # wrong release without any error, so check the commits, not just presence.
+    if fetch_clawpack_src.check_releases(release_trees_dir()) != 0:
+        sys.exit(1)
+
     if recovered:
         print(
             "\nNOTE: %s now build(s) successfully; remove from "