diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e275ddf..986cb70 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,8 +15,18 @@ env: jobs: build-and-push: - name: Build Docker image + name: Build ${{ matrix.variant }} Docker image runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - variant: lean + dockerfile: docker/Dockerfile + suffix: '' + - variant: larcv + dockerfile: docker/Dockerfile.larcv + suffix: '-larcv' permissions: contents: read packages: write @@ -40,16 +50,18 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | # For PRs, just tag with pr-number (don't push) - type=ref,event=pr - # For version tags: v0.1.1 -> 0.1.1, 0.1, latest - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=ref,event=pr,suffix=${{ matrix.suffix }} + # Lean: 1.2.0, 1.2, latest. LArCV: 1.2.0-larcv, 1.2-larcv, larcv. + type=semver,pattern={{version}},suffix=${{ matrix.suffix }} + type=semver,pattern={{major}}.{{minor}},suffix=${{ matrix.suffix }} + type=raw,value=latest,enable=${{ matrix.variant == 'lean' && startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=larcv,enable=${{ matrix.variant == 'larcv' && startsWith(github.ref, 'refs/tags/v') }} - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . + file: ${{ matrix.dockerfile }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/CHANGELOG.md b/CHANGELOG.md index e418b77..45b94e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.2.0] - 2026-09-18 + +### Added +- **Direct LArCV input**: Load LArCV2 ROOT files through SPINE v1.2.4 using an explicitly selected, versioned detector conversion bundle from `spine-prod`. +- **LArCV container flavor**: Publish a separate ROOT/LArCV-enabled image while retaining the existing lean image and tags as the default. + +### Changed +- **Docker layout**: Keep both image definitions, the local run helper, and container-build documentation under `docker/`. +- **Contextual LArCV configuration**: Request a detector converter only after opening a detected ROOT source, leaving the control hidden for other source types. +- **Source navigation**: Start newly opened files at entry 0 instead of carrying over the previous file's entry index. +- **Unified source entry**: Accept server paths and public URLs through one Path field, using the existing status indicator for remote loading. +- **Remote ROOT staging**: Prompt for a LArCV converter from a remote ``.root`` URL before downloading it, then validate its content while the standard loading indicator reports the transfer. +- **Converter labels**: Distinguish multiple detector-production converters using tags maintained by spine-prod rather than duplicating detector knowledge in Spinal Tap. + +### Fixed +- **Particle vertices from LArCV**: Build truth interactions alongside particles so the interaction-vertex overlay works in particle mode. +- **Cached URL inspection**: Refresh temporary-file retention without changing modification timestamps, preserving reader and built-event cache hits when inspecting objects from downloaded sources. +- **LArCV converter errors**: Validate required ROOT trees before constructing LArCV chains, reporting incompatible converter selections in the application instead of emitting ROOT-only terminal diagnostics. +- **LArCV fragment mode**: Preserve SPINE's particle dependency when building interactions alongside truth fragments for vertex-capable displays. +- **LArCV manifests**: Request and preserve one converter for homogeneous manifests of LArCV ROOT files. ## [1.1.2] - 2026-09-12 diff --git a/README.md b/README.md index 91b28ed..5f1e872 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ Spinal Tap is a Dash application that provides simple visualization tools for the Scalable Particle Imaging With Neural Embeddings ([SPINE](https://github.com/DeepLearnPhysics/spine)) package. +The standard installation and container remain lean and read SPINE HDF5 files. +An optional LArCV container adds ROOT, LArCV2, and maintained +[`spine-prod`](https://github.com/DeepLearnPhysics/spine-prod) detector +conversion bundles so raw LArCV ROOT files can be visualized directly. + ## Installation @@ -85,12 +90,26 @@ Docker images are automatically built and published to GitHub Container Registry docker pull ghcr.io/deeplearnphysics/spinal-tap:latest ``` +Use the separate LArCV flavor only when direct ROOT input is needed: + +```bash +docker pull ghcr.io/deeplearnphysics/spinal-tap:larcv +``` + To run locally with Docker: ```bash docker run -p 8888:8888 ghcr.io/deeplearnphysics/spinal-tap:latest ``` +For LArCV input, run the LArCV image, mount the source data, and select the +matching dated detector converter in the **Data** panel: + +```bash +docker run --rm -p 8888:8888 -v /host/data:/data:ro \ + ghcr.io/deeplearnphysics/spinal-tap:larcv +``` + ## Development & CI/CD - Code style is enforced with black, isort, and flake8 (pre-commit and CI). diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/docker/Dockerfile.larcv b/docker/Dockerfile.larcv new file mode 100644 index 0000000..87cfb06 --- /dev/null +++ b/docker/Dockerfile.larcv @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1 + +ARG LARCV_TAG=runtime-ubuntu24.04 +ARG SPINE_PROD_REF=3cb7ff9ac5fb1baf1a5958bc67e36be616adab52 + +FROM alpine:3.22 AS configs +ARG SPINE_PROD_REF +ADD https://github.com/DeepLearnPhysics/spine-prod/archive/${SPINE_PROD_REF}.tar.gz /tmp/spine-prod.tar.gz +RUN mkdir -p /opt/spine-prod && \ + tar -xzf /tmp/spine-prod.tar.gz --strip-components=1 -C /opt/spine-prod + +FROM ghcr.io/deeplearnphysics/larcv2:${LARCV_TAG} + +USER root +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_BREAK_SYSTEM_PACKAGES=1 \ + SPINE_CONFIG_PATH=/opt/spine-prod/config + +RUN apt-get update && \ + apt-get install -y --no-install-recommends python3-pip && \ + rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml MANIFEST.in README.md ./ +COPY src/ ./src/ + +# Keep the NumPy ABI used to compile LArCV's PyUtil bridge. Declaring the exact +# already-installed version prevents pip from trying to replace the Debian +# package with a newer, independently built NumPy wheel. +RUN pip3 install "numpy==1.26.4" . + +COPY --from=configs /opt/spine-prod/config /opt/spine-prod/config + +EXPOSE 8888 +ENTRYPOINT ["spinal-tap"] +CMD ["--host", "0.0.0.0", "--port", "8888"] diff --git a/docs/DOCKER_STRATEGY.md b/docker/README.md similarity index 67% rename from docs/DOCKER_STRATEGY.md rename to docker/README.md index 39c8cdc..46ca8dc 100644 --- a/docs/DOCKER_STRATEGY.md +++ b/docker/README.md @@ -1,11 +1,31 @@ # Docker Image Build Strategy +## Repository Layout + +All container-specific build files live in this directory: + +- `Dockerfile` builds the lean SPINE HDF5 viewer. +- `Dockerfile.larcv` builds the optional ROOT/LArCV flavor. +- `run.sh` builds and starts the lean image for local development. + +Both Dockerfiles use the repository root as their build context: + +```bash +docker build -f docker/Dockerfile -t spinal-tap:local . +docker build -f docker/Dockerfile.larcv -t spinal-tap:larcv-local . +``` + +The shared `.dockerignore` remains at the repository root because Docker reads +it from the build-context root, not from the Dockerfile directory. + ## Build Triggers The Docker image is built and published **only when**: 1. **A version tag is pushed** (e.g., `v0.1.1`, `v1.0.0`) - - Creates versioned tags: `0.1.1`, `0.1`, and `latest` + - Builds both the lean and LArCV image flavors + - Creates lean tags: `0.1.1`, `0.1`, and `latest` + - Creates LArCV tags: `0.1.1-larcv`, `0.1-larcv`, and `larcv` 2. **Pull requests** (for testing only) - Builds the image but doesn't push to registry @@ -32,12 +52,16 @@ When you create a tag like `v0.1.1`: ghcr.io/deeplearnphysics/spinal-tap:0.1.1 (full version) ghcr.io/deeplearnphysics/spinal-tap:0.1 (minor version) ghcr.io/deeplearnphysics/spinal-tap:latest (latest release) +ghcr.io/deeplearnphysics/spinal-tap:0.1.1-larcv (LArCV full version) +ghcr.io/deeplearnphysics/spinal-tap:0.1-larcv (LArCV minor version) +ghcr.io/deeplearnphysics/spinal-tap:larcv (latest LArCV release) ``` Benefits: - Users can pin to exact version: `image: spinal-tap:0.1.1` - Users can auto-update minor versions: `image: spinal-tap:0.1` - Users can always get latest: `image: spinal-tap:latest` +- ROOT/LArCV dependencies remain isolated in the `-larcv`/`larcv` flavor ## Automatic Cleanup @@ -55,12 +79,9 @@ The workflow includes automatic cleanup: - ✅ For public packages: **No practical limit** ### Image Size Optimization: -Our Dockerfile uses: -- **Python slim base** (~150MB vs 1GB for full Python) -- **Multi-stage builds** would further reduce size (can add if needed) -- **Layer caching** - Only changed layers are rebuilt - -Current estimated image size: **~500MB - 1GB** (depending on spine dependencies) +The lean Dockerfile uses a Python slim base and remains free of ROOT and LArCV. +The optional multi-stage LArCV Dockerfile starts from the dedicated +LArCV runtime image and copies only the `spine-prod` configuration tree. ## Workflow Summary @@ -69,9 +90,10 @@ Developer creates tag v0.1.1 ↓ GitHub Action triggered ↓ -Build Docker image +Build lean and LArCV Docker images ↓ -Push with tags: 0.1.1, 0.1, latest +Push lean tags: 0.1.1, 0.1, latest +Push LArCV tags: 0.1.1-larcv, 0.1-larcv, larcv ↓ Cleanup old untagged images ↓ diff --git a/docker_run.sh b/docker/run.sh similarity index 64% rename from docker_run.sh rename to docker/run.sh index b4cc087..89f012e 100755 --- a/docker_run.sh +++ b/docker/run.sh @@ -6,9 +6,11 @@ set -e IMAGE_NAME="spinal-tap" IMAGE_TAG="local" PORT=8888 +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" echo "🔨 Building Docker image..." -docker build -t ${IMAGE_NAME}:${IMAGE_TAG} . +docker build -f "${REPO_ROOT}/docker/Dockerfile" \ + -t "${IMAGE_NAME}:${IMAGE_TAG}" "${REPO_ROOT}" echo "✅ Build complete!" echo "" @@ -18,6 +20,6 @@ echo "🛑 Press Ctrl+C to stop" echo "" docker run --rm -it \ - -p ${PORT}:${PORT} \ + -p "${PORT}:${PORT}" \ --name spinal-tap-dev \ - ${IMAGE_NAME}:${IMAGE_TAG} + "${IMAGE_NAME}:${IMAGE_TAG}" diff --git a/docs/source/index.rst b/docs/source/index.rst index 9ce1365..a0cab2e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,7 @@ one browser application. The application is intended for three common workflows: -* inspect a SPINE HDF5 file on the same machine as the server; +* inspect a SPINE HDF5 file, or a LArCV ROOT file with the optional container; * share a reproducible event view with another collaborator; and * deploy a controlled, authenticated display beside experiment data. @@ -52,8 +52,9 @@ Install and launch Spinal Tap: spinal-tap Open ``http://localhost:8888``, enter an HDF5 path in **Path**, select **Open**, -and navigate with **Go** or the left/right arrows. See :doc:`sources` for -manifests, uploads, URLs, and shared view JSON files. +and navigate with **Go** or the left/right arrows. The optional LArCV image can +open ROOT input after selecting its detector conversion bundle. See +:doc:`sources` for manifests, uploads, URLs, and shared view JSON files. Choosing a renderer ------------------- diff --git a/docs/source/installation.rst b/docs/source/installation.rst index c0d9c12..a4d84e2 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -62,7 +62,21 @@ Released images are published to the GitHub Container Registry: docker run --rm -p 8888:8888 \ ghcr.io/deeplearnphysics/spinal-tap:latest -Mount data into the container when using **Path** sources: +The default image is the lean SPINE HDF5 viewer and does not include ROOT or +LArCV. A separate image supports direct LArCV ROOT input: + +.. code-block:: bash + + docker pull ghcr.io/deeplearnphysics/spinal-tap:larcv + docker run --rm -p 8888:8888 \ + -v /host/data:/data:ro \ + ghcr.io/deeplearnphysics/spinal-tap:larcv + +Release-specific tags use ``-larcv``. The LArCV image includes the +ROOT/LArCV runtime and versioned detector conversion bundles from +``spine-prod``; the Python package and default image deliberately do not. + +Mount data into the container when using local **Path** sources: .. code-block:: bash diff --git a/docs/source/sharing.rst b/docs/source/sharing.rst index b246720..c6b4786 100644 --- a/docs/source/sharing.rst +++ b/docs/source/sharing.rst @@ -17,9 +17,9 @@ View JSON --------- **Export JSON** downloads the same reproducible view state as a file. Open that -JSON through Path, Browse/Upload, or URL just like any other source. Loading the -view opens its underlying HDF5 source and applies the saved state once the event -is available. +JSON through Path or Browse/Upload just like any other source. The Path field +also accepts URLs. Loading the view opens its underlying HDF5 source and applies +the saved state once the event is available. The view is a starting point rather than a lock: after it loads, normal event navigation preserves the user's current camera behavior instead of repeatedly diff --git a/docs/source/sources.rst b/docs/source/sources.rst index 1d3da80..948c602 100644 --- a/docs/source/sources.rst +++ b/docs/source/sources.rst @@ -16,8 +16,9 @@ arrows to load entries. - Use it for - Behavior * - Path - - A path or glob visible to the server. - - Opens the host file directly without copying it. + - A server-visible path or glob, or a public HTTP(S) URL. + - Opens paths directly. Downloads URLs into the bounded temporary cache + before opening them. * - Browse - A local file selected with the operating-system picker. - Transfers the selected file into a private temporary cache. This label @@ -25,11 +26,9 @@ arrows to load entries. * - Upload - A workstation file sent to an authenticated hosted deployment. - Uses the same private temporary cache as Browse, in retryable chunks. - * - URL - - A public HTTP or HTTPS source. - - Downloads the source into the bounded temporary cache before opening it. -Pressing Enter in a Path or URL field performs the same action as **Open**. +The Path field also accepts public URLs. Pressing Enter performs the same action +as **Open**. Pressing Enter in the event selector performs the same action as **Go**. Accepted source documents @@ -38,9 +37,41 @@ Accepted source documents Spinal Tap identifies exact files by content rather than extension. It accepts: * SPINE HDF5 output; +* LArCV2 ROOT input when running the LArCV container flavor; * a saved Spinal Tap view JSON document; or * a UTF-8 file manifest. +LArCV conversion bundles +------------------------ + +LArCV tree names and detector geometry are producer-specific. Open a ROOT +source first; after Spinal Tap detects its content, it reveals **LArCV +converter** beneath the Open button and waits for a matching dated bundle. +Selecting the bundle resumes the load automatically and dismisses the chooser. +When a detector has multiple bundles, their labels include compatibility tags +maintained by ``spine-prod``, such as cryostat layout, CRT availability, and +optical-system generation. +For a remote URL whose path ends in ``.root``, the chooser appears immediately +and the download begins only after selection, with progress reported by the +standard status indicator. The suffix is only an early UI hint: Spinal Tap +still validates the downloaded content. Extensionless remote sources must be +downloaded before their type can be detected. The control stays hidden for +other source types. +Spinal Tap then parses and builds the SPINE truth representation in memory; it +does not write an intermediate HDF5 file. A manifest may contain several LArCV +files that use the same bundle, but it cannot mix LArCV and HDF5 files. + +Detector conversion knowledge is maintained in ``spine-prod`` and copied into +the LArCV image at build time. Install an additional configuration tree by +setting ``SPINAL_TAP_LARCV_CONFIG_ROOT`` (or ``SPINE_CONFIG_PATH``) to its +``config`` directory. ``SPINAL_TAP_LARCV_CONFIG`` can set a default converter +ID or an explicit bundle path for non-interactive deployments. + +Spinal Tap intentionally does not guess a detector or schema from tree names: +different production vintages can use overlapping names with different +semantics. If a bundle reports missing trees, select the matching vintage or +add the required variant in ``spine-prod``. + A manifest contains one source path per line. Blank lines and records beginning with ``#`` are ignored. Relative records are resolved beside the manifest, and globs are supported both in the Path field and in manifest records. The file diff --git a/pyproject.toml b/pyproject.toml index d393cb9..3104d05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ keywords = [ "WebGL" ] dependencies = [ - "spine[viz]>=1.2.1", + "spine[viz]>=1.2.4", "dash>=4.4.0", "flask>=2.0.0", "flask-session>=0.5.0", diff --git a/src/spinal_tap/assets/base.css b/src/spinal_tap/assets/base.css index 4205616..727b7ef 100644 --- a/src/spinal_tap/assets/base.css +++ b/src/spinal_tap/assets/base.css @@ -857,6 +857,7 @@ html[data-spinal-tap-theme="dark"] body .Select-control * { } .source-action-row { + position: relative; display: grid; grid-template-columns: minmax(0, 1fr) auto; align-items: start; @@ -945,6 +946,36 @@ html[data-spinal-tap-theme="dark"] body .Select-control * { min-width: 64px; } +.larcv-converter-popover { + position: absolute; + z-index: 80; + top: calc(100% + 7px); + right: 0; + width: min(360px, 100%); + padding: 10px; + border: 1px solid var(--line-strong); + border-radius: 7px; + background: var(--panel); + box-shadow: 0 10px 28px rgba(0, 0, 0, 0.28); +} + +.larcv-converter-popover::before { + position: absolute; + top: -6px; + right: 25px; + width: 10px; + height: 10px; + border-top: 1px solid var(--line-strong); + border-left: 1px solid var(--line-strong); + background: var(--panel); + content: ""; + transform: rotate(45deg); +} + +.larcv-converter-popover .field-label { + margin-bottom: 6px; +} + .source-open-summary { display: none; } diff --git a/src/spinal_tap/assets/webgl-viewer.js b/src/spinal_tap/assets/webgl-viewer.js index 19fb58a..9de2988 100644 --- a/src/spinal_tap/assets/webgl-viewer.js +++ b/src/spinal_tap/assets/webgl-viewer.js @@ -2429,7 +2429,7 @@ function sourceActivity() { const mode = document.querySelector("#source-mode input:checked")?.value; const source = document.getElementById("input-file-path")?.value?.trim(); - if (mode === "url" && source) { + if (/^https?:\/\//.test(source || "")) { try { const url = new URL(source); const name = url.pathname.split("/").filter(Boolean).pop(); @@ -2459,7 +2459,9 @@ const target = event.target instanceof Element ? event.target : null; if (!target) return; - if (target.closest("#button-load")) { + if (target.closest("#dropdown-larcv-config")) { + queueActivity(sourceActivity()); + } else if (target.closest("#button-load")) { queueActivity(sourceActivity()); } else if (target.closest( "#button-go, #button-previous, #button-next" diff --git a/src/spinal_tap/cache.py b/src/spinal_tap/cache.py index 9099424..405046b 100644 --- a/src/spinal_tap/cache.py +++ b/src/spinal_tap/cache.py @@ -72,6 +72,12 @@ def ensure_root(self) -> None: """Create the private cache root when it is first needed.""" self.root.mkdir(mode=0o700, parents=True, exist_ok=True) + @staticmethod + def _refresh_access(path: Path) -> None: + """Refresh cache retention without changing the content signature.""" + stat = path.stat() + os.utime(path, ns=(time.time_ns(), stat.st_mtime_ns)) + def session_root(self, session_id: str | None = None) -> Path: """Return and create the private directory for one browser session.""" self.ensure_root() @@ -87,7 +93,7 @@ def owns_path(self, file_path: str, session_id: str | None = None) -> bool: owned = path.is_file() and path.is_relative_to(root) if owned: # Keep sources which are still being viewed from expiring. - os.utime(path, None) + self._refresh_access(path) return owned except (OSError, RuntimeError, ValueError): return False @@ -99,7 +105,7 @@ def cleanup(self) -> None: with self._lock: for path in self.root.glob("*/*"): try: - if path.stat().st_mtime < cutoff: + if path.stat().st_atime < cutoff: path.unlink() except FileNotFoundError: pass @@ -242,7 +248,7 @@ def fetch_url(self, url: str) -> str: root = self.session_root("url") final_path = root / f"{url_id}-{filename}" if final_path.is_file(): - os.utime(final_path, None) + self._refresh_access(final_path) return str(final_path) part_path = root / f".{url_id}-{secrets.token_urlsafe(8)}.part" diff --git a/src/spinal_tap/callbacks.py b/src/spinal_tap/callbacks.py index f0f5bf8..9d45392 100644 --- a/src/spinal_tap/callbacks.py +++ b/src/spinal_tap/callbacks.py @@ -1,6 +1,7 @@ """Defines the callbacks of the Spinal Tap application.""" import json +import os import uuid from dataclasses import replace from pathlib import Path @@ -12,6 +13,7 @@ from spine.geo import GeoManager from spine.vis import Drawer, colorable_attributes, object_color_kind +from .converters import available_larcv_converters from .filtering import ( attach_object_filter_metadata, build_object_filter_options, @@ -23,9 +25,11 @@ from .source import read_file_manifest from .utils import ( canonicalize_data_path, + classify_reader_source, classify_source, get_reader_products, initialize_reader, + is_remote_root_hint, load_data, resolve_source_path, ) @@ -70,6 +74,7 @@ } NAVIGATION_TRIGGERS = SOURCE_OPEN_TRIGGERS | EVENT_NAVIGATION_TRIGGERS FILTER_RESET_TRIGGERS = NAVIGATION_TRIGGERS | { + "dropdown-larcv-config", "radio-object-mode", "radio-run-mode", "store-share-pending", @@ -182,6 +187,10 @@ def validate_view_state(state): ) if not isinstance(state.get("file"), str) or not state["file"]: raise ValueError("The view state does not contain a file path.") + if state.get("larcv_converter") is not None and not isinstance( + state["larcv_converter"], str + ): + raise ValueError("The view state contains an invalid LArCV converter.") try: entry = int(state.get("entry")) except (TypeError, ValueError) as error: @@ -1356,7 +1365,9 @@ def update_object_inspector( try: reader = initialize_reader( - loaded_event["file_path"], loaded_event.get("use_run", False) + loaded_event["file_path"], + loaded_event.get("use_run", False), + loaded_event.get("larcv_converter"), ) data, *_ = load_data(reader, loaded_event["entry"], mode, selected_family) collection = data[object_collection_key(prefix, selected_family)] @@ -1519,6 +1530,7 @@ def update_object_inspector( return [{ version: window.spinalTapShare?.version || 1, file: loadedEvent.file_path, + larcv_converter: loadedEvent.larcv_converter || null, entry: loadedEvent.entry, run: loadedEvent.run, subrun: loadedEvent.subrun, @@ -1765,23 +1777,24 @@ def update_object_inspector( } const setProps = window.dash_clientside.set_props; - const dataMode = /^https?:\/\//.test(state.file) ? 'url' : 'path'; + const dataMode = 'path'; const sourceMemory = window.spinalTapSourceState; + const importMode = ['browse', 'upload'].includes(state.import_mode) + ? state.import_mode : dataMode; const offClicks = (linkClicks || 0) % 2 ? (linkClicks || 0) + 1 : (linkClicks || 0); // Browse describes how the view definition was imported. Keep it - // selected, but remember the referenced data source in the Path or - // URL field where the user expects to find it later. + // selected, but remember the referenced data source in the combined + // Path field where the user expects to find it later. if (sourceMemory?.values) { sourceMemory.values[dataMode] = state.file; - sourceMemory.values[dataMode === 'url' ? 'path' : 'url'] = ''; } setProps('input-file-path', {value: state.file}); setProps('input-entry', {value: state.entry ?? 0}); setProps('source-mode', { - value: state.import_mode || dataMode + value: importMode }); setProps('entry-mode', {value: 'entry'}); setProps('input-entry', { @@ -2960,6 +2973,75 @@ def handle_login(n_clicks, experiment, password): return "Invalid credentials", None + @app.callback( + [ + Output("store-larcv-request", "data"), + Output("larcv-converter-row", "hidden"), + Output("dropdown-larcv-config", "value"), + ], + [ + Input("button-load", "n_clicks"), + Input("store-source-request", "data"), + Input("input-file-path", "n_submit"), + Input("dropdown-larcv-config", "value"), + ], + [ + State("input-file-path", "value"), + State("source-mode", "value"), + State("dropdown-larcv-config", "options"), + ], + prevent_initial_call=True, + ) + def request_larcv_converter( + n_clicks_load, + source_request, + n_submit_source, + larcv_converter, + file_path, + source_mode, + converter_options, + ): + """Reveal converter selection only after opening a LArCV ROOT source.""" + del n_clicks_load, n_submit_source + + if ctx.triggered_id == "dropdown-larcv-config": + if larcv_converter: + return no_update, True, no_update + return (no_update,) * 3 + + if ctx.triggered_id == "store-source-request": + if not source_request: + return None, True, None + file_path = source_request.get("file_path") + source_mode = source_request.get("source_mode", source_mode) + + if not file_path or not converter_options: + return None, True, None + + if is_remote_root_hint(file_path): + source_kind = "larcv" + else: + try: + source_kind, _ = classify_source(file_path) + if source_kind == "manifest": + source_kind = classify_reader_source(file_path) + except (OSError, ValueError): + # The graph callback owns user-visible source errors. + return None, True, None + + if source_kind != "larcv" or os.getenv("SPINAL_TAP_LARCV_CONFIG"): + return None, True, None + + return ( + { + "file_path": file_path, + "source_mode": source_mode, + "revision": uuid.uuid4().hex, + }, + False, + None, + ) + @app.callback( [ Output("div-evd", "children"), @@ -3006,11 +3088,13 @@ def handle_login(n_clicks, experiment, password): Input("radio-crt-mode", "value"), Input("checklist-draw-mode-2", "value"), Input("store-share-pending", "data"), + Input("dropdown-larcv-config", "value"), ], [ State("store-object-filter", "data"), State("input-file-path", "value"), State("source-mode", "value"), + State("store-larcv-request", "data"), State("input-entry", "value"), State("input-run", "value"), State("input-subrun", "value"), @@ -3063,9 +3147,11 @@ def update_graph( crt_mode, draw_mode_2, share_pending, + larcv_converter, object_filter, file_path, source_mode, + larcv_request, entry, run, subrun, @@ -3141,6 +3227,8 @@ def update_graph( Path to the input file source_mode : str Source mode used to open the input file. + larcv_converter : str, optional + Installed spine-prod conversion bundle used for LArCV input. entry : int Entry number entry_prev : int @@ -3207,6 +3295,12 @@ def update_graph( file_path = source_request.get("file_path") source_mode = source_request.get("source_mode", source_mode) + if trigger == "dropdown-larcv-config": + if not larcv_converter or not larcv_request: + return (no_update,) * 17 + file_path = larcv_request.get("file_path") + source_mode = larcv_request.get("source_mode", source_mode) + restoring_view = False share_request_output = no_update restored_state = None @@ -3245,12 +3339,24 @@ def update_graph( subrun = loaded_event.get("subrun") event = loaded_event.get("event") use_run = loaded_event.get("use_run", use_run) + larcv_converter = loaded_event.get("larcv_converter", larcv_converter) # Browse identifies the file used to import data or a saved view. Once # an event is open, navigation must follow that event's resolved HDF5 # source rather than re-opening the uploaded JSON document. file_path = navigation_file_path(file_path, source_mode, trigger, loaded_event) + # An entry index belongs to the file it was selected from. Start a + # genuinely different source at its first entry instead of carrying a + # potentially invalid index across from the currently displayed file. + opening_source = trigger in SOURCE_OPEN_TRIGGERS or trigger == ( + "dropdown-larcv-config" + ) + if opening_source and loaded_event and file_path: + previous_path = loaded_event.get("file_path") + if previous_path != canonicalize_data_path(file_path): + entry = 0 + # Updating store-entry reflects automatic file geometry into the two # dropdowns. Those programmatic value changes do not require another # scene build; the navigation callback already drew that geometry. @@ -3312,6 +3418,20 @@ def fail(message): if not is_valid: return fail(error_msg) + # A .root suffix in a remote URL is enough to stage converter choice, + # but not enough to classify the source. Defer the download until the + # user selects a converter; normal content detection below then + # validates the downloaded file. Without installed converter options, + # continue so the source produces its normal actionable error. + if ( + trigger in SOURCE_OPEN_TRIGGERS + and restored_state is None + and is_remote_root_hint(file_path) + and not os.getenv("SPINAL_TAP_LARCV_CONFIG") + and available_larcv_converters() + ): + return (no_update,) * 17 + # Dispatch exact files by content, not extension. URL sources are # materialized into the private cache before applying the same rules. try: @@ -3354,6 +3474,7 @@ def fail(message): overlays = display_state.get("overlays") or [] file_path = restored_state["file"] + larcv_converter = restored_state.get("larcv_converter", larcv_converter) entry = restored_state["entry"] entry_mode = "entry" use_run = False @@ -3409,10 +3530,29 @@ def fail(message): if source_kind == "json": return fail("A shared view cannot reference another shared view.") + reader_source_kind = source_kind if source_kind == "manifest": is_valid, error_msg = validate_manifest_access(resolved_source) if not is_valid: return fail(error_msg) + try: + reader_source_kind = classify_reader_source(file_path) + except (OSError, ValueError) as error: + return fail( + f"Could not inspect source manifest:\n" + f"{type(error).__name__}: {error}" + ) + + # Opening raw ROOT first requests an explicit parser selection. The + # dropdown change re-enters this callback with the source remembered + # in ``store-larcv-request`` and performs the actual load. + if ( + trigger in SOURCE_OPEN_TRIGGERS + and reader_source_kind == "larcv" + and restored_state is None + and not os.getenv("SPINAL_TAP_LARCV_CONFIG") + ): + return (no_update,) * 17 try: entry = parse_optional_int(entry) @@ -3424,14 +3564,16 @@ def fail(message): else: try: - reader = initialize_reader(file_path, use_run) + reader = initialize_reader(file_path, use_run, larcv_converter) msg = f"File(s) found with {len(reader)} entries" except FileNotFoundError: msg = f"File(s) not found:\n{file_path}" return fail(msg) - except Exception as e: - msg = repr(e) - return fail(msg) + except Exception as error: + return fail( + f"Could not initialize the source:\n" + f"{type(error).__name__}: {error}" + ) # Check that the appropriate information is provided, abort otherwise if not use_run and entry is None: @@ -3762,6 +3904,9 @@ def fail(message): "subrun": subrun, "event": event, "use_run": use_run, + "larcv_converter": ( + larcv_converter if reader_source_kind == "larcv" else None + ), "reco_filter": [ key for key in active_filter if key.startswith("reco:") ], @@ -3785,11 +3930,7 @@ def fail(message): "num_entries": len(reader), "temporary": temporary_source, "navigation_source_mode": ( - ( - "url" - if canonical_path.startswith(("http://", "https://")) - else "path" - ) + "path" if source_mode in {"browse", "upload"} and trigger in EVENT_NAVIGATION_TRIGGERS else None @@ -3907,16 +4048,8 @@ def update_source_mode(mode): shown = {"display": "flex"} if mode in {"browse", "upload"}: return "", True, hidden, {"display": "grid"}, {"display": "block"} - if mode == "url": - return ( - "HTTPS URL to HDF5, manifest, or view…", - False, - shown, - hidden, - {"display": "block"}, - ) return ( - "HDF5, manifest, or view path…", + "HDF5, LArCV ROOT, manifest, view path, or HTTPS URL…", False, shown, hidden, @@ -3931,13 +4064,11 @@ def update_source_mode(mode): } const file = loadedEvent.file_path; - const mode = /^https?:\/\//.test(file) ? 'url' : 'path'; - const other = mode === 'url' ? 'path' : 'url'; + const mode = 'path'; const navigationMode = loadedEvent.navigation_source_mode; const sourceMemory = window.spinalTapSourceState; if (sourceMemory?.values) { sourceMemory.values[mode] = file; - sourceMemory.values[other] = ''; if (navigationMode) { sourceMemory.active = navigationMode; setTimeout(function() { diff --git a/src/spinal_tap/converters.py b/src/spinal_tap/converters.py new file mode 100644 index 0000000..8ee5519 --- /dev/null +++ b/src/spinal_tap/converters.py @@ -0,0 +1,97 @@ +"""Discover and resolve versioned spine-prod LArCV conversion bundles.""" + +import os +from collections import Counter +from pathlib import Path + +import yaml + + +def _converter_tags(path: Path, roots: list[Path]) -> list[str]: + """Read display tags, falling back through the converter's includes.""" + + def collect(config_path: Path, visited: set[Path]) -> list[str]: + config_path = config_path.resolve() + if config_path in visited: + return [] + visited.add(config_path) + + try: + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + return [] + + tags = config.get("__meta__", {}).get("tags") or [] + if tags: + return [str(tag) for tag in tags] + + includes = config.get("include") or [] + if isinstance(includes, str): + includes = [includes] + + inherited = [] + for include in includes: + candidates = [config_path.parent / include] + candidates.extend(root / include for root in roots) + include_path = next( + (candidate for candidate in candidates if candidate.is_file()), None + ) + if include_path is not None: + inherited.extend(collect(include_path, visited)) + + return list(dict.fromkeys(inherited)) + + return collect(path, set()) + + +def _config_roots() -> list[Path]: + """Return configured SPINE configuration roots in lookup order.""" + configured = os.getenv("SPINAL_TAP_LARCV_CONFIG_ROOT") or os.getenv( + "SPINE_CONFIG_PATH", "" + ) + return [Path(path).expanduser() for path in configured.split(os.pathsep) if path] + + +def available_larcv_converters() -> list[dict[str, str]]: + """Return Dash options for all installed dated truth-conversion bundles.""" + roots = _config_roots() + converters: dict[str, Path] = {} + for root in roots: + convert_root = root / "convert" + if not convert_root.is_dir(): + continue + for path in convert_root.glob("*/truth_*.yaml"): + converter_id = f"{path.parent.name}/{path.stem}" + converters.setdefault(converter_id, path) + + detector_counts = Counter( + converter_id.split("/", 1)[0] for converter_id in converters + ) + options = [] + for converter_id, path in sorted(converters.items()): + detector = converter_id.split("/", 1)[0] + label = f"{detector} · {path.stem}" + if detector_counts[detector] > 1: + tags = _converter_tags(path, roots) + if tags: + label += f" · {', '.join(tags)}" + options.append({"label": label, "value": converter_id}) + + return options + + +def resolve_larcv_converter(value: str) -> str: + """Resolve a converter ID or explicit YAML path to an installed bundle.""" + candidate = Path(value).expanduser() + if candidate.is_file(): + return str(candidate.resolve()) + + relative = Path("convert") / f"{value.removesuffix('.yaml')}.yaml" + for root in _config_roots(): + candidate = root / relative + if candidate.is_file(): + return str(candidate.resolve()) + + raise FileNotFoundError( + f"LArCV converter {value!r} was not found in SPINE_CONFIG_PATH." + ) diff --git a/src/spinal_tap/layout.py b/src/spinal_tap/layout.py index ff47c38..b4ef0f2 100644 --- a/src/spinal_tap/layout.py +++ b/src/spinal_tap/layout.py @@ -4,6 +4,7 @@ from dash import dcc, html from spine.geo.factories import geo_dict +from .converters import available_larcv_converters from .version import __version__ SPINAL_TAP_LOGO_BLACK = "/assets/spinal-tap-logo-black.png" @@ -157,7 +158,7 @@ def entry_controls(): label = "Upload" if REQUIRE_AUTH else "Browse" value = "upload" if REQUIRE_AUTH else "browse" source_options.append({"label": label, "value": value}) - source_options.append({"label": "URL", "value": "url"}) + converter_options = available_larcv_converters() return section( "Data", @@ -172,7 +173,10 @@ def entry_controls(): id="input-file-path", type="text", value="", - placeholder="HDF5, manifest, or view path...", + placeholder=( + "HDF5, LArCV ROOT, manifest, " + "view path, or HTTPS URL..." + ), disabled=False, required=True, autoComplete="on", @@ -188,8 +192,8 @@ def entry_controls(): html.Label( [ html.Span( - "Choose an HDF5, manifest, " - "or view file…", + "Choose an HDF5, LArCV ROOT, " + "manifest, or view file…", id="upload-source-status", className="source-file-name", ), @@ -221,6 +225,25 @@ def entry_controls(): disabled=False, className="primary-button source-open-button", ), + html.Div( + [ + html.Label( + "Choose LArCV converter", + className="field-label", + ), + dcc.Dropdown( + id="dropdown-larcv-config", + options=converter_options, + value=None, + clearable=True, + placeholder="Detector conversion bundle…", + className="control-dropdown", + ), + ], + id="larcv-converter-row", + className="larcv-converter-popover", + hidden=True, + ), html.Div( id="source-open-summary", className="source-open-summary", @@ -1296,6 +1319,7 @@ def main_layout(): dcc.Store(id="store-loaded-event"), dcc.Store(id="store-source-memory"), dcc.Store(id="store-source-request"), + dcc.Store(id="store-larcv-request"), dcc.Store(id="store-dropdown-commit"), dcc.Store(id="store-dropdown-pending"), dcc.Store(id="store-attribute-options"), diff --git a/src/spinal_tap/source.py b/src/spinal_tap/source.py index ecbe660..ba13e2d 100644 --- a/src/spinal_tap/source.py +++ b/src/spinal_tap/source.py @@ -20,13 +20,19 @@ def read_file_manifest(file_path: str | Path) -> list[str]: def detect_source_file(file_path: str | Path) -> str: - """Identify an exact local file as HDF5, JSON, or a path manifest.""" + """Identify an exact local file as HDF5, LArCV ROOT, JSON, or a manifest.""" import h5py file_path = Path(file_path) if h5py.is_hdf5(file_path): return "hdf5" + # ROOT files start with the literal ``root`` magic. Content detection keeps + # LArCV support independent of filename conventions such as ``.root``. + with file_path.open("rb") as source: + if source.read(4) == b"root": + return "larcv" + try: with open(file_path, encoding="utf-8-sig") as source: json.load(source) @@ -36,5 +42,5 @@ def detect_source_file(file_path: str | Path) -> str: return "manifest" except UnicodeDecodeError as error: raise ValueError( - "The source is not HDF5, JSON, or a UTF-8 file manifest." + "The source is not HDF5, LArCV ROOT, JSON, or a UTF-8 file manifest." ) from error diff --git a/src/spinal_tap/utils.py b/src/spinal_tap/utils.py index 16768bf..bd9c232 100644 --- a/src/spinal_tap/utils.py +++ b/src/spinal_tap/utils.py @@ -6,11 +6,13 @@ from pathlib import Path from threading import RLock from typing import Any, Dict, Iterable, Optional, Tuple +from urllib.parse import unquote, urlsplit from spine.constants import NuInteractionScheme from spine.construct import BuildManager from spine.io.read import HDF5Reader +from .converters import resolve_larcv_converter from .source import detect_source_file, read_file_manifest S3DF_DATA_ROOT = "/sdf/data/neutrino" @@ -22,6 +24,55 @@ _CACHE_LOCK = RLock() +def is_remote_root_hint(source: str) -> bool: + """Return whether an HTTP(S) URL advertises a ROOT file in its path. + + This is only a UI hint used to request a LArCV converter before a remote + file is downloaded. Source dispatch still validates the cached content. + """ + try: + parsed = urlsplit((source or "").strip()) + except (TypeError, ValueError): + return False + + return ( + parsed.scheme.lower() in {"http", "https"} + and bool(parsed.netloc) + and unquote(parsed.path).lower().endswith(".root") + ) + + +class LArCVDataReader: + """Expose a parsed LArCV dataset through spinal-tap's reader contract.""" + + backend = "larcv" + + def __init__(self, dataset: Any, cfg: dict[str, Any]) -> None: + self.dataset = dataset + self.reader = dataset.reader + self.cfg = cfg + self.file_paths = self.reader.file_paths + self.entry_index = self.reader.entry_index + self.run_info = getattr(self.reader, "run_info", None) + self.object_defaults: dict[str, dict[str, Any]] = {} + + writer = cfg.get("io", {}).get("writer", {}) + self.data_products = set(writer.get("keys", ())) + + def __len__(self) -> int: + return len(self.dataset) + + def get(self, idx: int) -> dict[str, Any]: + return self.dataset[idx] + + def get_run_event_index(self, run: int, subrun: int, event: int) -> int: + return self.reader.get_run_event_index(run, subrun, event) + + @staticmethod + def is_remote_path(path: str) -> bool: + return HDF5Reader.is_remote_path(path) + + def _configure_reader_object_defaults(reader: HDF5Reader) -> None: """Apply provenance defaults inferred from embedded detector geometry.""" cfg = getattr(reader, "cfg", None) or {} @@ -43,6 +94,50 @@ def _configure_reader_object_defaults(reader: HDF5Reader) -> None: reader.object_defaults = defaults +def _validate_larcv_schema_trees( + file_keys: tuple[str, ...], schema: dict[str, Any], dtype: str +) -> None: + """Reject a converter/file mismatch before ROOT constructs noisy chains.""" + from spine.config.factory import instantiate + from spine.io.dataset.larcv import PARSER_DICT + from spine.utils.conditional import ROOT + + tree_keys = { + key + for parser_cfg in schema.values() + for key in instantiate( + PARSER_DICT, parser_cfg, alt_name="parser", dtype=dtype + ).tree_keys + } + required = {f"{key}_tree" for key in tree_keys} + + missing_by_file = [] + for file_path in file_keys: + root_file = ROOT.TFile.Open(file_path, "READ") + if not root_file: + raise OSError(f"Could not open LArCV ROOT file: {file_path}") + try: + if root_file.IsZombie(): + raise OSError(f"Could not open LArCV ROOT file: {file_path}") + available = {key.GetName() for key in root_file.GetListOfKeys()} + missing = sorted(required - available) + if missing: + missing_by_file.append((file_path, missing)) + finally: + root_file.Close() + + if missing_by_file: + details = "; ".join( + f"{Path(file_path).name}: {', '.join(missing)}" + for file_path, missing in missing_by_file + ) + raise ValueError( + "The selected LArCV converter is incompatible with the source. " + f"Missing required tree(s): {details}. Choose a converter matching " + "this detector and production." + ) + + def canonicalize_data_path(file_path: str) -> str: """Canonicalize equivalent S3DF and container data paths. @@ -161,6 +256,54 @@ def _initialize_reader_cached( return reader +@lru_cache(maxsize=READER_CACHE_SIZE) +def _initialize_larcv_reader_cached( + file_keys: tuple[str, ...], + use_run: bool, + file_signature: tuple[tuple[str, int, int], ...], + config_path: str, +) -> LArCVDataReader: + """Construct one parsed LArCV reader for an unchanged source/config pair.""" + del file_signature + + from spine.config import load_config_file + from spine.geo import GeoManager + from spine.io.dataset import LArCVDataset + + cfg = load_config_file(config_path, download=False) + if "geo" in cfg: + GeoManager.initialize_or_get(**cfg["geo"]) + dataset_cfg = dict(cfg["io"]["loader"]["dataset"]) + dataset_cfg.pop("name", None) + dataset_cfg.pop("file_keys", None) + + schema = dataset_cfg.pop("schema") + dtype = cfg.get("base", {}).get("dtype", "float32") + run_info_key = None + if use_run: + run_cfg = schema.get("run_info", {}) + run_info_key = run_cfg.get("sparse_event") + if run_info_key is None: + raise ValueError( + "The selected LArCV conversion schema has no run-info source." + ) + + _validate_larcv_schema_trees(file_keys, schema, dtype) + + keys: str | list[str] = file_keys[0] if len(file_keys) == 1 else list(file_keys) + dataset = LArCVDataset( + file_keys=keys, + schema=schema, + dtype=dtype, + create_run_map=use_run, + run_info_key=run_info_key, + **dataset_cfg, + ) + reader = LArCVDataReader(dataset, cfg) + _configure_reader_object_defaults(reader) + return reader + + def resolve_source_path(source: str) -> str: """Materialize a URL or resolve an equivalent local data path.""" if source.strip().startswith(("http://", "https://")): @@ -207,10 +350,28 @@ def resolve_reader_keys(source: str) -> tuple[tuple[str, ...], tuple]: return keys, get_file_signature(keys) -def initialize_reader(file_path: str, use_run: bool = False) -> HDF5Reader: - """Initialize the HDF5 reader. +def _classify_reader_keys(file_keys: tuple[str, ...]) -> str: + """Return the common data kind for resolved reader inputs.""" + source_kinds = { + detect_source_file(path) for path in file_keys if os.path.isfile(path) + } + if len(source_kinds) > 1: + raise ValueError("A source collection cannot mix HDF5 and LArCV files.") + return next(iter(source_kinds), "hdf5") + + +def classify_reader_source(source: str) -> str: + """Return the effective data kind behind an exact source or manifest.""" + file_keys, _ = resolve_reader_keys(source) + return _classify_reader_keys(file_keys) + - TODO: add option to read from LArCV files (more tricky) +def initialize_reader( + file_path: str, + use_run: bool = False, + larcv_config: str | None = None, +) -> HDF5Reader | LArCVDataReader: + """Initialize an HDF5 reader or a configured LArCV parser dataset. Parameters ---------- @@ -221,17 +382,32 @@ def initialize_reader(file_path: str, use_run: bool = False) -> HDF5Reader: Returns ------- - HDF5Reader + HDF5Reader or LArCVDataReader File reader """ file_keys, signature = resolve_reader_keys(file_path) + source_kind = _classify_reader_keys(file_keys) + # Reader construction and persistent HDF5 handles are shared per process with _CACHE_LOCK: + if source_kind == "larcv": + config_path = larcv_config or os.getenv("SPINAL_TAP_LARCV_CONFIG") + if not config_path: + raise ValueError( + "LArCV input requires a spine-prod conversion bundle. Set " + "SPINAL_TAP_LARCV_CONFIG or select a detector converter." + ) + return _initialize_larcv_reader_cached( + file_keys, + use_run, + signature, + resolve_larcv_converter(config_path), + ) return _initialize_reader_cached(file_keys, use_run, signature) -def get_reader_products(reader: HDF5Reader) -> set[str]: +def get_reader_products(reader: HDF5Reader | LArCVDataReader) -> set[str]: """Return the event products advertised by a SPINE reader configuration. Parameters @@ -244,6 +420,9 @@ def get_reader_products(reader: HDF5Reader) -> set[str]: set[str] Product names expected in each event. """ + if hasattr(reader, "data_products"): + return set(reader.data_products) + cfg = reader.cfg or {} writer = cfg.get("io", {}).get("writer", {}) products = set(writer.get("keys", [])) @@ -337,11 +516,20 @@ def _load_data(reader: HDF5Reader, entry: int, mode: str, obj: str) -> Tuple[ data = reader.get(entry) # Initialize the builder + build_mode = mode + if getattr(reader, "backend", None) == "larcv": + build_mode = reader.cfg.get("build", {}).get("mode", mode) + + is_larcv = getattr(reader, "backend", None) == "larcv" + build_interactions = obj == "interactions" or ( + is_larcv and "truth_interactions" in get_reader_products(reader) + ) + build_particles = obj in ["particles", "interactions"] or build_interactions builder = BuildManager( obj == "fragments", - obj in ["particles", "interactions"], - obj == "interactions", - mode=mode, + build_particles, + build_interactions, + mode=build_mode, ) # Process the entry through the builder @@ -367,11 +555,13 @@ def clear_data_caches() -> None: with _CACHE_LOCK: _load_data_cached.cache_clear() _initialize_reader_cached.cache_clear() + _initialize_larcv_reader_cached.cache_clear() def get_data_cache_info() -> dict[str, Any]: """Return process-local reader and event cache statistics.""" return { "readers": _initialize_reader_cached.cache_info(), + "larcv_readers": _initialize_larcv_reader_cached.cache_info(), "events": _load_data_cached.cache_info(), } diff --git a/src/spinal_tap/version.py b/src/spinal_tap/version.py index dd324c2..25bd108 100644 --- a/src/spinal_tap/version.py +++ b/src/spinal_tap/version.py @@ -1,3 +1,3 @@ """Module which stores the current software version.""" -__version__ = "1.1.2" +__version__ = "1.2.0" diff --git a/test/test_cache.py b/test/test_cache.py index f3e0f5b..b7e256d 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -216,7 +216,9 @@ def open(self, request, timeout): ) path = manager.fetch_url("https://example.org/file.h5") assert os.path.isfile(path) + initial_mtime = os.stat(path).st_mtime_ns assert manager.fetch_url("https://example.org/file.h5") == path + assert os.stat(path).st_mtime_ns == initial_mtime assert opens == [("https://example.org/file.h5", 30)] diff --git a/test/test_callbacks.py b/test/test_callbacks.py index 0679b76..9ba50da 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -967,6 +967,15 @@ def test_view_state_is_validated_and_gets_a_restore_revision(tmp_path): {"version": 1, "file": "/data/a.h5", "entry": -1}, "entry number", ), + ( + { + "version": 1, + "file": "/data/a.h5", + "entry": 0, + "larcv_converter": ["2x2/truth"], + }, + "LArCV converter", + ), ( { "version": 1, @@ -1144,6 +1153,8 @@ def test_scene_refresh_inputs_use_committed_multi_selects(): assert ("store-dropdown-commit", "data") in inputs assert ("input-file-path", "n_submit") in inputs + assert ("dropdown-larcv-config", "value") in inputs + assert ("store-larcv-request", "data") in states for field in ("input-entry", "input-run", "input-subrun", "input-event"): assert (field, "n_submit") in inputs assert ("radio-run-mode", "value") in inputs @@ -1371,8 +1382,7 @@ def test_registered_login_entry_and_source_callbacks(callback_app, monkeypatch): {"display": "none"}, {"display": "grid"}, ) - assert source("url")[0].startswith("HTTPS URL") - assert source("path")[0].startswith("HDF5") + assert "HTTPS URL" in source("path")[0] class FakeReader: @@ -1452,6 +1462,8 @@ def graph_arguments(**overrides): "draw_mode_2": ["split_scene", "sync"], "axes": ["axes"], "file_path": "/tmp/events.h5", + "larcv_converter": None, + "larcv_request": None, "source_mode": "path", "entry": 0, "run": None, @@ -1470,6 +1482,145 @@ def graph_arguments(**overrides): return values +def test_larcv_converter_is_requested_only_for_root_sources(callback_app, monkeypatch): + """Opening ROOT should reveal a fresh selector without occupying HDF5 UI.""" + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG", raising=False) + callback = registered_callback(callback_app, "request_larcv_converter") + options = [{"label": "2x2", "value": "2x2/truth_240819"}] + + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="button-load") + ) + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("larcv", path) + ) + request, hidden, value = callback( + 1, None, 0, None, "/tmp/events.root", "path", options + ) + assert request["file_path"] == "/tmp/events.root" + assert request["source_mode"] == "path" + assert request["revision"] + assert hidden is False + assert value is None + + monkeypatch.setattr( + callback_module, + "classify_source", + lambda path: (_ for _ in ()).throw(AssertionError("downloaded too early")), + ) + request, hidden, value = callback( + 1, + None, + 0, + None, + "https://example.org/events.ROOT?token=abc", + "path", + options, + ) + assert request["file_path"].startswith("https://") + assert hidden is False + assert value is None + + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("larcv", path) + ) + monkeypatch.setenv("SPINAL_TAP_LARCV_CONFIG", "2x2/truth_240819") + assert callback(1, None, 0, None, "/tmp/events.root", "path", options) == ( + None, + True, + None, + ) + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG") + + monkeypatch.setattr(callback_module, "classify_source", lambda path: ("hdf5", path)) + assert callback(1, None, 0, None, "/tmp/events.h5", "path", options) == ( + None, + True, + None, + ) + assert callback(1, None, 0, None, "", "path", options) == (None, True, None) + assert callback(1, None, 0, None, "/tmp/events.root", "path", []) == ( + None, + True, + None, + ) + + monkeypatch.setattr( + callback_module, + "classify_source", + lambda path: (_ for _ in ()).throw(ValueError("bad source")), + ) + assert callback(1, None, 0, None, "/tmp/bad", "path", options) == ( + None, + True, + None, + ) + + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="store-source-request") + ) + assert callback(1, None, 0, None, "/tmp/old", "path", options) == ( + None, + True, + None, + ) + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("larcv", path) + ) + request, hidden, _ = callback( + 1, + {"file_path": "/tmp/upload.root", "source_mode": "browse"}, + 0, + None, + "/tmp/old", + "path", + options, + ) + assert request["file_path"] == "/tmp/upload.root" + assert request["source_mode"] == "browse" + assert hidden is False + + monkeypatch.setattr( + callback_module, + "ctx", + SimpleNamespace(triggered_id="dropdown-larcv-config"), + ) + assert callback( + 1, + None, + 0, + "2x2/truth_240819", + "/tmp/events.root", + "path", + options, + ) == (no_update, True, no_update) + assert ( + callback(1, None, 0, None, "/tmp/events.root", "path", options) + == (no_update,) * 3 + ) + + +def test_larcv_converter_is_requested_for_root_manifests(callback_app, monkeypatch): + """A homogeneous LArCV manifest should stage one shared converter.""" + callback = registered_callback(callback_app, "request_larcv_converter") + options = [{"label": "2x2", "value": "2x2/truth_240819"}] + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG", raising=False) + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="button-load") + ) + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("manifest", path) + ) + monkeypatch.setattr(callback_module, "classify_reader_source", lambda path: "larcv") + + request, hidden, value = callback( + 1, None, 0, None, "/tmp/events.list", "path", options + ) + assert request["file_path"] == "/tmp/events.list" + assert hidden is False + assert value is None + + @pytest.fixture def graph_callback(callback_app, monkeypatch): """Prepare the graph callback with deterministic lightweight dependencies.""" @@ -1509,6 +1660,176 @@ def graph_callback(callback_app, monkeypatch): return registered_callback(callback_app, "update_graph") +def test_graph_waits_for_larcv_converter_then_loads(graph_callback, monkeypatch): + """ROOT Open should stage selection and the selected converter should load it.""" + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG", raising=False) + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("larcv", path) + ) + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="button-load") + ) + assert ( + graph_callback(**graph_arguments(file_path="/tmp/events.root")) + == (no_update,) * 17 + ) + + monkeypatch.setenv("SPINAL_TAP_LARCV_CONFIG", "2x2/truth_240819") + result = graph_callback(**graph_arguments(file_path="/tmp/events.root")) + assert isinstance(result[0], html.Div) + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG") + + request = {"file_path": "/tmp/events.root", "source_mode": "path"} + monkeypatch.setattr( + callback_module, + "ctx", + SimpleNamespace(triggered_id="dropdown-larcv-config"), + ) + assert ( + graph_callback(**graph_arguments(larcv_converter=None, larcv_request=request)) + == (no_update,) * 17 + ) + + result = graph_callback( + **graph_arguments( + file_path="/tmp/stale.h5", + larcv_converter="2x2/truth_240819", + larcv_request=request, + entry=1, + loaded_event={ + "file_path": "/tmp/previous.h5", + "entry": 1, + "use_run": False, + }, + ) + ) + assert isinstance(result[0], html.Div) + assert result[1] == 0 + assert result[14]["file_path"] == "/tmp/events.root" + assert result[14]["larcv_converter"] == "2x2/truth_240819" + + +def test_graph_waits_for_larcv_manifest_converter(graph_callback, monkeypatch): + """Opening a ROOT manifest should pause until one converter is selected.""" + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG", raising=False) + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("manifest", path) + ) + monkeypatch.setattr(callback_module, "classify_reader_source", lambda path: "larcv") + monkeypatch.setattr( + callback_module, "validate_manifest_access", lambda path: (True, None) + ) + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="button-load") + ) + + assert ( + graph_callback(**graph_arguments(file_path="/tmp/events.list")) + == (no_update,) * 17 + ) + + +def test_graph_defers_remote_root_download_until_converter_selection( + graph_callback, monkeypatch +): + """A remote .root hint should stage selection without fetching the file.""" + source = "https://example.org/events.root?token=abc" + request = {"file_path": source, "source_mode": "path"} + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG", raising=False) + monkeypatch.setattr( + callback_module, + "available_larcv_converters", + lambda: [{"label": "2x2", "value": "2x2/truth_240819"}], + ) + monkeypatch.setattr( + callback_module, + "classify_source", + lambda path: (_ for _ in ()).throw(AssertionError("downloaded too early")), + ) + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="button-load") + ) + + assert graph_callback(**graph_arguments(file_path=source)) == (no_update,) * 17 + + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("larcv", path) + ) + monkeypatch.setattr( + callback_module, + "ctx", + SimpleNamespace(triggered_id="dropdown-larcv-config"), + ) + result = graph_callback( + **graph_arguments( + file_path="/tmp/stale.h5", + larcv_converter="2x2/truth_240819", + larcv_request=request, + ) + ) + assert isinstance(result[0], html.Div) + assert result[14]["file_path"] == source + + +@pytest.mark.parametrize( + "trigger,file_path,source_mode,source_request", + [ + ("button-load", "/tmp/next.h5", "path", None), + ("button-load", "https://example.org/next.h5", "url", None), + ( + "store-source-request", + "/tmp/stale.h5", + "path", + {"file_path": "/tmp/browsed.h5", "source_mode": "browse"}, + ), + ], +) +def test_graph_resets_entry_when_opening_a_different_source( + graph_callback, + monkeypatch, + trigger, + file_path, + source_mode, + source_request, +): + """Path, URL, and Browse sources should start at entry zero.""" + monkeypatch.setattr(callback_module, "ctx", SimpleNamespace(triggered_id=trigger)) + current = {"file_path": "/tmp/current.h5", "entry": 1, "use_run": False} + + result = graph_callback( + **graph_arguments( + file_path=file_path, + source_mode=source_mode, + source_request=source_request, + entry=1, + loaded_event=current, + ) + ) + assert result[1] == 0 + assert result[14]["file_path"] == ( + source_request["file_path"] if source_request else file_path + ) + + +def test_graph_preserves_entry_when_reopening_the_same_source( + graph_callback, monkeypatch +): + """Reopening the active source should preserve an explicit entry.""" + monkeypatch.setattr( + callback_module, "ctx", SimpleNamespace(triggered_id="button-load") + ) + current = {"file_path": "/tmp/current.h5", "entry": 1, "use_run": False} + + result = graph_callback( + **graph_arguments( + file_path="/tmp/current.h5", + entry=1, + loaded_event=current, + ) + ) + assert result[1] == 1 + + def test_graph_callback_renders_webgl_and_plotly(graph_callback, monkeypatch): """Both renderers should use the same loaded event and control payload.""" monkeypatch.setattr( @@ -1993,9 +2314,10 @@ def test_graph_callback_fast_refresh_shortcuts(graph_callback, monkeypatch): ("access", "Access denied"), ("classify", "Could not open source"), ("manifest", "manifest denied"), + ("manifest-inspect", "Could not inspect source manifest"), ("view", "Could not load shared view"), ("missing", "File(s) not found"), - ("reader", "reader exploded"), + ("reader", "Could not initialize the source:\nRuntimeError: reader exploded"), ("load", "Could not load the selected event"), ], ) @@ -2027,6 +2349,18 @@ def test_graph_callback_reports_source_pipeline_failures( "validate_manifest_access", lambda path: (False, "manifest denied"), ) + elif failure == "manifest-inspect": + monkeypatch.setattr( + callback_module, "classify_source", lambda path: ("manifest", path) + ) + monkeypatch.setattr( + callback_module, "validate_manifest_access", lambda path: (True, None) + ) + monkeypatch.setattr( + callback_module, + "classify_reader_source", + lambda path: (_ for _ in ()).throw(ValueError("mixed inputs")), + ) elif failure == "view": monkeypatch.setattr( callback_module, "classify_source", lambda path: ("json", path) diff --git a/test/test_converters.py b/test/test_converters.py new file mode 100644 index 0000000..b7be2e0 --- /dev/null +++ b/test/test_converters.py @@ -0,0 +1,84 @@ +"""Tests for installed spine-prod conversion-bundle discovery.""" + +import os + +import pytest + +from spinal_tap.converters import ( + _converter_tags, + available_larcv_converters, + resolve_larcv_converter, +) + + +def test_converter_discovery_and_resolution(monkeypatch, tmp_path): + """Dated bundles should be listed and resolved by stable relative IDs.""" + first_root = tmp_path / "first" + second_root = tmp_path / "second" + first = first_root / "convert" / "2x2" / "truth_240819.yaml" + duplicate = second_root / "convert" / "2x2" / "truth_240819.yaml" + second = second_root / "convert" / "icarus" / "truth_240812.yaml" + ignored = second_root / "convert" / "icarus" / "data_240812.yaml" + for path in (first, duplicate, second, ignored): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}") + + monkeypatch.delenv("SPINAL_TAP_LARCV_CONFIG_ROOT", raising=False) + monkeypatch.setenv( + "SPINE_CONFIG_PATH", os.pathsep.join((str(first_root), str(second_root))) + ) + + assert available_larcv_converters() == [ + {"label": "2x2 · truth_240819", "value": "2x2/truth_240819"}, + {"label": "icarus · truth_240812", "value": "icarus/truth_240812"}, + ] + assert resolve_larcv_converter("2x2/truth_240819") == str(first.resolve()) + assert resolve_larcv_converter("icarus/truth_240812.yaml") == str(second.resolve()) + assert resolve_larcv_converter(str(second)) == str(second.resolve()) + + +def test_converter_options_inherit_spine_prod_tags(monkeypatch, tmp_path): + """Ambiguous detector vintages should use their config metadata labels.""" + root = tmp_path / "config" + convert = root / "convert" / "sbnd" + io = root / "infer" / "sbnd" / "io" + convert.mkdir(parents=True) + io.mkdir(parents=True) + (io / "io_240720.yaml").write_text( + "__meta__:\n kind: bundle\n tags: [without-crt, with-xa]\n" + ) + (convert / "truth_240720.yaml").write_text( + "__meta__:\n kind: bundle\ninclude: infer/sbnd/io/io_240720.yaml\n" + ) + (convert / "truth_250328.yaml").write_text( + "__meta__:\n kind: bundle\n tags: [with-crt, with-xa]\n" + ) + monkeypatch.setenv("SPINAL_TAP_LARCV_CONFIG_ROOT", str(root)) + + assert available_larcv_converters() == [ + { + "label": "sbnd · truth_240720 · without-crt, with-xa", + "value": "sbnd/truth_240720", + }, + { + "label": "sbnd · truth_250328 · with-crt, with-xa", + "value": "sbnd/truth_250328", + }, + ] + + cycle = root / "cycle.yaml" + cycle.write_text("include: cycle.yaml\n") + assert _converter_tags(cycle, [root]) == [] + malformed = root / "malformed.yaml" + malformed.write_text("[not valid yaml") + assert _converter_tags(malformed, [root]) == [] + + +def test_converter_root_override_and_missing_error(monkeypatch, tmp_path): + """The spinal-tap-specific root should override generic SPINE lookup.""" + monkeypatch.setenv("SPINE_CONFIG_PATH", str(tmp_path / "ignored")) + monkeypatch.setenv("SPINAL_TAP_LARCV_CONFIG_ROOT", str(tmp_path / "selected")) + + assert available_larcv_converters() == [] + with pytest.raises(FileNotFoundError, match="missing/truth_000000"): + resolve_larcv_converter("missing/truth_000000") diff --git a/test/test_layout.py b/test/test_layout.py index 6d81043..5e344cd 100644 --- a/test/test_layout.py +++ b/test/test_layout.py @@ -195,6 +195,7 @@ def test_layout_contains_refresh_state_stores(): assert component_by_id(layout, "store-loaded-event") is not None assert component_by_id(layout, "store-source-memory") is not None assert component_by_id(layout, "store-source-request") is not None + assert component_by_id(layout, "store-larcv-request") is not None assert component_by_id(layout, "store-dropdown-commit") is not None assert component_by_id(layout, "store-dropdown-pending") is not None assert component_by_id(layout, "store-attribute-options") is not None @@ -249,7 +250,9 @@ def test_layout_contains_shared_view_controls(): "run", ] assert export.disabled is True - assert file_path.placeholder == "HDF5, manifest, or view path..." + assert file_path.placeholder == ( + "HDF5, LArCV ROOT, manifest, view path, or HTTPS URL..." + ) assert component_by_id(layout, "upload-view-state") is None assert component_by_id(layout, "upload-view-state-data") is None assert component_by_id(actions, "share-menu") is None @@ -269,14 +272,20 @@ def test_local_data_controls_use_native_file_picker(): modes = component_by_id(controls, "source-mode") entry_mode = component_by_id(controls, "entry-mode") - assert [option["value"] for option in modes.options] == ["path", "browse", "url"] + assert [option["value"] for option in modes.options] == ["path", "browse"] + assert modes.options[0]["label"] == "Path" assert [option["value"] for option in entry_mode.options] == ["entry", "run"] assert all(option["disabled"] for option in entry_mode.options) assert component_by_id(controls, "upload-source-file").type == "file" assert ( component_by_id(controls, "upload-source-status").children - == "Choose an HDF5, manifest, or view file…" + == "Choose an HDF5, LArCV ROOT, manifest, or view file…" ) + assert component_by_id(controls, "dropdown-larcv-config") is not None + assert component_by_id(controls, "source-open-loading") is None + converter = component_by_id(controls, "larcv-converter-row") + assert converter.hidden is True + assert converter.className == "larcv-converter-popover" assert component_by_id(controls, "button-browse-source") is None assert component_by_id(controls, "button-source") is None @@ -325,7 +334,7 @@ def test_authenticated_data_controls_offer_upload_not_server_browse(monkeypatch) controls = main_layout() modes = component_by_id(controls, "source-mode") - assert [option["value"] for option in modes.options] == ["path", "upload", "url"] + assert [option["value"] for option in modes.options] == ["path", "upload"] def test_object_filters_live_in_the_viewer_toolbar(): diff --git a/test/test_source.py b/test/test_source.py index c285b55..a325d78 100644 --- a/test/test_source.py +++ b/test/test_source.py @@ -29,7 +29,7 @@ def test_read_manifest_rejects_empty_and_binary_files(tmp_path): def test_detect_source_file_by_content(tmp_path): - """HDF5, JSON and manifest sources should not depend on suffixes.""" + """HDF5, ROOT, JSON and manifest sources should not depend on suffixes.""" hdf5_path = tmp_path / "hdf5.data" with h5py.File(hdf5_path, "w") as output: output.create_dataset("events", data=[0]) @@ -37,12 +37,15 @@ def test_detect_source_file_by_content(tmp_path): json_path.write_text(json.dumps({"version": 1})) manifest_path = tmp_path / "manifest.data" manifest_path.write_text("events.h5\n") + root_path = tmp_path / "larcv.data" + root_path.write_bytes(b"root" + bytes(32)) assert detect_source_file(hdf5_path) == "hdf5" + assert detect_source_file(root_path) == "larcv" assert detect_source_file(json_path) == "json" assert detect_source_file(manifest_path) == "manifest" invalid = tmp_path / "invalid.data" invalid.write_bytes(b"\xff\xfe") - with pytest.raises(ValueError, match="not HDF5, JSON"): + with pytest.raises(ValueError, match="not HDF5, LArCV ROOT, JSON"): detect_source_file(invalid) diff --git a/test/test_utils.py b/test/test_utils.py index 487d0f4..0cb23e9 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -5,7 +5,9 @@ from spinal_tap import app from spinal_tap.callbacks import validate_file_access from spinal_tap.utils import ( + _validate_larcv_schema_trees, canonicalize_data_path, + classify_reader_source, classify_source, clear_data_caches, get_data_cache_info, @@ -13,6 +15,7 @@ get_file_signature, get_reader_products, initialize_reader, + is_remote_root_hint, load_data, resolve_data_path, resolve_reader_keys, @@ -20,6 +23,24 @@ ) +@pytest.mark.parametrize( + "source,expected", + [ + ("https://example.org/events.root", True), + ("https://example.org/events.ROOT?token=abc#entry", True), + ("http://example.org/a%2Eroot", True), + ("https://example.org/events.h5", False), + ("/tmp/events.root", False), + ("events.root", False), + ("not a URL", False), + ("https://[invalid/events.root", False), + ], +) +def test_remote_root_hint(source, expected): + """Only HTTP(S) URL paths ending in .root should prompt before download.""" + assert is_remote_root_hint(source) is expected + + def test_get_reader_products_uses_writer_configuration(): """Capability discovery should avoid loading an event when metadata exists.""" @@ -128,6 +149,237 @@ def make_reader(*args, **kwargs): clear_data_caches() +def test_initialize_larcv_reader_requires_conversion_bundle(tmp_path): + """LArCV input should explain how to supply its parser schema.""" + path = tmp_path / "events.data" + path.write_bytes(b"root" + bytes(32)) + + clear_data_caches() + with pytest.raises(ValueError, match="SPINAL_TAP_LARCV_CONFIG"): + initialize_reader(str(path)) + clear_data_caches() + + +def test_initialize_larcv_reader_uses_spine_prod_bundle(monkeypatch, tmp_path): + """A conversion bundle should configure parsing and the reader adapter.""" + from spine.geo import GeoManager + from spine.io import dataset as dataset_module + + path = tmp_path / "events.data" + path.write_bytes(b"root" + bytes(32)) + config_path = tmp_path / "truth.yaml" + config_path.write_text("placeholder") + calls = [] + + cfg = { + "base": {"dtype": "float32"}, + "geo": {"detector": "2x2"}, + "build": { + "mode": "truth", + "fragments": False, + "particles": True, + "interactions": True, + }, + "io": { + "loader": { + "dataset": { + "name": "larcv", + "file_keys": None, + "schema": { + "run_info": { + "parser": "run_info", + "sparse_event": "sparse3d_data", + }, + "particles": { + "parser": "particle", + "particle_event": "particle_data", + }, + }, + } + }, + "writer": { + "keys": [ + "run_info", + "meta", + "points_label", + "truth_particles", + "truth_interactions", + ] + }, + }, + } + + class BackendReader: + file_paths = [str(path)] + entry_index = [0, 1] + run_info = [(1, 2, 3), (1, 2, 4)] + + def get_run_event_index(self, run, subrun, event): + return self.run_info.index((run, subrun, event)) + + class Dataset: + reader = BackendReader() + data_keys = ("index", "run_info", "particles") + + def __init__(self, **kwargs): + calls.append(kwargs) + + def __len__(self): + return 2 + + def __getitem__(self, idx): + return {"index": idx} + + monkeypatch.setattr("spine.config.load_config_file", lambda *args, **kwargs: cfg) + geometry_calls = [] + monkeypatch.setattr( + GeoManager, + "initialize_or_get", + lambda **kwargs: geometry_calls.append(kwargs), + ) + monkeypatch.setattr(dataset_module, "LArCVDataset", Dataset) + monkeypatch.setattr( + "spinal_tap.utils._validate_larcv_schema_trees", lambda *args: None + ) + + clear_data_caches() + reader = initialize_reader(str(path), use_run=True, larcv_config=str(config_path)) + + assert len(reader) == 2 + assert reader.get(1) == {"index": 1} + assert reader.get_run_event_index(1, 2, 4) == 1 + assert reader.is_remote_path(str(path)) is False + assert reader.cfg is cfg + assert geometry_calls == [{"detector": "2x2"}] + assert get_reader_products(reader) == { + "run_info", + "meta", + "points_label", + "truth_particles", + "truth_interactions", + } + assert calls == [ + { + "file_keys": str(path), + "schema": cfg["io"]["loader"]["dataset"]["schema"], + "dtype": "float32", + "create_run_map": True, + "run_info_key": "sparse3d_data", + } + ] + clear_data_caches() + + +def test_validate_larcv_schema_trees_reports_converter_mismatch(monkeypatch): + """Missing converter inputs should be reported before TChain creation.""" + from spine.config import factory + from spine.utils import conditional + + closed = [] + + class Parser: + tree_keys = ("sparse3d_data", "opflash_light") + + class Key: + def __init__(self, name): + self.name = name + + def GetName(self): + return self.name + + class RootFile: + def __init__(self, names, zombie=False): + self.names = names + self.zombie = zombie + + def IsZombie(self): + return self.zombie + + def GetListOfKeys(self): + return [Key(name) for name in self.names] + + def Close(self): + closed.append(self) + + files = { + "/data/good.root": RootFile(["sparse3d_data_tree", "opflash_light_tree"]), + "/data/bad.root": RootFile(["sparse3d_data_tree"]), + "/data/zombie.root": RootFile([], zombie=True), + "/data/unreadable.root": None, + } + root = type( + "Root", + (), + {"TFile": type("TFile", (), {"Open": staticmethod(files.get)})}, + ) + monkeypatch.setattr(factory, "instantiate", lambda *args, **kwargs: Parser()) + monkeypatch.setattr(conditional, "ROOT", root) + schema = {"data": {"parser": "sparse3d"}} + + _validate_larcv_schema_trees(("/data/good.root",), schema, "float32") + with pytest.raises(ValueError, match="bad.root: opflash_light_tree"): + _validate_larcv_schema_trees(("/data/bad.root",), schema, "float32") + with pytest.raises(OSError, match="zombie.root"): + _validate_larcv_schema_trees(("/data/zombie.root",), schema, "float32") + with pytest.raises(OSError, match="unreadable.root"): + _validate_larcv_schema_trees(("/data/unreadable.root",), schema, "float32") + assert closed == [ + files["/data/good.root"], + files["/data/bad.root"], + files["/data/zombie.root"], + ] + + +def test_initialize_larcv_reader_requires_run_info(monkeypatch, tmp_path): + """Run-based navigation should reject schemas without a run-info source.""" + path = tmp_path / "events.root" + path.write_bytes(b"root" + bytes(32)) + config_path = tmp_path / "truth.yaml" + config_path.write_text("placeholder") + cfg = { + "io": { + "loader": {"dataset": {"name": "larcv", "schema": {}}}, + "writer": {"keys": []}, + } + } + monkeypatch.setattr("spine.config.load_config_file", lambda *args, **kwargs: cfg) + + clear_data_caches() + with pytest.raises(ValueError, match="no run-info source"): + initialize_reader(str(path), use_run=True, larcv_config=str(config_path)) + clear_data_caches() + + +def test_initialize_reader_rejects_mixed_larcv_collection(tmp_path): + """One manifest cannot combine already-converted and raw event files.""" + import h5py + + converted = tmp_path / "converted.h5" + with h5py.File(converted, "w") as output: + output.create_dataset("events", data=[0]) + raw = tmp_path / "raw.root" + raw.write_bytes(b"root" + bytes(32)) + manifest = tmp_path / "mixed.list" + manifest.write_text(f"{converted.name}\n{raw.name}\n") + + clear_data_caches() + with pytest.raises(ValueError, match="cannot mix HDF5 and LArCV"): + initialize_reader(str(manifest)) + clear_data_caches() + + +def test_classify_reader_source_resolves_larcv_manifest(tmp_path): + """A homogeneous ROOT manifest should expose its effective reader kind.""" + first = tmp_path / "first.root" + second = tmp_path / "second.root" + first.write_bytes(b"root" + bytes(32)) + second.write_bytes(b"root" + bytes(32)) + manifest = tmp_path / "events.list" + manifest.write_text(f"{first.name}\n{second.name}\n") + + assert classify_reader_source(str(manifest)) == "larcv" + + @pytest.mark.parametrize("detector", ["2x2", "2x2-single", "ND-LAr", "fsd"]) def test_initialize_reader_defaults_legacy_dune_schemes( monkeypatch, tmp_path, detector @@ -264,6 +516,75 @@ def __call__(self, data): clear_data_caches() +def test_load_data_builds_larcv_interactions_for_particle_vertices(monkeypatch): + """LArCV particle views should build interactions required by vertices.""" + + class Reader: + backend = "larcv" + cfg = { + "build": {"mode": "truth"}, + "geo": {"detector": "2x2"}, + "io": {"writer": {"keys": ["truth_particles", "truth_interactions"]}}, + } + + def get(self, entry): + return {"index": entry} + + calls = [] + + class Builder: + def __init__(self, *args, **kwargs): + calls.append((args, kwargs)) + + def __call__(self, data): + pass + + clear_data_caches() + monkeypatch.setattr("spinal_tap.utils.BuildManager", Builder) + load_data(Reader(), 0, "both", "particles") + + assert calls == [((False, True, True), {"mode": "truth"})] + clear_data_caches() + + +def test_load_data_closes_larcv_fragment_build_dependencies(monkeypatch): + """LArCV fragment views should retain particle/interaction dependencies.""" + + class Reader: + backend = "larcv" + cfg = { + "build": {"mode": "truth"}, + "io": { + "writer": { + "keys": [ + "truth_fragments", + "truth_particles", + "truth_interactions", + ] + } + }, + } + + def get(self, entry): + return {"index": entry} + + calls = [] + + class Builder: + def __init__(self, *args, **kwargs): + calls.append((args, kwargs)) + + def __call__(self, data): + pass + + clear_data_caches() + monkeypatch.setattr("spinal_tap.utils.BuildManager", Builder) + load_data(Reader(), 0, "truth", "fragments") + + assert calls == [((True, True, True), {"mode": "truth"})] + clear_data_caches() + + def test_resolve_url_and_reader_source_kinds(monkeypatch, tmp_path): """URLs, JSON views, globs and missing paths should dispatch correctly.""" monkeypatch.setattr(