Fix Lunaris repository discovery and harvesting

backend/app/services/ogm_harvest/aardvark_reader.py

@@ -5,6 +5,8 @@
 from pathlib import Path
 from typing import Dict, Generator, Iterable
 
+from app.services.ogm_harvest.layout import metadata_root
+
 
 @dataclass(frozen=True)
 class AardvarkRecordRef:
@@ -13,11 +15,17 @@ class AardvarkRecordRef:
 
 
 def iter_aardvark_json_files(repo_dir: Path) -> Iterable[Path]:
-    """Yield Aardvark JSON files from metadata-aardvark directories."""
-    base = repo_dir / "metadata-aardvark"
+    """Yield JSON candidates from the repository's supported metadata root."""
+    base = repo_dir / metadata_root(repo_dir.name)
     if not base.exists():
         return []
-    return (p for p in base.rglob("*.json") if p.name != "layers.json")
+    return (
+        p
+        for p in base.rglob("*.json")
+        if p.is_file()
+        and p.name != "layers.json"
+        and not any(part.startswith(".") for part in p.relative_to(base).parts)
+    )
 
 
 def read_aardvark_records(repo_dir: Path) -> Generator[AardvarkRecordRef, None, None]:
@@ -45,6 +53,9 @@ def read_aardvark_records(repo_dir: Path) -> Generator[AardvarkRecordRef, None,
 
         rel = str(path.relative_to(repo_dir))
         for record in records:
+            # Root layouts can contain unrelated JSON or older schema exports.
+            if metadata_root(repo_dir.name) == "." and record.get("gbl_mdVersion_s") != "Aardvark":
+                continue
             yield AardvarkRecordRef(record=record, source_path=rel)
 
 

backend/app/services/ogm_harvest/layout.py

@@ -0,0 +1,9 @@
+"""Metadata locations shared by GitHub discovery and checkout harvesting."""
+
+
+def metadata_root(repo_name: str) -> str:
+    # Lunaris publishes Aardvark JSON directly under provider directories.
+    # Keep this explicit: scanning arbitrary utility repos would ingest fixtures.
+    if repo_name == "ca.lunaris":
+        return "."
+    return "metadata-aardvark"

backend/scripts/populate_ogm_repos.py

@@ -6,7 +6,7 @@
 
 This script:
 - lists repos in a GitHub org via the REST API
-- checks whether each repo has a top-level `metadata-aardvark/` directory
+- checks whether each repo has its supported metadata directory
 - upserts into Postgres table `ogm_repos`
 - flags repos missing aardvark via `ogm_tags["ogm_missing_aardvark"] = true`
 
@@ -29,6 +29,8 @@
 from sqlalchemy import create_engine
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
+from app.services.ogm_harvest.layout import metadata_root
+
 # Keep script self-contained: import the SQLAlchemy Table definitions.
 from db.models import ogm_repos
 
@@ -119,7 +121,9 @@ def repo_has_metadata_aardvark(
     org: str, repo_name: str, default_branch: Optional[str], token: Optional[str]
 ) -> bool:
     # GitHub contents API for a directory returns a JSON array (200) if present, 404 if missing.
-    url = f"https://api.github.com/repos/{org}/{repo_name}/contents/metadata-aardvark"
+    root = metadata_root(repo_name)
+    suffix = "" if root == "." else f"/{root}"
+    url = f"https://api.github.com/repos/{org}/{repo_name}/contents{suffix}"
     params = {}
     if default_branch:
         params["ref"] = default_branch
@@ -131,7 +135,7 @@ def repo_has_metadata_aardvark(
         return False
     # Treat other errors as non-fatal but recordable upstream; caller can decide.
     raise RuntimeError(
-        f"GitHub API error checking metadata-aardvark for {org}/{repo_name}: "
+        f"GitHub API error checking metadata root {root} for {org}/{repo_name}: "
         f"{resp.status_code} {resp.text[:500]}"
     )
 
@@ -155,7 +159,7 @@ def build_repo_row(repo: Dict[str, Any], *, has_aardvark: bool) -> Dict[str, Any
     }
 
     # Policy:
-    # - if no aardvark directory, disable by default (so it won't be harvested)
+    # - if no supported metadata directory, disable by default
     # - otherwise enable and default to both webhook + nightly reconciliation
     #   (can be edited via admin endpoint)
     ogm_enabled = bool(has_aardvark and not archived)

backend/tests/scripts/test_populate_ogm_repos.py

@@ -20,6 +20,37 @@ def setup_function():
     populate_ogm_repos._REJECTED_GITHUB_TOKENS.clear()
 
 
+@pytest.mark.parametrize("name,suffix", [("ca.lunaris", ""), ("edu.example", "/metadata-aardvark")])
+def test_discovery_uses_supported_metadata_root(monkeypatch, name, suffix):
+    def fake_get(url, token, params, timeout):
+        assert url == f"https://api.github.com/repos/OpenGeoMetadata/{name}/contents{suffix}"
+        assert params == {"ref": "master"}
+        return FakeResponse(200, [{"name": "provider", "type": "dir"}])
+
+    monkeypatch.setattr(populate_ogm_repos, "_github_get", fake_get)
+    has_aardvark = populate_ogm_repos.repo_has_metadata_aardvark(
+        "OpenGeoMetadata", name, "master", None
+    )
+    row = populate_ogm_repos.build_repo_row(
+        {"name": name, "archived": False}, has_aardvark=has_aardvark
+    )
+    assert row["ogm_enabled"] is True
+    assert row["ogm_watch_mode"] == "both"
+
+
+def test_missing_standard_directory_remains_disabled(monkeypatch):
+    monkeypatch.setattr(populate_ogm_repos, "_github_get", lambda *a, **kw: FakeResponse(404, {}))
+    has_aardvark = populate_ogm_repos.repo_has_metadata_aardvark(
+        "OpenGeoMetadata", "utility", "main", None
+    )
+    assert (
+        populate_ogm_repos.build_repo_row({"name": "utility"}, has_aardvark=has_aardvark)[
+            "ogm_enabled"
+        ]
+        is False
+    )
+
+
 def test_list_org_repos_retries_without_rejected_token(monkeypatch, capsys):
     responses = [
         FakeResponse(401, {"message": "Bad credentials"}),

backend/tests/services/test_aardvark_reader.py

@@ -0,0 +1,48 @@
+import json
+
+from app.services.ogm_harvest.aardvark_reader import read_aardvark_records
+
+
+def write_json(repo, path, value):
+    target = repo / path
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(value))
+
+
+def test_lunaris_provider_layout_filters_records_and_preserves_provenance(tmp_path):
+    repo = tmp_path / "ca.lunaris"
+    record = {"id": "transit", "gbl_mdVersion_s": "Aardvark"}
+    write_json(repo, "abacus-open-data/transit.json", record)
+    write_json(
+        repo,
+        "another-provider/nested/records.json",
+        [
+            {"id": "second", "gbl_mdVersion_s": "Aardvark"},
+            {"id": "legacy", "geoblacklight_version": "1.0"},
+            "not a record",
+        ],
+    )
+    write_json(repo, "package.json", {"name": "unrelated"})
+    write_json(repo, ".git/hidden.json", record)
+    write_json(repo, "abacus-open-data/layers.json", record)
+    (repo / "broken.json").write_text("not json")
+
+    refs = sorted(read_aardvark_records(repo), key=lambda ref: ref.record["id"])
+    assert [ref.record["id"] for ref in refs] == ["second", "transit"]
+    assert refs[1].source_path == "abacus-open-data/transit.json"
+
+
+def test_standard_layout_keeps_existing_behavior(tmp_path):
+    repo = tmp_path / "edu.example"
+    record = {"id": "existing"}
+    write_json(repo, "metadata-aardvark/nested/record.json", record)
+    write_json(repo, "fixtures/unrelated.json", {"id": "fixture", "gbl_mdVersion_s": "Aardvark"})
+    refs = list(read_aardvark_records(repo))
+    assert [ref.record for ref in refs] == [record]
+    assert refs[0].source_path == "metadata-aardvark/nested/record.json"
+
+
+def test_unknown_repo_does_not_fall_back_to_root(tmp_path):
+    repo = tmp_path / "utility"
+    write_json(repo, "fixtures/record.json", {"id": "fixture", "gbl_mdVersion_s": "Aardvark"})
+    assert list(read_aardvark_records(repo)) == []

docs/harvester.md

@@ -64,6 +64,13 @@ The harvester denylist automatically excludes non-metadata repositories such as
 
 ## Related Backend Flows
 
+Application discovery and harvesting share explicit metadata locations. Most
+repositories use `metadata-aardvark/`; `ca.lunaris` uses the repository root with
+records nested under provider directories. For that root layout, the reader
+accepts only records declaring `gbl_mdVersion_s: Aardvark` and skips hidden paths
+and `layers.json`. After deploying this support, the next repository discovery
+refresh enables Lunaris and the nightly pipeline harvests its records.
+
 For the full application workflow, these backend scripts usually matter more than the standalone harvester:
 
 - `scripts/populate_ogm_repos.py`