diff --git a/backend/app/api/v1/endpoint_modules/ogm.py b/backend/app/api/v1/endpoint_modules/ogm.py index 5d8d1f5..af5c9d3 100644 --- a/backend/app/api/v1/endpoint_modules/ogm.py +++ b/backend/app/api/v1/endpoint_modules/ogm.py @@ -51,29 +51,24 @@ async def ogm_repo_dashboard(request: Request): repos = await ogm_repo.list_public_repo_summaries() dashboard_repos = [] - total_harvested = 0 - total_available = 0 + counts = await ogm_repo.get_public_dashboard_counts() repos_with_aardvark = 0 enabled_repos = 0 never_harvested = 0 for repo in repos: - harvested_count = int(repo.get("harvested_record_count") or 0) - available_count = int(repo.get("available_record_count") or 0) unpublished_count = int(repo.get("unpublished_record_count") or 0) has_aardvark = bool(repo.get("ogm_has_aardvark")) - enabled = bool(repo.get("ogm_enabled")) + enabled = bool(repo.get("ogm_enabled")) and not repo.get("ogm_archived", False) last_harvest_completed = repo.get("last_crawl_completed_at") - hidden_count = max(harvested_count - available_count, 0) - other_hidden_count = max(hidden_count - unpublished_count, 0) api_hidden_breakdown = [] if unpublished_count: api_hidden_breakdown.append({"count": unpublished_count, "label": "unpublished"}) - if other_hidden_count: - api_hidden_breakdown.append({"count": other_hidden_count, "label": "not yet synced"}) + if repo.get("other_active_source_count"): + api_hidden_breakdown.append( + {"count": repo["other_active_source_count"], "label": "also in active sources"} + ) - total_harvested += harvested_count - total_available += available_count repos_with_aardvark += int(has_aardvark) enabled_repos += int(enabled) never_harvested += int(not bool(last_harvest_completed)) @@ -86,8 +81,13 @@ async def ogm_repo_dashboard(request: Request): "display_last_harvest_started_at": _format_timestamp( repo.get("last_crawl_started_at") ), - "harvest_gap_count": hidden_count, + "source_status": "Archived" + if repo.get("ogm_archived") + else ("Active" if enabled else "Disabled"), "api_hidden_breakdown": api_hidden_breakdown, + "aardvark_status": {True: "present", False: "missing", None: "unknown"}.get( + repo.get("ogm_has_aardvark"), "unknown" + ), } ) @@ -96,18 +96,21 @@ async def ogm_repo_dashboard(request: Request): "enabled_repo_count": enabled_repos, "repos_with_aardvark_count": repos_with_aardvark, "never_harvested_count": never_harvested, - "harvested_record_count": total_harvested, - "available_record_count": total_available, + "harvested_record_count": counts["active_record_count"], + "available_record_count": counts["published_record_count"], } if templates is None: rows = "".join( ( "" - f"{escape(str(repo.get('ogm_repo_name') or ''))}" + f'' + f"{escape(str(repo.get('ogm_repo_name') or ''))} " + f"({escape(repo['source_status'])}) " + f'Search API' f"{escape(str(repo.get('display_last_commit_at') or '-'))}" f"{escape(str(repo.get('display_last_harvest_at') or '-'))}" - f"{'yes' if repo.get('ogm_has_aardvark') else 'no'}" + f"{repo['aardvark_status']}" f"{int(repo.get('harvested_record_count') or 0)}" f"{int(repo.get('available_record_count') or 0)}" "" @@ -123,7 +126,8 @@ async def ogm_repo_dashboard(request: Request): "

Templates are unavailable, showing a minimal fallback view.

" "" "" - "" + "" + "" f"{rows}
RepositoryLast commitLast harvestAardvarkHarvestedAvailableAardvarkLast-seen source recordsPublished with source tag (database)
" ) ) diff --git a/backend/app/services/ogm_harvest/repository.py b/backend/app/services/ogm_harvest/repository.py index 169d12e..ea17fe0 100644 --- a/backend/app/services/ogm_harvest/repository.py +++ b/backend/app/services/ogm_harvest/repository.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional +from urllib.parse import quote, urlencode from sqlalchemy import String, cast, func, literal, select, true, update from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -10,6 +11,35 @@ from db.models import ogm_harvest_runs, ogm_repos, ogm_resource_state, resources +def _published_resources(): + return func.lower( + func.coalesce( + func.nullif(resources.c.b1g_publication_state_s, cast(literal(""), String())), + func.nullif(resources.c.publication_state, cast(literal(""), String())), + cast(literal("published"), String()), + ) + ) == cast(literal("published"), String()) + + +def _active_memberships(): + return ( + select( + ogm_resource_state.c.ogm_resource_id, + ogm_resource_state.c.ogm_repo_name, + ) + .select_from( + ogm_resource_state.join( + ogm_repos, ogm_resource_state.c.ogm_repo_name == ogm_repos.c.ogm_repo_name + ) + ) + .where( + ogm_resource_state.c.ogm_missing_since.is_(None), + ogm_repos.c.ogm_enabled.is_(True), + func.coalesce(ogm_repos.c.ogm_tags["ogm_archived"].as_boolean(), False).is_(False), + ) + ) + + class OGMHarvestRepository: """ Minimal async repository for OGM harvest tables. @@ -22,6 +52,26 @@ async def list_repos(self) -> List[Dict[str, Any]]: rows = await database.fetch_all(select(ogm_repos).order_by(ogm_repos.c.ogm_repo_name)) return [dict(r) for r in rows] + async def get_public_dashboard_counts(self) -> Dict[str, int]: + """Count unique active-source records and all published database records.""" + active = _active_memberships().subquery("active_memberships") + row = await database.fetch_one( + select( + select(func.count(func.distinct(active.c.ogm_resource_id))) + .select_from(active.join(resources, resources.c.id == active.c.ogm_resource_id)) + .scalar_subquery() + .label("active_record_count"), + select(func.count()) + .select_from(resources) + .where(_published_resources()) + .scalar_subquery() + .label("published_record_count"), + ) + ) + return { + key: int(row[key] or 0) for key in ("active_record_count", "published_record_count") + } + async def list_public_repo_summaries(self) -> List[Dict[str, Any]]: """ Public-facing OGM repo summaries. @@ -86,9 +136,32 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]: .subquery("resource_counts") ) + active = _active_memberships().subquery("active_memberships") + other_active_counts = ( + select( + ogm_resource_state.c.ogm_repo_name, + func.count(func.distinct(ogm_resource_state.c.ogm_resource_id)).label( + "other_active_source_count" + ), + ) + .select_from( + ogm_resource_state.join( + active, + (ogm_resource_state.c.ogm_resource_id == active.c.ogm_resource_id) + & (ogm_resource_state.c.ogm_repo_name != active.c.ogm_repo_name), + ) + ) + .where(ogm_resource_state.c.ogm_missing_since.is_(None)) + .group_by(ogm_resource_state.c.ogm_repo_name) + .subquery("other_active_counts") + ) + q = ( select( ogm_repos.c.ogm_repo_name, + func.coalesce(other_active_counts.c.other_active_source_count, 0).label( + "other_active_source_count" + ), ogm_repos.c.ogm_enabled, ogm_repos.c.ogm_watch_mode, ogm_repos.c.ogm_last_harvest_started_at, @@ -116,8 +189,10 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]: ) .select_from( ogm_repos.outerjoin( - latest_run_ids, ogm_repos.c.ogm_repo_name == latest_run_ids.c.repo_name + other_active_counts, + ogm_repos.c.ogm_repo_name == other_active_counts.c.ogm_repo_name, ) + .outerjoin(latest_run_ids, ogm_repos.c.ogm_repo_name == latest_run_ids.c.repo_name) .outerjoin( ogm_harvest_runs, ogm_harvest_runs.c.ogm_id == latest_run_ids.c.latest_ogm_id ) @@ -154,19 +229,25 @@ def _to_int(value: Any) -> int: "ogm_last_harvest_completed_at" ) repo_full_name = tags.get("ogm_repo_full_name") or item.get("ogm_repo_name") + if repo_full_name and "/" not in repo_full_name: + repo_full_name = f"OpenGeoMetadata/{repo_full_name}" has_aardvark = tags.get("ogm_has_aardvark") - if has_aardvark is None: - has_aardvark = not bool(tags.get("ogm_missing_aardvark")) + if has_aardvark is None and "ogm_missing_aardvark" in tags: + has_aardvark = not bool(tags["ogm_missing_aardvark"]) summaries.append( { "ogm_repo_name": item.get("ogm_repo_name"), "ogm_repo_full_name": repo_full_name, "ogm_github_url": ( - f"https://github.com/{repo_full_name}" if repo_full_name else None + f"https://github.com/{quote(repo_full_name, safe='/')}" + if repo_full_name + else None ), + "ogm_search_url": "/api/v1/search?" + + urlencode({"ogm_repo": item["ogm_repo_name"]}), "ogm_enabled": item.get("ogm_enabled"), "ogm_watch_mode": item.get("ogm_watch_mode"), - "ogm_has_aardvark": bool(has_aardvark), + "ogm_has_aardvark": has_aardvark, "ogm_default_branch": tags.get("ogm_default_branch"), "ogm_archived": bool(tags.get("ogm_archived", False)), "last_commit_at": tags.get("ogm_pushed_at"), @@ -178,6 +259,7 @@ def _to_int(value: Any) -> int: "harvested_success_count": _to_int(stats.get("imported")), "harvested_failure_count": _to_int(stats.get("errors")), "harvested_record_count": _to_int(item.get("harvested_record_count")), + "other_active_source_count": _to_int(item.get("other_active_source_count")), "available_record_count": _to_int(item.get("available_record_count")), "suppressed_record_count": _to_int(item.get("suppressed_record_count")), "unpublished_record_count": _to_int(item.get("unpublished_record_count")), diff --git a/backend/templates/ogm_repo_dashboard.html b/backend/templates/ogm_repo_dashboard.html index e6a13ba..5f07442 100644 --- a/backend/templates/ogm_repo_dashboard.html +++ b/backend/templates/ogm_repo_dashboard.html @@ -37,8 +37,9 @@

Repository health, harvest freshness, and API availability in one place.

This page tracks every repository discovered in the OpenGeoMetadata GitHub organization, whether it exposes a metadata-aardvark/ directory, when it last changed on GitHub, - when this app last harvested it, and how many records are currently harvested versus published - through this API. + when this app last harvested it, and how many records were last seen in each source. Archived and disabled + sources retain their harvest history. Publication counts come from the database; + this page does not verify the live search index.

Generated {{ generated_at }}. Times shown in UTC.

@@ -53,15 +54,15 @@

Repository health, harvest freshness, and API availability in one place.

{{ summary.repos_with_aardvark_count }}

-

Nightly Enabled

+

Harvesting Enabled

{{ summary.enabled_repo_count }}

-

Current Harvested Records

+

Unique Records in Active Sources

{{ "{:,}".format(summary.harvested_record_count) }}

-

Published in This API

+

Published Records (Database)

{{ "{:,}".format(summary.available_record_count) }}

@@ -74,6 +75,8 @@

Repository health, harvest freshness, and API availability in one place.

Repositories

+

Per-repository counts can overlap and should not be added together. + “Also in active sources” describes overlapping source records, not a synchronization failure.

JSON source: /api/v1/ogm/repos

@@ -86,9 +89,9 @@

Repositories

Last Commit Last Harvest Aardvark - Harvest Status - Harvested Records - API Visibility + Last Harvest Result + Last-Seen Source Records + Published with Source Tag (Database) @@ -98,6 +101,8 @@

Repositories

{{ repo.ogm_repo_name }}

{{ repo.ogm_repo_full_name or repo.ogm_repo_name }}

+

Search API

+

{{ repo.source_status }}

mode: {{ repo.ogm_watch_mode or "manual" }} {% if repo.last_commit_sha %} @@ -120,10 +125,12 @@

Repositories

{% endif %} - {% if repo.ogm_has_aardvark %} + {% if repo.ogm_has_aardvark is sameas true %} Present - {% else %} + {% elif repo.ogm_has_aardvark is sameas false %} Missing + {% else %} + Unknown {% endif %} @@ -155,8 +162,7 @@

Repositories

{{ "{:,}".format(item.count) }} {{ item.label }}{% if not loop.last %} · {% endif %} {% endfor %}

- {% elif repo.harvest_gap_count %} -

{{ "{:,}".format(repo.harvest_gap_count) }} hidden from API

+ {% endif %} diff --git a/backend/tests/api/v1/test_ogm_public_endpoints.py b/backend/tests/api/v1/test_ogm_public_endpoints.py index b0a5682..ff65b6d 100644 --- a/backend/tests/api/v1/test_ogm_public_endpoints.py +++ b/backend/tests/api/v1/test_ogm_public_endpoints.py @@ -1,8 +1,12 @@ from unittest.mock import AsyncMock, patch +from fastapi import FastAPI from fastapi.testclient import TestClient -from app.main import app +from app.api.v1.endpoint_modules import ogm + +app = FastAPI() +app.include_router(ogm.router, prefix="/api/v1") client = TestClient(app) @@ -41,6 +45,7 @@ def test_public_ogm_repo_dashboard_renders_html_monitor(): "ogm_repo_name": "edu.utexas", "ogm_repo_full_name": "OpenGeoMetadata/edu.utexas", "ogm_github_url": "https://github.com/OpenGeoMetadata/edu.utexas", + "ogm_search_url": "/api/v1/search?ogm_repo=edu.utexas", "ogm_enabled": True, "ogm_watch_mode": "nightly", "ogm_has_aardvark": True, @@ -58,11 +63,16 @@ def test_public_ogm_repo_dashboard_renders_html_monitor(): } ] - with patch( - "app.api.v1.endpoint_modules.ogm.ogm_repo.list_public_repo_summaries", - new_callable=AsyncMock, - ) as mock_summaries: - mock_summaries.return_value = sample_repos + with ( + patch.object( + ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=sample_repos) + ), + patch.object( + ogm.ogm_repo, + "get_public_dashboard_counts", + AsyncMock(return_value={"active_record_count": 904, "published_record_count": 1234}), + ), + ): response = client.get("/api/v1/ogm/repos/dashboard") assert response.status_code == 200 @@ -72,6 +82,8 @@ def test_public_ogm_repo_dashboard_renders_html_monitor(): assert "OpenGeoMetadata/edu.utexas" in response.text assert "/api/v1/ogm/repos" in response.text assert "3 unpublished" in response.text + assert 'href="https://github.com/OpenGeoMetadata/edu.utexas"' in response.text + assert 'href="/api/v1/search?ogm_repo=edu.utexas"' in response.text def test_public_ogm_failures_endpoint_lists_failed_runs(): @@ -155,3 +167,59 @@ def test_public_ogm_failures_endpoint_can_limit_to_hard_failures_only(): limit=50, offset=0, ) + + +def test_dashboard_distinguishes_retired_sources_and_uses_global_counts(): + repos = [ + { + "ogm_repo_name": "edu.umn", + "ogm_enabled": True, + "ogm_archived": True, + "last_crawl_status": "success", + "harvested_record_count": 6332, + "available_record_count": 0, + "unpublished_record_count": 441, + "other_active_source_count": 5891, + }, + { + "ogm_repo_name": "gov.usgs", + "ogm_enabled": False, + "harvested_record_count": 177789, + "available_record_count": 0, + "other_active_source_count": 177789, + }, + { + "ogm_repo_name": "geobtaa", + "ogm_enabled": True, + "harvested_record_count": 40703, + "available_record_count": 40703, + }, + ] + with ( + patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=repos)), + patch.object( + ogm.ogm_repo, + "get_public_dashboard_counts", + AsyncMock( + return_value={"active_record_count": 315135, "published_record_count": 318408} + ), + ), + ): + response = client.get("/api/v1/ogm/repos/dashboard") + assert response.status_code == 200 + html = response.text + for expected in ( + "Archived", + "Disabled", + "Active", + "315,135", + "318,408", + "5,891 also in active sources", + "177,789 also in active sources", + "Last Harvest Result", + "Published Records (Database)", + ): + assert expected in html + assert "not yet synced" not in html + assert "hidden from API" not in html + assert "does not verify the live search index" in html diff --git a/backend/tests/services/test_ogm_harvest_repository.py b/backend/tests/services/test_ogm_harvest_repository.py index 509966f..1fc757a 100644 --- a/backend/tests/services/test_ogm_harvest_repository.py +++ b/backend/tests/services/test_ogm_harvest_repository.py @@ -54,6 +54,12 @@ async def test_public_repo_counts_preserve_tag_and_publication_semantics( if not url: pytest.skip("Set OGM_QUERY_TEST_DATABASE_URL to run isolated PostgreSQL query tests") connection = await asyncpg.connect(url) + import json + + for name in ("json", "jsonb"): + await connection.set_type_codec( + name, schema="pg_catalog", encoder=json.dumps, decoder=json.loads + ) try: for table in (ogm_repos, ogm_harvest_runs, ogm_resource_state): ddl = str(CreateTable(table).compile(dialect=database._backend._dialect)) @@ -114,5 +120,77 @@ async def fetch_all(query): assert summaries[0]["last_crawl_status"] == "success" assert summaries[1]["ogm_enabled"] is False assert summaries[2]["last_run_id"] is None + # A second dataset covers migrations, overlapping active sources, missing + # records, and published records without repository attribution. + await connection.execute(""" + TRUNCATE ogm_resource_state, ogm_harvest_runs, ogm_repos, resources; + INSERT INTO ogm_repos (ogm_repo_name, ogm_enabled, ogm_tags) VALUES + ('live',true,'{}'), ('live2',true,'{}'), + ('archived',true,'{"ogm_archived":true}'), ('disabled',false,'{}'); + INSERT INTO resources VALUES + ('shared',ARRAY['ogm_repo:live','ogm_repo:live2'],'published','published',false), + ('migrated',ARRAY['ogm_repo:live'],'published','published',false), + ('retired',ARRAY['ogm_repo:archived'],'unpublished','unpublished',false), + ('untagged',NULL,NULL,NULL,false), + ('missing',ARRAY['ogm_repo:live'],'published','published',false), + ('archived-only',ARRAY['ogm_repo:archived'],'published','published',false), + ('disabled-only',ARRAY['ogm_repo:disabled'],'published','published',false), + ('draft',ARRAY['ogm_repo:live'],'DRAFT','published',false); + INSERT INTO ogm_resource_state (ogm_repo_name,ogm_resource_id,ogm_missing_since) VALUES + ('live','shared',NULL), ('live2','shared',NULL), + ('live','migrated',NULL), ('archived','migrated',NULL), + ('archived','retired',NULL), ('archived','archived-only',NULL), + ('disabled','disabled-only',NULL), ('live','missing',now()), + ('live','draft',NULL); + """) + + async def fetch_one(query): + rows = await fetch_all(query) + return rows[0] + + monkeypatch.setattr(ogm_repository.database, "fetch_one", fetch_one) + assert await OGMHarvestRepository().get_public_dashboard_counts() == { + "active_record_count": 3, + "published_record_count": 6, + } + rows = { + r["ogm_repo_name"]: r for r in await OGMHarvestRepository().list_public_repo_summaries() + } + assert rows["archived"]["other_active_source_count"] == 1 + assert rows["live"]["other_active_source_count"] == 1 + assert rows["live2"]["other_active_source_count"] == 1 + assert rows["disabled"]["other_active_source_count"] == 0 + assert rows["disabled"]["ogm_github_url"] == "https://github.com/OpenGeoMetadata/disabled" + assert rows["disabled"]["ogm_has_aardvark"] is None + assert rows["live"]["ogm_search_url"] == "/api/v1/search?ogm_repo=live" finally: await connection.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("full_name", [None, "edu.example", "OpenGeoMetadata/edu.example"]) +async def test_repository_links_use_org_and_active_source_filter(monkeypatch, full_name): + from unittest.mock import AsyncMock + from urllib.parse import parse_qs, urlparse + + from app.services.search_service import SearchService + + monkeypatch.setattr( + ogm_repository.database, + "fetch_all", + AsyncMock( + return_value=[ + { + "ogm_repo_name": "edu.example", + "ogm_tags": {"ogm_repo_full_name": full_name}, + } + ] + ), + ) + row = (await OGMHarvestRepository().list_public_repo_summaries())[0] + assert row["ogm_github_url"] == "https://github.com/OpenGeoMetadata/edu.example" + query = urlparse(row["ogm_search_url"]).query + assert parse_qs(query) == {"ogm_repo": ["edu.example"]} + included, excluded = SearchService().extract_new_style_filters(query) + assert included == {"ogm_repo": ["edu.example"]} + assert excluded == {}