Skip to content

Fix repository dashboard accuracy and source links #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ewlarson merged 1 commit into from
Oct 3, 2026
Merged

Fix repository dashboard accuracy and source links #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions backend/app/api/v1/endpoint_modules/ogm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,29 +51,24 @@ async def ogm_repo_dashboard(request: Request):
repos = await ogm_repo.list_public_repo_summaries()

dashboard_repos = []
total_harvested = 0
total_available = 0
counts = await ogm_repo.get_public_dashboard_counts()
repos_with_aardvark = 0
enabled_repos = 0
never_harvested = 0

for repo in repos:
harvested_count = int(repo.get("harvested_record_count") or 0)
available_count = int(repo.get("available_record_count") or 0)
unpublished_count = int(repo.get("unpublished_record_count") or 0)
has_aardvark = bool(repo.get("ogm_has_aardvark"))
enabled = bool(repo.get("ogm_enabled"))
enabled = bool(repo.get("ogm_enabled")) and not repo.get("ogm_archived", False)
last_harvest_completed = repo.get("last_crawl_completed_at")
hidden_count = max(harvested_count - available_count, 0)
other_hidden_count = max(hidden_count - unpublished_count, 0)
api_hidden_breakdown = []
if unpublished_count:
api_hidden_breakdown.append({"count": unpublished_count, "label": "unpublished"})
if other_hidden_count:
api_hidden_breakdown.append({"count": other_hidden_count, "label": "not yet synced"})
if repo.get("other_active_source_count"):
api_hidden_breakdown.append(
{"count": repo["other_active_source_count"], "label": "also in active sources"}
)

total_harvested += harvested_count
total_available += available_count
repos_with_aardvark += int(has_aardvark)
enabled_repos += int(enabled)
never_harvested += int(not bool(last_harvest_completed))
Expand All @@ -86,8 +81,13 @@ async def ogm_repo_dashboard(request: Request):
"display_last_harvest_started_at": _format_timestamp(
repo.get("last_crawl_started_at")
),
"harvest_gap_count": hidden_count,
"source_status": "Archived"
if repo.get("ogm_archived")
else ("Active" if enabled else "Disabled"),
"api_hidden_breakdown": api_hidden_breakdown,
"aardvark_status": {True: "present", False: "missing", None: "unknown"}.get(
repo.get("ogm_has_aardvark"), "unknown"
),
}
)

Expand All @@ -96,18 +96,21 @@ async def ogm_repo_dashboard(request: Request):
"enabled_repo_count": enabled_repos,
"repos_with_aardvark_count": repos_with_aardvark,
"never_harvested_count": never_harvested,
"harvested_record_count": total_harvested,
"available_record_count": total_available,
"harvested_record_count": counts["active_record_count"],
"available_record_count": counts["published_record_count"],
}

if templates is None:
rows = "".join(
(
"<tr>"
f"<td>{escape(str(repo.get('ogm_repo_name') or ''))}</td>"
f'<td><a href="{escape(repo["ogm_github_url"], quote=True)}">'
f"{escape(str(repo.get('ogm_repo_name') or ''))}</a> "
f"({escape(repo['source_status'])}) "
f'<a href="{escape(repo["ogm_search_url"], quote=True)}">Search API</a></td>'
f"<td>{escape(str(repo.get('display_last_commit_at') or '-'))}</td>"
f"<td>{escape(str(repo.get('display_last_harvest_at') or '-'))}</td>"
f"<td>{'yes' if repo.get('ogm_has_aardvark') else 'no'}</td>"
f"<td>{repo['aardvark_status']}</td>"
f"<td>{int(repo.get('harvested_record_count') or 0)}</td>"
f"<td>{int(repo.get('available_record_count') or 0)}</td>"
"</tr>"
Expand All @@ -123,7 +126,8 @@ async def ogm_repo_dashboard(request: Request):
"<p>Templates are unavailable, showing a minimal fallback view.</p>"
"<table><thead><tr>"
"<th>Repository</th><th>Last commit</th><th>Last harvest</th>"
"<th>Aardvark</th><th>Harvested</th><th>Available</th>"
"<th>Aardvark</th><th>Last-seen source records</th>"
"<th>Published with source tag (database)</th>"
f"</tr></thead><tbody>{rows}</tbody></table></body></html>"
)
)
Expand Down
92 changes: 87 additions & 5 deletions backend/app/services/ogm_harvest/repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from datetime import datetime
from typing import Any, Dict, List, Optional
from urllib.parse import quote, urlencode

from sqlalchemy import String, cast, func, literal, select, true, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
Expand All @@ -10,6 +11,35 @@
from db.models import ogm_harvest_runs, ogm_repos, ogm_resource_state, resources


def _published_resources():
return func.lower(
func.coalesce(
func.nullif(resources.c.b1g_publication_state_s, cast(literal(""), String())),
func.nullif(resources.c.publication_state, cast(literal(""), String())),
cast(literal("published"), String()),
)
) == cast(literal("published"), String())


def _active_memberships():
return (
select(
ogm_resource_state.c.ogm_resource_id,
ogm_resource_state.c.ogm_repo_name,
)
.select_from(
ogm_resource_state.join(
ogm_repos, ogm_resource_state.c.ogm_repo_name == ogm_repos.c.ogm_repo_name
)
)
.where(
ogm_resource_state.c.ogm_missing_since.is_(None),
ogm_repos.c.ogm_enabled.is_(True),
func.coalesce(ogm_repos.c.ogm_tags["ogm_archived"].as_boolean(), False).is_(False),
)
)


class OGMHarvestRepository:
"""
Minimal async repository for OGM harvest tables.
Expand All @@ -22,6 +52,26 @@ async def list_repos(self) -> List[Dict[str, Any]]:
rows = await database.fetch_all(select(ogm_repos).order_by(ogm_repos.c.ogm_repo_name))
return [dict(r) for r in rows]

async def get_public_dashboard_counts(self) -> Dict[str, int]:
"""Count unique active-source records and all published database records."""
active = _active_memberships().subquery("active_memberships")
row = await database.fetch_one(
select(
select(func.count(func.distinct(active.c.ogm_resource_id)))
.select_from(active.join(resources, resources.c.id == active.c.ogm_resource_id))
.scalar_subquery()
.label("active_record_count"),
select(func.count())
.select_from(resources)
.where(_published_resources())
.scalar_subquery()
.label("published_record_count"),
)
)
return {
key: int(row[key] or 0) for key in ("active_record_count", "published_record_count")
}

async def list_public_repo_summaries(self) -> List[Dict[str, Any]]:
"""
Public-facing OGM repo summaries.
Expand Down Expand Up @@ -86,9 +136,32 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]:
.subquery("resource_counts")
)

active = _active_memberships().subquery("active_memberships")
other_active_counts = (
select(
ogm_resource_state.c.ogm_repo_name,
func.count(func.distinct(ogm_resource_state.c.ogm_resource_id)).label(
"other_active_source_count"
),
)
.select_from(
ogm_resource_state.join(
active,
(ogm_resource_state.c.ogm_resource_id == active.c.ogm_resource_id)
& (ogm_resource_state.c.ogm_repo_name != active.c.ogm_repo_name),
)
)
.where(ogm_resource_state.c.ogm_missing_since.is_(None))
.group_by(ogm_resource_state.c.ogm_repo_name)
.subquery("other_active_counts")
)

q = (
select(
ogm_repos.c.ogm_repo_name,
func.coalesce(other_active_counts.c.other_active_source_count, 0).label(
"other_active_source_count"
),
ogm_repos.c.ogm_enabled,
ogm_repos.c.ogm_watch_mode,
ogm_repos.c.ogm_last_harvest_started_at,
Expand Down Expand Up @@ -116,8 +189,10 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]:
)
.select_from(
ogm_repos.outerjoin(
latest_run_ids, ogm_repos.c.ogm_repo_name == latest_run_ids.c.repo_name
other_active_counts,
ogm_repos.c.ogm_repo_name == other_active_counts.c.ogm_repo_name,
)
.outerjoin(latest_run_ids, ogm_repos.c.ogm_repo_name == latest_run_ids.c.repo_name)
.outerjoin(
ogm_harvest_runs, ogm_harvest_runs.c.ogm_id == latest_run_ids.c.latest_ogm_id
)
Expand Down Expand Up @@ -154,19 +229,25 @@ def _to_int(value: Any) -> int:
"ogm_last_harvest_completed_at"
)
repo_full_name = tags.get("ogm_repo_full_name") or item.get("ogm_repo_name")
if repo_full_name and "/" not in repo_full_name:
repo_full_name = f"OpenGeoMetadata/{repo_full_name}"
has_aardvark = tags.get("ogm_has_aardvark")
if has_aardvark is None:
has_aardvark = not bool(tags.get("ogm_missing_aardvark"))
if has_aardvark is None and "ogm_missing_aardvark" in tags:
has_aardvark = not bool(tags["ogm_missing_aardvark"])
summaries.append(
{
"ogm_repo_name": item.get("ogm_repo_name"),
"ogm_repo_full_name": repo_full_name,
"ogm_github_url": (
f"https://github.com/{repo_full_name}" if repo_full_name else None
f"https://github.com/{quote(repo_full_name, safe='/')}"
if repo_full_name
else None
),
"ogm_search_url": "/api/v1/search?"
+ urlencode({"ogm_repo": item["ogm_repo_name"]}),
"ogm_enabled": item.get("ogm_enabled"),
"ogm_watch_mode": item.get("ogm_watch_mode"),
"ogm_has_aardvark": bool(has_aardvark),
"ogm_has_aardvark": has_aardvark,
"ogm_default_branch": tags.get("ogm_default_branch"),
"ogm_archived": bool(tags.get("ogm_archived", False)),
"last_commit_at": tags.get("ogm_pushed_at"),
Expand All @@ -178,6 +259,7 @@ def _to_int(value: Any) -> int:
"harvested_success_count": _to_int(stats.get("imported")),
"harvested_failure_count": _to_int(stats.get("errors")),
"harvested_record_count": _to_int(item.get("harvested_record_count")),
"other_active_source_count": _to_int(item.get("other_active_source_count")),
"available_record_count": _to_int(item.get("available_record_count")),
"suppressed_record_count": _to_int(item.get("suppressed_record_count")),
"unpublished_record_count": _to_int(item.get("unpublished_record_count")),
Expand Down
30 changes: 18 additions & 12 deletions backend/templates/ogm_repo_dashboard.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ <h2>Repository health, harvest freshness, and API availability in one place.</h2
<p class="lede">
This page tracks every repository discovered in the OpenGeoMetadata GitHub organization,
whether it exposes a <code>metadata-aardvark/</code> directory, when it last changed on GitHub,
when this app last harvested it, and how many records are currently harvested versus published
through this API.
when this app last harvested it, and how many records were last seen in each source. Archived and disabled
sources retain their harvest history. Publication counts come from the database;
this page does not verify the live search index.
</p>
<p class="meta-note">Generated {{ generated_at }}. Times shown in UTC.</p>
</section>
Expand All @@ -53,15 +54,15 @@ <h2>Repository health, harvest freshness, and API availability in one place.</h2
<p class="summary-value">{{ summary.repos_with_aardvark_count }}</p>
</article>
<article class="summary-card">
<p class="summary-label">Nightly Enabled</p>
<p class="summary-label">Harvesting Enabled</p>
<p class="summary-value">{{ summary.enabled_repo_count }}</p>
</article>
<article class="summary-card">
<p class="summary-label">Current Harvested Records</p>
<p class="summary-label">Unique Records in Active Sources</p>
<p class="summary-value">{{ "{:,}".format(summary.harvested_record_count) }}</p>
</article>
<article class="summary-card">
<p class="summary-label">Published in This API</p>
<p class="summary-label">Published Records (Database)</p>
<p class="summary-value">{{ "{:,}".format(summary.available_record_count) }}</p>
</article>
<article class="summary-card">
Expand All @@ -74,6 +75,8 @@ <h2>Repository health, harvest freshness, and API availability in one place.</h2
<div class="monitor-table-head">
<div>
<h3>Repositories</h3>
<p class="meta-note">Per-repository counts can overlap and should not be added together.
“Also in active sources” describes overlapping source records, not a synchronization failure.</p>
<p class="meta-note">JSON source: <a href="/api/v1/ogm/repos"><code>/api/v1/ogm/repos</code></a></p>
</div>
</div>
Expand All @@ -86,9 +89,9 @@ <h3>Repositories</h3>
<th scope="col">Last Commit</th>
<th scope="col">Last Harvest</th>
<th scope="col">Aardvark</th>
<th scope="col">Harvest Status</th>
<th scope="col">Harvested Records</th>
<th scope="col">API Visibility</th>
<th scope="col">Last Harvest Result</th>
<th scope="col">Last-Seen Source Records</th>
<th scope="col">Published with Source Tag (Database)</th>
</tr>
</thead>
<tbody>
Expand All @@ -98,6 +101,8 @@ <h3>Repositories</h3>
<div class="repo-cell">
<a href="{{ repo.ogm_github_url }}" target="_blank" rel="noreferrer">{{ repo.ogm_repo_name }}</a>
<p class="repo-subtitle">{{ repo.ogm_repo_full_name or repo.ogm_repo_name }}</p>
<p><a href="{{ repo.ogm_search_url }}" aria-label="Search API records from {{ repo.ogm_repo_name }}">Search API</a></p>
<p><span class="status-pill {{ 'status-ok' if repo.source_status == 'Active' else 'status-muted' }}">{{ repo.source_status }}</span></p>
<p class="repo-subtitle">
mode: <code>{{ repo.ogm_watch_mode or "manual" }}</code>
{% if repo.last_commit_sha %}
Expand All @@ -120,10 +125,12 @@ <h3>Repositories</h3>
{% endif %}
</td>
<td>
{% if repo.ogm_has_aardvark %}
{% if repo.ogm_has_aardvark is sameas true %}
<span class="status-pill status-ok">Present</span>
{% else %}
{% elif repo.ogm_has_aardvark is sameas false %}
<span class="status-pill status-muted">Missing</span>
{% else %}
<span class="status-pill status-muted">Unknown</span>
{% endif %}
</td>
<td>
Expand Down Expand Up @@ -155,8 +162,7 @@ <h3>Repositories</h3>
<span>{{ "{:,}".format(item.count) }} {{ item.label }}</span>{% if not loop.last %}<span> · </span>{% endif %}
{% endfor %}
</p>
{% elif repo.harvest_gap_count %}
<p class="repo-subtitle">{{ "{:,}".format(repo.harvest_gap_count) }} hidden from API</p>

{% endif %}
</td>
</tr>
Expand Down
Loading
Loading