diff --git a/.github/workflows/catalog-health.yaml b/.github/workflows/catalog-health.yaml index cc84489..eebaeed 100644 --- a/.github/workflows/catalog-health.yaml +++ b/.github/workflows/catalog-health.yaml @@ -4,7 +4,8 @@ name: Catalog health # can be deleted, retagged, or re-cut after a plugin lands here. PR validation # only catches breakage at merge time, so this re-verifies every plugin's # download links and checksums on a schedule and opens an issue if the catalog -# has rotted — before a user hits it during `datumctl plugin install`. +# has rotted — before a user hits it during `datumctl plugin install`. It also +# checks that index.yaml still matches plugins/. on: schedule: - cron: "0 12 * * 1" # Mondays, 12:00 UTC @@ -30,6 +31,12 @@ jobs: - name: Verify every plugin's download links and checksums run: python3 scripts/verify_manifests.py + # The Regenerate index.yaml workflow keeps index.yaml in step with + # plugins/ on every merge, so drift here means that workflow is failing. + - name: Check that index.yaml matches plugins/ + if: ${{ !cancelled() }} + run: python3 scripts/generate_index.py --check + - name: Open an issue if the catalog is unhealthy if: failure() env: @@ -40,7 +47,7 @@ jobs: --color b60205 \ --description "A plugin download link or checksum is broken" 2>/dev/null || true - body=$(printf 'The scheduled catalog health check failed: one or more plugin download URLs no longer resolve, or their checksums no longer match the published release assets.\n\nUsers may be unable to install or verify the affected plugin(s) with datumctl until this is fixed. See the run log for the specific plugin and platform:\n\n%s\n' "$RUN_URL") + body=$(printf 'The scheduled catalog health check failed. Either a plugin download URL no longer resolves or its checksum no longer matches the published release asset, or index.yaml no longer matches plugins/ because the Regenerate index.yaml workflow is failing.\n\nUsers may be unable to install, verify, or see current versions of the affected plugin(s) with datumctl until this is fixed. See the run log for the failing step:\n\n%s\n' "$RUN_URL") number=$(gh issue list --state open --label catalog-health --json number --jq '.[0].number // empty') if [ -n "$number" ]; then diff --git a/.github/workflows/generate-index.yaml b/.github/workflows/generate-index.yaml index 358ee7a..050f242 100644 --- a/.github/workflows/generate-index.yaml +++ b/.github/workflows/generate-index.yaml @@ -1,13 +1,14 @@ name: Regenerate index.yaml # index.yaml is what datumctl reads, and it is fully derived from plugins/*.yaml. -# Rather than asking every contributor (and the release bot) to regenerate and -# commit it in their PR, it is reconciled here on merge: when plugins/ changes on -# main, regenerate the index and commit it straight back to main. +# When plugins/ changes on main, regenerate the index and open (or refresh) a +# pull request with the result, set to merge itself once required checks pass. # -# main is not a protected branch, so the built-in GITHUB_TOKEN can push directly -# — no app token needed. The commit touches only index.yaml (not plugins/**) and -# GITHUB_TOKEN pushes don't retrigger workflows, so this never loops. +# main requires the "ci / Build and test" check, so the index can only reach it +# through a pull request that CI has run on. Pull requests opened with +# GITHUB_TOKEN do not trigger workflows, so the pull request is opened with the +# catalog's GitHub App token instead. The pull request touches only index.yaml, +# not plugins/**, so merging it never retriggers this workflow. on: push: branches: [main] @@ -15,13 +16,13 @@ on: - "plugins/**" workflow_dispatch: -# Serialize so two quick merges can't race to push index.yaml. +# Serialize so two quick merges can't race on the regenerate-index branch. concurrency: group: regenerate-index cancel-in-progress: false permissions: - contents: write + contents: read jobs: regenerate: @@ -34,19 +35,40 @@ jobs: python-version: "3.x" - name: Install tooling - run: pip install --quiet "PyYAML>=6" + run: pip install --quiet "PyYAML==6.0.3" - name: Regenerate index.yaml run: python3 scripts/generate_index.py - - name: Commit if index.yaml changed + - name: Mint catalog token from a GitHub App + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.PLUGIN_INDEX_APP_ID }} + private-key: ${{ secrets.PLUGIN_INDEX_APP_PRIVATE_KEY }} + + # When index.yaml is already in sync, this closes any open regenerate + # pull request and deletes its branch. + - name: Open or update the regenerate pull request + id: pr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.app-token.outputs.token }} + base: main + branch: regenerate-index + delete-branch: true + sign-commits: true + add-paths: index.yaml + commit-message: "chore: Regenerate index.yaml" + title: "chore: Regenerate index.yaml" + body: | + Brings `index.yaml` back in line with `plugins/*.yaml` as of ${{ github.sha }}. + + The Regenerate index.yaml workflow opened this pull request, and it merges itself once required checks pass. + + - name: Merge the pull request once checks pass + if: steps.pr.outputs.pull-request-number != '' && steps.pr.outputs.pull-request-operation != 'closed' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if git diff --quiet -- index.yaml; then - echo "index.yaml already in sync; nothing to do." - exit 0 - fi - EXPECTED_HEAD=$(git rev-parse HEAD) - export EXPECTED_HEAD - scripts/sign-commit main "chore: regenerate index.yaml" index.yaml + GH_TOKEN: ${{ steps.app-token.outputs.token }} + PR_NUMBER: ${{ steps.pr.outputs.pull-request-number }} + run: gh pr merge "$PR_NUMBER" --auto --merge diff --git a/README.md b/README.md index 8f112a7..f3aaea9 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ datumctl ipam pool list 1. Add the `datumctl-plugin` topic to your plugin's GitHub repository. 2. Open a pull request adding `plugins/.yaml`, following the [schema](schema/plugin-v1alpha1.json). That's the only file you edit. 3. CI validates every manifest: schema conformance, that each download URL resolves, and that each SHA256 matches the published archive. A weekly [health check](.github/workflows/catalog-health.yaml) re-verifies the whole catalog so a plugin's links can't rot unnoticed. -4. On merge, [`index.yaml`](index.yaml) — the file `datumctl` actually reads — is regenerated from `plugins/*.yaml` and committed automatically. You never edit it by hand. (To preview locally: `python3 scripts/generate_index.py`.) +4. On merge, [`index.yaml`](index.yaml) — the file `datumctl` actually reads — is regenerated from `plugins/*.yaml` in a follow-up pull request that merges itself once CI passes. You never edit it by hand. (To preview locally: `python3 scripts/generate_index.py`.) ## Plugin manifest format diff --git a/index.yaml b/index.yaml index 73a9e2f..bdf9211 100644 --- a/index.yaml +++ b/index.yaml @@ -8,49 +8,48 @@ items: - apiVersion: datumctl.datum.net/v1alpha1 kind: Plugin metadata: - name: inventory + name: billing spec: - shortDescription: Browse and populate the Datum Cloud inventory graph (typed nodes - and edges) - homepage: https://github.com/datum-cloud/inventory - version: v0.1.0 + shortDescription: View metered usage and spend for your organization or project + homepage: https://github.com/milo-os/billing + version: v0.6.0 platforms: - selector: matchLabels: os: linux arch: amd64 - uri: https://github.com/datum-cloud/inventory/releases/download/v0.1.0/datumctl-inventory_Linux_x86_64.tar.gz - sha256: 9c392c1bc6368d00a4496f4449e8de06c84a1d29da5d02999df4ccf19acc778d + uri: https://github.com/milo-os/billing/releases/download/v0.6.0/milo-billing_Linux_x86_64.tar.gz + sha256: 632f1d16c940af1d297429fbb799e83d6f1d0bb8312eb634a5ddd2dc985c4c81 - selector: matchLabels: os: linux arch: arm64 - uri: https://github.com/datum-cloud/inventory/releases/download/v0.1.0/datumctl-inventory_Linux_arm64.tar.gz - sha256: c1a1698faf072091c1fd11ed62bbc451e5d1be55b872014042241dc10061fdbc + uri: https://github.com/milo-os/billing/releases/download/v0.6.0/milo-billing_Linux_arm64.tar.gz + sha256: a3283d256692767c8ee1ea0bf805c1c22c11fa8090c23b49b26a9c786fa081ea - selector: matchLabels: os: darwin arch: amd64 - uri: https://github.com/datum-cloud/inventory/releases/download/v0.1.0/datumctl-inventory_Darwin_x86_64.tar.gz - sha256: 4a8b958a7e91454729da493d7f1abf90f0303df7c35328fdaf181f141c98f1c2 + uri: https://github.com/milo-os/billing/releases/download/v0.6.0/milo-billing_Darwin_x86_64.tar.gz + sha256: 6f866b9d0200cdff2f95bf1d6ba1833d1bc5e2668ee06f3c8afb0c50fffe4dc4 - selector: matchLabels: os: darwin arch: arm64 - uri: https://github.com/datum-cloud/inventory/releases/download/v0.1.0/datumctl-inventory_Darwin_arm64.tar.gz - sha256: 5966496965bf8bec2675af3bdff9a71ae2429473e5b2b55ee4137571f0f8b0a5 + uri: https://github.com/milo-os/billing/releases/download/v0.6.0/milo-billing_Darwin_arm64.tar.gz + sha256: 28cc4e060742dba933188fe5d7f872eecc5c4a2eee740f2538fc87c21c7c16eb - selector: matchLabels: os: windows arch: amd64 - uri: https://github.com/datum-cloud/inventory/releases/download/v0.1.0/datumctl-inventory_Windows_x86_64.zip - sha256: 03507d79ac88360bf2c6eab187a383542933e7686d8826c36df43df9bb91d902 + uri: https://github.com/milo-os/billing/releases/download/v0.6.0/milo-billing_Windows_x86_64.zip + sha256: 84349b8f85b58e60a0bd5fd770a280f2bd66b952e400522f932f528ece9afb49 - selector: matchLabels: os: windows arch: arm64 - uri: https://github.com/datum-cloud/inventory/releases/download/v0.1.0/datumctl-inventory_Windows_arm64.zip - sha256: 86688afbb1968ceff1714562dea79176bdaabfb3c45cb61ce4a7a1c14f6a391a + uri: https://github.com/milo-os/billing/releases/download/v0.6.0/milo-billing_Windows_arm64.zip + sha256: 5ce1ba584b096c93b85339f6744c82c41d5303fa2c9e180fa8599b7e2c6fb91d - apiVersion: datumctl.datum.net/v1alpha1 kind: Plugin metadata: @@ -58,41 +57,132 @@ items: spec: shortDescription: Manage IP address space (pools and prefixes) across the platform homepage: https://github.com/milo-os/ipam - version: v0.3.1 + version: v0.5.5 platforms: - selector: matchLabels: os: linux arch: amd64 - uri: https://github.com/milo-os/ipam/releases/download/v0.3.1/milo-ipam_Linux_x86_64.tar.gz - sha256: d4000e6d82f3406d20e01d1c2feae972c3878ff56ebf10b00bbb5bbaa02898f2 + uri: https://github.com/milo-os/ipam/releases/download/v0.5.5/milo-ipam_Linux_x86_64.tar.gz + sha256: 93fccece6124dec5db3aaf88c57b08c6dabb814d2436d8d775b68a2603515d79 - selector: matchLabels: os: linux arch: arm64 - uri: https://github.com/milo-os/ipam/releases/download/v0.3.1/milo-ipam_Linux_arm64.tar.gz - sha256: ce8e0d93648a93eb8caf2510ca8ba6158a158b0fb4f51f10a8b9fed9325726a0 + uri: https://github.com/milo-os/ipam/releases/download/v0.5.5/milo-ipam_Linux_arm64.tar.gz + sha256: d7b69db76ebf13ebc3e7c917611e00604dc1d6cd4fb18dbbf2a9466f7aa2c7b6 - selector: matchLabels: os: darwin arch: amd64 - uri: https://github.com/milo-os/ipam/releases/download/v0.3.1/milo-ipam_Darwin_x86_64.tar.gz - sha256: 06dab9f12cf22074e7a3c3b3ff0374adf0907f8792ab0c6c5e58a9307d4aac51 + uri: https://github.com/milo-os/ipam/releases/download/v0.5.5/milo-ipam_Darwin_x86_64.tar.gz + sha256: 24ea9a7de7232f0495a31e907cb69a19a39af3ddc8c1e7c269e671d726dce006 - selector: matchLabels: os: darwin arch: arm64 - uri: https://github.com/milo-os/ipam/releases/download/v0.3.1/milo-ipam_Darwin_arm64.tar.gz - sha256: 875fa4874a38c813ae5c079d1b47a67ebcf3613b8d54aac8470e0451b93e917f + uri: https://github.com/milo-os/ipam/releases/download/v0.5.5/milo-ipam_Darwin_arm64.tar.gz + sha256: 609d90f5fab095538e995e8976ad01de9d66e9f79b359ade7ea575dc68b46320 - selector: matchLabels: os: windows arch: amd64 - uri: https://github.com/milo-os/ipam/releases/download/v0.3.1/milo-ipam_Windows_x86_64.zip - sha256: d3d57f419f2fbc04c91472fa01642a6c8a138385964afc96a21eb37eca5a8277 + uri: https://github.com/milo-os/ipam/releases/download/v0.5.5/milo-ipam_Windows_x86_64.zip + sha256: 5395655631269122f2f4025dbea40f0978c86f52a900d3f4bc62ca2d8605f77f - selector: matchLabels: os: windows arch: arm64 - uri: https://github.com/milo-os/ipam/releases/download/v0.3.1/milo-ipam_Windows_arm64.zip - sha256: 3f9b7525708fabdce9047a49c2fa541d4acb3a62dd5a668270edd76472bfc8e4 + uri: https://github.com/milo-os/ipam/releases/download/v0.5.5/milo-ipam_Windows_arm64.zip + sha256: 1728008242cde49826f40ad6f85358b2d73d3b0a27b1bcfa3ef2f34f88bef313 +- apiVersion: datumctl.datum.net/v1alpha1 + kind: Plugin + metadata: + name: search + spec: + shortDescription: Search for resources across the platform by kind, name, and + project scope + homepage: https://github.com/milo-os/search + version: v0.8.7 + platforms: + - selector: + matchLabels: + os: linux + arch: amd64 + uri: https://github.com/milo-os/search/releases/download/v0.8.7/milo-search_Linux_x86_64.tar.gz + sha256: 39eccc1ec528b9117f85ee1c9c56276b864d77cfd1da194c68a39d2fe41a92d0 + - selector: + matchLabels: + os: linux + arch: arm64 + uri: https://github.com/milo-os/search/releases/download/v0.8.7/milo-search_Linux_arm64.tar.gz + sha256: 323bc0febc40fdf2ced70edd50299d2270e2d6f7fa3ded9ef4520cd0810abffe + - selector: + matchLabels: + os: darwin + arch: amd64 + uri: https://github.com/milo-os/search/releases/download/v0.8.7/milo-search_Darwin_x86_64.tar.gz + sha256: c5d3f9ee37b2aa86115e712bfd55c4a9e42232f01d431524933477981db18d3b + - selector: + matchLabels: + os: darwin + arch: arm64 + uri: https://github.com/milo-os/search/releases/download/v0.8.7/milo-search_Darwin_arm64.tar.gz + sha256: 42b47d0000a3594f292b2f630f14792179de2229b50bf50f65f681cb96f2f1f2 + - selector: + matchLabels: + os: windows + arch: amd64 + uri: https://github.com/milo-os/search/releases/download/v0.8.7/milo-search_Windows_x86_64.zip + sha256: 306a6fa1bccbabe65454e9780b1c531f8bdc322d4d58da540b727fa3aeb9f693 + - selector: + matchLabels: + os: windows + arch: arm64 + uri: https://github.com/milo-os/search/releases/download/v0.8.7/milo-search_Windows_arm64.zip + sha256: ba3527be5b55f6662239e83b9cc986e540dba026b2c912475410dcfbb2e97563 +- apiVersion: datumctl.datum.net/v1alpha1 + kind: Plugin + metadata: + name: telemetry + spec: + shortDescription: Query logs for your Datum Cloud projects + homepage: https://github.com/milo-os/telemetry + version: v0.6.3 + platforms: + - selector: + matchLabels: + os: linux + arch: amd64 + uri: https://github.com/milo-os/telemetry/releases/download/v0.6.3/milo-telemetry_Linux_x86_64.tar.gz + sha256: 184aeb1e7395734249bb0076fb160bb90d77bcd0a41026a200b784976e88ab6c + - selector: + matchLabels: + os: linux + arch: arm64 + uri: https://github.com/milo-os/telemetry/releases/download/v0.6.3/milo-telemetry_Linux_arm64.tar.gz + sha256: 4ea2ce16aa30af34f794297b968d8e6bbefd6fd2b80e8ece99f2e2366e3c4bac + - selector: + matchLabels: + os: darwin + arch: amd64 + uri: https://github.com/milo-os/telemetry/releases/download/v0.6.3/milo-telemetry_Darwin_x86_64.tar.gz + sha256: 4387c8b927fb65a0e2e5d2585f94db75ea2dda8f95ec2715ce6b34223886af4a + - selector: + matchLabels: + os: darwin + arch: arm64 + uri: https://github.com/milo-os/telemetry/releases/download/v0.6.3/milo-telemetry_Darwin_arm64.tar.gz + sha256: 963e1e376a26377fa9fa547b4b64bb40cceff69340ce62f7895f264a4aa211a4 + - selector: + matchLabels: + os: windows + arch: amd64 + uri: https://github.com/milo-os/telemetry/releases/download/v0.6.3/milo-telemetry_Windows_x86_64.zip + sha256: d1fe9ac731fc86d43c4f9e28aa689a73e9ae6f2cd8bebc5c3f054afd94fa6030 + - selector: + matchLabels: + os: windows + arch: arm64 + uri: https://github.com/milo-os/telemetry/releases/download/v0.6.3/milo-telemetry_Windows_arm64.zip + sha256: 18148cde32bfaf0d36c9a884ec7b5674082ccc881ea93e60469ed39886ae487d diff --git a/scripts/generate_index.py b/scripts/generate_index.py index cd92fcf..82b0d7f 100755 --- a/scripts/generate_index.py +++ b/scripts/generate_index.py @@ -3,8 +3,8 @@ index.yaml is the single file datumctl reads to discover plugins, their versions, and per-platform download locations. It is fully derived from -plugins/*.yaml, so it is regenerated deterministically here and checked for -drift in CI (`--check`) rather than hand-edited. +plugins/*.yaml, so it is regenerated deterministically here rather than +hand-edited, and the catalog health check fails on drift (`--check`). Usage: scripts/generate_index.py # write index.yaml diff --git a/scripts/sign-commit b/scripts/sign-commit deleted file mode 100755 index 2082bbe..0000000 --- a/scripts/sign-commit +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# Commit files to a branch through the GitHub API so GitHub signs the commit. -# Usage: sign-commit [-z] BRANCH MESSAGE [PATH...] -# A PATH that is a file is committed with its current contents; one that does -# not exist at all is deleted from the branch. Directories and symlinks are -# refused. With -z, further PATHs are read from stdin, NUL-separated, so -# `git diff -z --name-only` output passes through without quoting or splitting. -# Exits 1 when there is nothing to commit, and 3 when the remote branch is not -# at the expected head, so a caller can refetch and try again. -# Set EXPECTED_HEAD to the commit the job checked out: the commit fails unless -# the remote branch is still there. Set BASE_SHA instead to create BRANCH at -# that commit (or reset it there) first. One of the two is required. -# Needs GH_TOKEN and GITHUB_REPOSITORY. -set -euo pipefail - -from_stdin=0 -if [ "${1:-}" = "-z" ]; then - from_stdin=1 - shift -fi -if [ $# -lt 2 ]; then - echo "Usage: sign-commit [-z] BRANCH MESSAGE [PATH...]" >&2 - exit 2 -fi - -branch=$1 -message=$2 -shift 2 - -repo=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required} -if [ -z "${BASE_SHA:-}" ] && [ -z "${EXPECTED_HEAD:-}" ]; then - echo "sign-commit: set EXPECTED_HEAD (or BASE_SHA to create the branch)." >&2 - exit 2 -fi -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT - -: > "$work/additions" -: > "$work/deletions" -n=0 -adds=0 -dels=0 - -add_path() { - local path=$1 - if [ -L "$path" ]; then - echo "sign-commit: ${path} is a symlink; the API cannot commit one." >&2 - exit 1 - elif [ -d "$path" ]; then - echo "sign-commit: ${path} is a directory; pass the files in it." >&2 - exit 1 - elif [ -f "$path" ]; then - n=$((n + 1)) - adds=$((adds + 1)) - base64 < "$path" | tr -d '\n' > "$work/$n.b64" - jq -nc --arg path "$path" --rawfile contents "$work/$n.b64" '{path: $path, contents: $contents}' >> "$work/additions" - elif [ ! -e "$path" ]; then - dels=$((dels + 1)) - jq -nc --arg path "$path" '{path: $path}' >> "$work/deletions" - else - echo "sign-commit: ${path} is not a regular file." >&2 - exit 1 - fi -} - -for path in "$@"; do - add_path "$path" -done -if [ "$from_stdin" = 1 ]; then - while IFS= read -r -d '' path; do - add_path "$path" - done -fi - -if [ $((adds + dels)) -eq 0 ]; then - echo "sign-commit: nothing to commit." >&2 - exit 1 -fi - -remote_head() { - gh api "repos/${repo}/git/ref/heads/${branch}" --jq .object.sha -} - -if [ -n "${BASE_SHA:-}" ]; then - if ! gh api -X POST "repos/${repo}/git/refs" -f "ref=refs/heads/${branch}" -f "sha=${BASE_SHA}" >/dev/null 2>&1; then - gh api -X PATCH "repos/${repo}/git/refs/heads/${branch}" -f "sha=${BASE_SHA}" -F force=true >/dev/null - fi - expected=$BASE_SHA -else - expected=$EXPECTED_HEAD - head=$(remote_head) - if [ "$head" != "$expected" ]; then - echo "sign-commit: ${branch} is at ${head}, but this job checked out ${expected}." >&2 - exit 3 - fi -fi - -headline=${message%%$'\n'*} -body="" -if [ "$headline" != "$message" ]; then - body=${message#*$'\n'} - body=${body#$'\n'} -fi - -query='mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { url } } }' - -jq -n --arg query "$query" --arg repo "$repo" --arg branch "$branch" --arg head "$expected" \ - --arg headline "$headline" --arg body "$body" \ - --slurpfile additions "$work/additions" --slurpfile deletions "$work/deletions" '{ - query: $query, - variables: {input: { - branch: {repositoryNameWithOwner: $repo, branchName: $branch}, - expectedHeadOid: $head, - message: ({headline: $headline} + (if $body == "" then {} else {body: $body} end)), - fileChanges: {additions: $additions, deletions: $deletions} - }} - }' > "$work/request.json" - -for attempt in 1 2 3 4 5; do - if gh api graphql --input "$work/request.json" --jq .data.createCommitOnBranch.commit.url; then - exit 0 - fi - head=$(remote_head) - if [ "$head" != "$expected" ]; then - echo "sign-commit: ${branch} moved to ${head} while committing; not retrying." >&2 - exit 3 - fi - echo "Attempt ${attempt} failed, retrying." >&2 - sleep $((attempt * 2)) -done -echo "Could not commit to ${branch}." >&2 -exit 1