feat(ci/cd): release packages to repo.linuxfabrik.ch without a manual step

.github/workflows/auto-merge.yml

@@ -64,6 +64,37 @@ jobs:
           GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
           PR_URL: '${{ github.event.pull_request.html_url }}'
 
+  # An upstream version bump from lf-check-upstream.yml goes in without review
+  # once the required checks pass, including `Build result` from
+  # lf-build-pr.yml, which builds and smoke-tests the package on all its target
+  # distros. Its merge then starts lf-release.yml, which publishes to the
+  # `testing` channel first.
+  #
+  # Auto-merge is enabled with the app token, not GITHUB_TOKEN: GitHub starts no
+  # workflow for a push made by GITHUB_TOKEN, so the merge would never trigger
+  # the release.
+  upstream-bump:
+    runs-on: 'ubuntu-latest'
+    if: >-
+      github.actor == 'linuxfabrik-automation[bot]'
+      && startsWith(github.head_ref, 'bump/')
+    permissions:
+      contents: 'read'
+    steps:
+
+      - name: 'Generate app token'
+        id: 'app-token'
+        uses: 'actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510' # v3.3.0
+        with:
+          client-id: '${{ vars.LF_AUTOMATION_CLIENT_ID }}'
+          private-key: '${{ secrets.LF_AUTOMATION_APP_PRIVATE_KEY }}'
+
+      - run: |
+          gh pr merge --auto --squash "$PR_URL"
+        env:
+          GH_TOKEN: '${{ steps.app-token.outputs.token }}'
+          PR_URL: '${{ github.event.pull_request.html_url }}'
+
   # The weekly hook bump carries nothing but `rev:` changes in
   # .pre-commit-config.yaml, and it arrives in every repository at once.
   # Merging that by hand is pure overhead, so it goes in as soon as the

.github/workflows/lf-build-pr.yml

@@ -0,0 +1,77 @@
+name: 'Linuxfabrik: Build changed packages'
+
+# Builds and smoke-tests every package a pull request touches, so that an
+# upstream bump is only merged once it builds and installs on all its target
+# distros. A change below build/ rebuilds every package.
+#
+# No `paths` filter on purpose: the `Build result` job is a required check, and a
+# required check whose workflow never starts keeps every other pull request
+# blocked.
+
+on:
+  pull_request: {}
+
+permissions:
+  contents: 'read'
+
+jobs:
+  changed-packages:
+    runs-on:
+      - 'ubuntu-24.04'
+    outputs:
+      packages: '${{ steps.changed.outputs.packages }}'
+    steps:
+      - name: 'Harden the runner (Audit all outbound calls)'
+        uses: 'step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2' # v2.22.2
+        with:
+          egress-policy: 'audit'
+
+      - name: 'git clone https://github.com/Linuxfabrik/packaging'
+        uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - id: 'changed'
+        env:
+          BASE_SHA: '${{ github.event.pull_request.base.sha }}'
+          HEAD_SHA: '${{ github.event.pull_request.head.sha }}'
+        run: |
+          packages="$(bash build/changed-packages.sh --include-build "$BASE_SHA" "$HEAD_SHA" \
+              | jq --raw-input --slurp --compact-output 'split("\n") | map(select(length > 0))')"
+          echo "Packages to build: $packages"
+          echo "packages=$packages" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: 'changed-packages'
+    if: "needs.changed-packages.outputs.packages != '[]'"
+    strategy:
+      fail-fast: false
+      matrix:
+        package: '${{ fromJson(needs.changed-packages.outputs.packages) }}'
+    uses: './.github/workflows/lf-build-linux-x86_64.yml'
+    with:
+      package: '${{ matrix.package }}'
+
+  # One stable check name for the ruleset, whatever packages the matrix covers.
+  # A skipped build (nothing to build) counts as success.
+  build-result:
+    name: 'Build result'
+    needs:
+      - 'build'
+      - 'changed-packages'
+    if: 'always()'
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+      - name: 'Harden the runner (Audit all outbound calls)'
+        uses: 'step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2' # v2.22.2
+        with:
+          egress-policy: 'audit'
+
+      - env:
+          BUILD_RESULT: '${{ needs.build.result }}'
+          CHANGED_RESULT: '${{ needs.changed-packages.result }}'
+        run: |
+          echo "changed-packages: $CHANGED_RESULT, build: $BUILD_RESULT"
+          [ "$CHANGED_RESULT" = 'success' ]
+          [ "$BUILD_RESULT" = 'success' ] || [ "$BUILD_RESULT" = 'skipped' ]

.github/workflows/lf-check-upstream.yml

@@ -1,14 +1,25 @@
 name: 'Linuxfabrik: Check upstream versions'
 
 on:
+  # Linuxfabrik's own projects send `upstream-release` from their release
+  # workflow, so their packages do not wait for the next scheduled run:
+  #   gh api repos/Linuxfabrik/packaging/dispatches --field event_type=upstream-release
+  repository_dispatch:
+    types:
+      - 'upstream-release'
   schedule:
-    # Daily at 04:00 UTC.
-    - cron: '0 4 * * *'
+    # Hourly, at minute 17.
+    - cron: '17 * * * *'
   workflow_dispatch:
 
 permissions:
   contents: 'read'
 
+# A dispatch arriving during the scheduled run would open the same bump twice.
+concurrency:
+  group: 'lf-check-upstream'
+  cancel-in-progress: false
+
 jobs:
   check-upstream:
     runs-on:
@@ -45,11 +56,22 @@ jobs:
           git config user.name 'linuxfabrik-automation[bot]'
           git config user.email 'linuxfabrik-automation[bot]@users.noreply.github.com'
 
+          # A package whose upstream cannot be checked fails the run at the end,
+          # after all other packages had their turn; it must not pass as "up to
+          # date" unnoticed.
+          failed=''
           for dir in packages/*/; do
             name="$(basename "$dir")"
 
-            # check-upstream.sh exits 0 only when a newer upstream version exists.
-            if versions="$(bash build/check-upstream.sh "$dir")"; then
+            # check-upstream.sh: 0 newer upstream, 1 up to date or no upstream,
+            # 2 error
+            rc=0
+            versions="$(bash build/check-upstream.sh "$dir")" || rc=$?
+            if [ "$rc" -eq 2 ]; then
+              echo "::error::Could not check the upstream of $name, see the log above."
+              failed="$failed $name"
+            fi
+            if [ "$rc" -eq 0 ]; then
               current="$(echo "$versions" | cut -d' ' -f1)"
               latest="$(echo "$versions" | cut -d' ' -f2)"
               branch="bump/$name-$latest"
@@ -71,8 +93,13 @@ jobs:
                 --base 'main' \
                 --head "$branch" \
                 --title "build($name): bump to $latest" \
-                --body "Upstream released \`$latest\` (packaged: \`$current\`). Merging this only updates the manifest. Build \`$name\` $latest afterwards with \`gh workflow run lf-build-linux-x86_64.yml --field package=$name\`."
+                --body "Upstream released \`$latest\` (packaged: \`$current\`). Merged automatically once it builds and passes the smoke test on all target distros. The merge publishes \`$name\` $latest to the \`testing\` channel, from where it is promoted to \`release\` after 7 days."
 
               git checkout main
             fi
           done
+
+          if [ -n "$failed" ]; then
+            echo "Upstream check failed for:$failed" >&2
+            exit 1
+          fi

.github/workflows/lf-promote.yml

@@ -0,0 +1,96 @@
+name: 'Linuxfabrik: Promote releases'
+
+# A package lands in the `testing` channel as a GitHub prerelease (lf-release.yml).
+# This workflow promotes every prerelease that is at least 7 days old to a full
+# release, which the repository server then publishes to the `release` channel.
+#
+# Manual intervention:
+#
+#   gh workflow run lf-promote.yml --field release=<tag> --field action=promote
+#     releases a prerelease right away
+#   gh workflow run lf-promote.yml --field release=<tag> --field action=reject
+#     appends "(rejected)" to its title: it is never promoted, and the
+#     repository server removes it from `testing`
+
+on:
+  schedule:
+    # Daily at 05:00 UTC.
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    inputs:
+      action:
+        description: 'What to do with the release given below.'
+        required: false
+        type: 'choice'
+        default: 'promote'
+        options:
+          - 'promote'
+          - 'reject'
+      release:
+        description: 'Tag of a prerelease (e.g. `glances-4.5.8-1`). Leave empty to run the regular 7-day promotion.'
+        required: false
+        default: ''
+
+permissions:
+  contents: 'read'
+
+# Never let the scheduled run and a manual one edit the same release at once.
+concurrency:
+  group: 'lf-promote'
+  cancel-in-progress: false
+
+jobs:
+  promote:
+    runs-on:
+      - 'ubuntu-24.04'
+    permissions:
+      contents: 'write'
+    steps:
+      - name: 'Harden the runner (Audit all outbound calls)'
+        uses: 'step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2' # v2.22.2
+        with:
+          egress-policy: 'audit'
+
+      - name: 'Promote or reject'
+        env:
+          ACTION: '${{ inputs.action }}'
+          GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+          RELEASE: '${{ inputs.release }}'
+          TESTING_DAYS: '7'
+        run: |
+          promote() {
+            echo "Promote $1 to release"
+            gh release edit "$1" --latest=false --prerelease=false --repo "$GITHUB_REPOSITORY"
+          }
+
+          if [ -n "$RELEASE" ]; then
+            read -r prerelease name < <(gh release view "$RELEASE" \
+                --json 'isPrerelease,name' --jq '"\(.isPrerelease) \(.name)"' \
+                --repo "$GITHUB_REPOSITORY")
+            if [ "$prerelease" != 'true' ]; then
+              echo "$RELEASE is not a prerelease (not in testing), nothing to $ACTION." >&2
+              exit 1
+            fi
+            if [[ "$name" == *'(rejected)' ]]; then
+              echo "$RELEASE is already rejected." >&2
+              exit 1
+            fi
+            if [ "$ACTION" = 'reject' ]; then
+              echo "Reject $RELEASE"
+              gh release edit "$RELEASE" --repo "$GITHUB_REPOSITORY" --title "$name (rejected)"
+            else
+              promote "$RELEASE"
+            fi
+            exit 0
+          fi
+
+          cutoff="$(date --utc --date="$TESTING_DAYS days ago" +%Y-%m-%dT%H:%M:%SZ)"
+          gh release list \
+              --exclude-drafts \
+              --json 'isPrerelease,name,publishedAt,tagName' \
+              --limit 1000 \
+              --repo "$GITHUB_REPOSITORY" \
+              --jq ".[] | select(.isPrerelease and (.name | endswith(\"(rejected)\") | not) and .publishedAt <= \"$cutoff\") | .tagName" \
+            | while read -r tag; do
+                promote "$tag"
+              done