ci: informational benchmark workflow

.github/workflows/bench.yml

@@ -0,0 +1,167 @@
+# Criterion benchmarks (bottlecap/benches). Informational: no job here blocks
+# a PR. See bottlecap/benches/README.md, section "CI".
+#
+# - bench-build:   every PR, `cargo bench --no-run` with LTO off (compile only).
+# - bench-measure: PRs with the `bench` label, nightly on main, or manual.
+#                  Production profile. Nightly/full runs save the baseline
+#                  artifact `bench-baseline-main`; a PR run compares against it
+#                  with critcmp and writes the table to the step summary.
+name: Benchmarks
+
+on:
+  pull_request:
+    # `labeled` so the `bench` label starts bench-measure.
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - "bottlecap/**"
+      - "scripts/ci/bench-measure.sh"
+      - ".github/workflows/bench.yml"
+  schedule:
+    # Nightly on main, after the vulnerability scan.
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "smoke: short list, compare with the main baseline. full: whole suite, save the baseline (main only)."
+        type: choice
+        default: smoke
+        options: [smoke, full]
+
+concurrency:
+  # An unrelated label gets its own group, so it does not cancel a running measurement.
+  group: ${{ github.workflow }}-${{ github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'bench' && format('-{0}', github.run_id) || '' }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  PB_VERSION: 25.3
+  PB_URL: https://github.com/protocolbuffers/protobuf/releases/download
+  PB_TARGET: linux-x86_64
+  CARGO_INCREMENTAL: "0"
+  SCCACHE_GHA_ENABLED: "true"
+  RUSTC_WRAPPER: "sccache"
+
+jobs:
+  bench-build:
+    name: Bench Build (informational)
+    # Not on `labeled`: a label must not rebuild; it starts bench-measure.
+    if: github.event_name == 'pull_request' && github.event.action != 'labeled'
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    env:
+      # Compile check only: skip the fat-LTO link, which is the slow part.
+      CARGO_PROFILE_BENCH_LTO: "false"
+      CARGO_PROFILE_BENCH_CODEGEN_UNITS: "16"
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
+        with:
+          cache: false
+      # Nothing to compile before the criterion suite lands; a bench-profile
+      # build of the lib and bin harnesses would only fill the sccache cache.
+      - id: probe
+        working-directory: bottlecap
+        run: |
+          if cargo metadata --no-deps --format-version 1 --locked \
+            | jq -e '.packages[].targets[] | select(.kind | index("bench"))' > /dev/null; then
+            echo "present=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "present=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=No bench targets::no [[bench]] target in bottlecap; nothing to build."
+          fi
+
+      # Same protoc install as rs_ci.yml.
+      - if: steps.probe.outputs.present == 'true'
+        run: sudo apt-get update
+      - if: steps.probe.outputs.present == 'true'
+        run: sudo apt-get install -y curl unzip cmake
+      - if: steps.probe.outputs.present == 'true'
+        name: Install protobuf ${{ env.PB_VERSION }} compiler from binary for ${{ env.PB_TARGET }}
+        run: |
+          curl -LO "${{ env.PB_URL }}/v${{ env.PB_VERSION }}/protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip"
+          unzip "protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" -d "$HOME/.local"
+          export PATH="$PATH:$HOME/.local/bin"
+
+      - if: steps.probe.outputs.present == 'true'
+        uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
+      - id: build
+        if: steps.probe.outputs.present == 'true'
+        continue-on-error: true
+        working-directory: bottlecap
+        run: cargo bench --no-run --locked
+      - if: steps.build.outcome == 'failure'
+        run: |
+          echo "::warning title=Bench build failed::cargo bench --no-run failed; see the job log. Informational, does not block."
+          echo "### Bench build failed (informational)" >> "$GITHUB_STEP_SUMMARY"
+          echo "See the \`cargo bench --no-run\` step of this job." >> "$GITHUB_STEP_SUMMARY"
+
+  bench-measure:
+    name: Bench Measure (informational)
+    if: >-
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'bench') &&
+        (github.event.action != 'labeled' || github.event.label.name == 'bench'))
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+    permissions:
+      contents: read
+      # `gh run download` of the baseline artifact.
+      actions: read
+    env:
+      BENCH_MODE: ${{ (github.event_name == 'schedule' || inputs.mode == 'full') && 'full' || 'smoke' }}
+      BENCH_BASELINE_DIR: ${{ github.workspace }}/.bench-baseline
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y curl unzip cmake
+      - name: Install protobuf ${{ env.PB_VERSION }} compiler from binary for ${{ env.PB_TARGET }}
+        run: |
+          curl -LO "${{ env.PB_URL }}/v${{ env.PB_VERSION }}/protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip"
+          unzip "protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" -d "$HOME/.local"
+          export PATH="$PATH:$HOME/.local/bin"
+
+      - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
+        with:
+          cache: false
+      - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
+      - name: Install critcmp
+        run: cargo install critcmp --locked --version 0.1.8
+
+      - name: Download the main baseline
+        if: env.BENCH_MODE == 'smoke'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: scripts/ci/bench-measure.sh download-baseline "$BENCH_BASELINE_DIR"
+
+      - id: measure
+        continue-on-error: true
+        working-directory: bottlecap
+        run: ../scripts/ci/bench-measure.sh run
+      - if: steps.measure.outcome == 'failure'
+        run: |
+          echo "::warning title=Bench measure failed::a bench panicked or did not build; see the job log. Informational, does not block."
+          echo "### Bench measure failed (informational)" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Save the main baseline
+        if: env.BENCH_MODE == 'full' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: bench-baseline-main
+          path: bottlecap/.bench-export/main.json
+          include-hidden-files: true
+          retention-days: 90
+          if-no-files-found: warn
+      - name: Upload the exported numbers
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: bench-export-${{ github.run_id }}
+          path: bottlecap/.bench-export/
+          include-hidden-files: true
+          retention-days: 14
+          if-no-files-found: ignore

bottlecap/benches/README.md

@@ -130,6 +130,40 @@ faster than production. The ratios between benches, and between main and a fix,
 The lambda-sim environment (`scripts/lambda-sim/`) can reuse this image and the fixture
 corpus to measure the whole extension under a CPU quota.
 
+## CI
+
+`.github/workflows/bench.yml` runs a non-blocking part of this plan: `bench-build` compiles the
+suite on every PR that touches `bottlecap/`, and `bench-measure` (`scripts/ci/bench-measure.sh`)
+runs the smoke list of item 2 on PRs with the `bench` label, nightly on `main`, and on demand,
+then compares it with the last `main` baseline. The list below is the full plan.
+
+Goal: catch a broken bench and a large regression on every PR without a flaky gate.
+
+1. **On every PR, blocking:** `cargo bench --no-run` with
+   `CARGO_PROFILE_BENCH_LTO=false CARGO_PROFILE_BENCH_CODEGEN_UNITS=16` (compile only, a few
+   minutes with the cache), and `cargo test --test fixtures_corpus` (the generators). This
+   keeps the suite compiling as the code changes.
+2. **On every PR, non-blocking job `bench-smoke`:** run a fixed short list with the production
+   profile and `--quick` (about 2 minutes of measurement): `logs/processor_process`,
+   `logs/oom_check`, `traces/process_traces/default`, `traces/decode_v04`,
+   `invocation/trigger/from_value/sqs_event`, `enhanced_metrics/set_max`,
+   `config/get_config/typical`. Save the baseline `pr`, download the baseline artifact from the
+   last `main` run, and post `critcmp main pr -t 10` as a PR comment (or a job summary). The job
+   never fails on a number; it fails only if a bench panics. A shared GitHub runner is noisy,
+   so 10 % is the reporting threshold, and the comment says "informational".
+3. **On `main`, nightly or per merge:** the full suite with the production profile, baseline
+   saved as an artifact (`critcmp --export main > main.json`) and kept 90 days. A second
+   nightly job runs `scripts/bench-docker-arm64.sh` on an arm64 runner for the Linux-only
+   `/proc` benches. Track the exported JSON over time (a small script or a Datadog metric per
+   bench id) and alert on a 20 % move that holds for 3 nights, which filters runner noise.
+4. **Cost note:** `rs_ci.yml` runs `cargo clippy --all-targets` for `default`, `fips` and
+   `default,test-mode`, so the six bench targets compile three times per PR. The `fips` clippy
+   does not run on a Mac (libddwaf-sys aborts at load); check it on Linux before merging.
+5. **Hardware note:** GitHub-hosted runners vary between runs. If the numbers matter more
+   than the trend, use a self-hosted or dedicated runner and pin the CPU set. Until then, the
+   nightly trend and the PR comment are hints; the developer confirms a regression locally
+   with `scripts/bench-compare.sh`.
+
 ## Backlog to bench
 
 Review = `consolidated-findings-v3` (Confluence page 7384761389). Numbers are the first

scripts/ci/bench-measure.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Helper for .github/workflows/bench.yml (job bench-measure).
+#
+#   bench-measure.sh download-baseline <dir>   # latest nightly `bench-baseline-main` artifact (needs gh)
+#   bench-measure.sh run                        # from bottlecap/; BENCH_MODE=smoke|full
+#
+# smoke: a short fixed list with `--quick`, exported as pr.json, compared with
+#        <BENCH_BASELINE_DIR>/main.json when present (critcmp, 10 % threshold).
+# full:  the whole suite, baseline "main", exported as main.json.
+# Exports land in .bench-export/. With no [[bench]] target in the checkout the
+# run is a no-op, so the job is green before the suite lands on main.
+set -euo pipefail
+
+export_dir=".bench-export"
+# bottlecap/benches/README.md, "CI" item 2: one id per hot path.
+smoke_list="logs:processor_process logs:oom_check traces:process_traces/default traces:decode_v04 invocation:trigger/from_value/sqs_event enhanced_metrics:set_max config:get_config/typical"
+
+summary() {
+  if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then tee -a "$GITHUB_STEP_SUMMARY"; else cat; fi
+}
+
+case "${1:-}" in
+download-baseline)
+  dir="${2:?usage: bench-measure.sh download-baseline <dir>}"
+  mkdir -p "$dir"
+  # A nightly whose measure step failed is still green but has no artifact, so walk back to
+  # the newest run that has one.
+  run_ids="$(gh run list --workflow bench.yml --branch main --event schedule --status success --limit 10 \
+    --json databaseId --jq '.[].databaseId' || true)"
+  for run_id in $run_ids; do
+    if gh run download "$run_id" --name bench-baseline-main --dir "$dir"; then
+      echo "baseline: run $run_id -> $dir/main.json"
+      exit 0
+    fi
+    echo "run $run_id has no bench-baseline-main artifact; trying an older run"
+  done
+  echo "no bench-baseline-main artifact in the last 10 successful nightly runs on main; no baseline"
+  ;;
+
+run)
+  mode="${BENCH_MODE:-smoke}"
+  baseline_dir="${BENCH_BASELINE_DIR:-}"
+  targets="$(cargo metadata --no-deps --format-version 1 --locked \
+    | jq -r '.packages[].targets[] | select(.kind | index("bench")) | .name')"
+  if [ -z "$targets" ]; then
+    echo "no [[bench]] targets in this checkout; nothing to measure"
+    exit 0
+  fi
+  mkdir -p "$export_dir"
+  if [ "$mode" = "full" ]; then
+    cargo bench --locked -- --save-baseline main
+    critcmp --export main > "$export_dir/main.json"
+    { echo "### Bench full run (baseline main)"; echo '```'; critcmp main; echo '```'; } | summary
+    exit 0
+  fi
+  for pair in $smoke_list; do
+    bench="${pair%%:*}"
+    filter="${pair#*:}"
+    if ! grep -qx "$bench" <<<"$targets"; then
+      echo "skip $bench/$filter: no such bench target"
+      continue
+    fi
+    cargo bench --locked --bench "$bench" -- --save-baseline pr --quick "$filter"
+  done
+  critcmp --export pr > "$export_dir/pr.json"
+  if [ -n "$baseline_dir" ] && [ -f "$baseline_dir/main.json" ]; then
+    cp "$baseline_dir/main.json" "$export_dir/main.json"
+    {
+      echo "### Bench smoke vs main (informational: shared runner, differences under 10 % hidden)"
+      echo '```'
+      critcmp "$export_dir/main.json" "$export_dir/pr.json" -t 10
+      echo '```'
+    } | summary
+  else
+    { echo "### Bench smoke (no main baseline yet)"; echo '```'; critcmp "$export_dir/pr.json"; echo '```'; } | summary
+  fi
+  ;;
+
+*)
+  sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'
+  exit 2
+  ;;
+esac