diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 000000000..0dde52470 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,167 @@ +# Criterion benchmarks (bottlecap/benches). Informational: no job here blocks +# a PR. See bottlecap/benches/README.md, section "CI". +# +# - bench-build: every PR, `cargo bench --no-run` with LTO off (compile only). +# - bench-measure: PRs with the `bench` label, nightly on main, or manual. +# Production profile. Nightly/full runs save the baseline +# artifact `bench-baseline-main`; a PR run compares against it +# with critcmp and writes the table to the step summary. +name: Benchmarks + +on: + pull_request: + # `labeled` so the `bench` label starts bench-measure. + types: [opened, synchronize, reopened, labeled] + paths: + - "bottlecap/**" + - "scripts/ci/bench-measure.sh" + - ".github/workflows/bench.yml" + schedule: + # Nightly on main, after the vulnerability scan. + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + mode: + description: "smoke: short list, compare with the main baseline. full: whole suite, save the baseline (main only)." + type: choice + default: smoke + options: [smoke, full] + +concurrency: + # An unrelated label gets its own group, so it does not cancel a running measurement. + group: ${{ github.workflow }}-${{ github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'bench' && format('-{0}', github.run_id) || '' }} + cancel-in-progress: true + +permissions: + contents: read + +env: + PB_VERSION: 25.3 + PB_URL: https://github.com/protocolbuffers/protobuf/releases/download + PB_TARGET: linux-x86_64 + CARGO_INCREMENTAL: "0" + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + bench-build: + name: Bench Build (informational) + # Not on `labeled`: a label must not rebuild; it starts bench-measure. + if: github.event_name == 'pull_request' && github.event.action != 'labeled' + runs-on: ubuntu-22.04 + timeout-minutes: 45 + env: + # Compile check only: skip the fat-LTO link, which is the slow part. + CARGO_PROFILE_BENCH_LTO: "false" + CARGO_PROFILE_BENCH_CODEGEN_UNITS: "16" + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + # Nothing to compile before the criterion suite lands; a bench-profile + # build of the lib and bin harnesses would only fill the sccache cache. + - id: probe + working-directory: bottlecap + run: | + if cargo metadata --no-deps --format-version 1 --locked \ + | jq -e '.packages[].targets[] | select(.kind | index("bench"))' > /dev/null; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "::notice title=No bench targets::no [[bench]] target in bottlecap; nothing to build." + fi + + # Same protoc install as rs_ci.yml. + - if: steps.probe.outputs.present == 'true' + run: sudo apt-get update + - if: steps.probe.outputs.present == 'true' + run: sudo apt-get install -y curl unzip cmake + - if: steps.probe.outputs.present == 'true' + name: Install protobuf ${{ env.PB_VERSION }} compiler from binary for ${{ env.PB_TARGET }} + run: | + curl -LO "${{ env.PB_URL }}/v${{ env.PB_VERSION }}/protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" + unzip "protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" -d "$HOME/.local" + export PATH="$PATH:$HOME/.local/bin" + + - if: steps.probe.outputs.present == 'true' + uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - id: build + if: steps.probe.outputs.present == 'true' + continue-on-error: true + working-directory: bottlecap + run: cargo bench --no-run --locked + - if: steps.build.outcome == 'failure' + run: | + echo "::warning title=Bench build failed::cargo bench --no-run failed; see the job log. Informational, does not block." + echo "### Bench build failed (informational)" >> "$GITHUB_STEP_SUMMARY" + echo "See the \`cargo bench --no-run\` step of this job." >> "$GITHUB_STEP_SUMMARY" + + bench-measure: + name: Bench Measure (informational) + if: >- + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'bench') && + (github.event.action != 'labeled' || github.event.label.name == 'bench')) + runs-on: ubuntu-22.04 + timeout-minutes: 120 + permissions: + contents: read + # `gh run download` of the baseline artifact. + actions: read + env: + BENCH_MODE: ${{ (github.event_name == 'schedule' || inputs.mode == 'full') && 'full' || 'smoke' }} + BENCH_BASELINE_DIR: ${{ github.workspace }}/.bench-baseline + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - run: sudo apt-get update + - run: sudo apt-get install -y curl unzip cmake + - name: Install protobuf ${{ env.PB_VERSION }} compiler from binary for ${{ env.PB_TARGET }} + run: | + curl -LO "${{ env.PB_URL }}/v${{ env.PB_VERSION }}/protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" + unzip "protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" -d "$HOME/.local" + export PATH="$PATH:$HOME/.local/bin" + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Install critcmp + run: cargo install critcmp --locked --version 0.1.8 + + - name: Download the main baseline + if: env.BENCH_MODE == 'smoke' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: scripts/ci/bench-measure.sh download-baseline "$BENCH_BASELINE_DIR" + + - id: measure + continue-on-error: true + working-directory: bottlecap + run: ../scripts/ci/bench-measure.sh run + - if: steps.measure.outcome == 'failure' + run: | + echo "::warning title=Bench measure failed::a bench panicked or did not build; see the job log. Informational, does not block." + echo "### Bench measure failed (informational)" >> "$GITHUB_STEP_SUMMARY" + + - name: Save the main baseline + if: env.BENCH_MODE == 'full' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: bench-baseline-main + path: bottlecap/.bench-export/main.json + include-hidden-files: true + retention-days: 90 + if-no-files-found: warn + - name: Upload the exported numbers + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: bench-export-${{ github.run_id }} + path: bottlecap/.bench-export/ + include-hidden-files: true + retention-days: 14 + if-no-files-found: ignore diff --git a/bottlecap/benches/README.md b/bottlecap/benches/README.md index 939f10cee..8938a60e0 100644 --- a/bottlecap/benches/README.md +++ b/bottlecap/benches/README.md @@ -127,6 +127,40 @@ faster than production. The ratios between benches, and between main and a fix, A lambda-sim environment (not in this repository) can reuse this image and the fixture corpus to measure the whole extension under a CPU quota. +## CI + +`.github/workflows/bench.yml` runs a non-blocking part of this plan: `bench-build` compiles the +suite on every PR that touches `bottlecap/`, and `bench-measure` (`scripts/ci/bench-measure.sh`) +runs the smoke list of item 2 on PRs with the `bench` label, nightly on `main`, and on demand, +then compares it with the last `main` baseline. The list below is the full plan. + +Goal: catch a broken bench and a large regression on every PR without a flaky gate. + +1. **On every PR, blocking:** `cargo bench --no-run` with + `CARGO_PROFILE_BENCH_LTO=false CARGO_PROFILE_BENCH_CODEGEN_UNITS=16` (compile only, a few + minutes with the cache), and `cargo test --test fixtures_corpus` (the generators). This + keeps the suite compiling as the code changes. +2. **On every PR, non-blocking job `bench-smoke`:** run a fixed short list with the production + profile and `--quick`: `logs/processor_process`, `logs/oom_check`, + `traces/process_traces/default`, `traces/decode_v04`, + `invocation/trigger/from_value/sqs_event`, `enhanced_metrics/set_max`, + `config/get_config/typical`. Save the baseline `pr`, download the baseline artifact from the + last `main` run, and post `critcmp main pr -t 10` as a PR comment (or a job summary). The job + never fails on a number; it fails only if a bench panics. A shared GitHub runner is noisy, + so 10 % is the reporting threshold, and the comment says "informational". +3. **On `main`, nightly or per merge:** the full suite with the production profile, baseline + saved as an artifact (`critcmp --export main > main.json`) and kept 90 days. A second + nightly job runs `scripts/bench-docker-arm64.sh` on an arm64 runner for the Linux-only + `/proc` benches. Track the exported JSON over time (a small script or a Datadog metric per + bench id) and alert on a 20 % move that holds for 3 nights, which filters runner noise. +4. **Cost note:** `rs_ci.yml` runs `cargo clippy --all-targets` for `default`, `fips` and + `default,test-mode`, so the six bench targets compile three times per PR. The `fips` clippy + does not run on a Mac (libddwaf-sys aborts at load); check it on Linux before merging. +5. **Hardware note:** GitHub-hosted runners vary between runs. If the numbers matter more + than the trend, use a self-hosted or dedicated runner and pin the CPU set. Until then, the + nightly trend and the PR comment are hints; the developer confirms a regression locally + with `scripts/bench-compare.sh`. + ## Review findings to benches The investigation numbers are in the Confluence page [Bottlecap review: combined findings](https://datadoghq.atlassian.net/wiki/spaces/~93500838/pages/7384761389); the bench CI workflow (`.github/workflows/bench.yml`) has the current numbers. diff --git a/scripts/ci/bench-measure.sh b/scripts/ci/bench-measure.sh new file mode 100755 index 000000000..b60f140cd --- /dev/null +++ b/scripts/ci/bench-measure.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Helper for .github/workflows/bench.yml (job bench-measure). +# +# bench-measure.sh download-baseline