diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 27ac4fa44..6c0a95001 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,15 +8,15 @@ name: Bench # A job is one CLI benchmark, or one `benches/` target with all the benchmarks it prints. # scripts/ab.sh does all of it, and runs the same comparison on a developer's machine. # -# The x86-64 testbed is the self-hosted runner `size-attester7870-x64` (with `self-hosted-ghr`, -# an ephemeral VM per job: an Intel Xeon Gold 6548N, 32 vCPUs, 64 GB): base and PR build and -# run on that one machine, so it is built for -# `target-cpu=native`, AVX-512, VPCLMULQDQ and GFNI included, and its cache is keyed on the CPU -# model so a native build never lands on another CPU. A fork has no such runner, so in a fork's -# own repository the testbed falls back to a GitHub-hosted runner built for haswell (a fork's PR -# to this repository runs here, on the self-hosted runner). The arm64 testbed is GitHub-hosted -# and pins its ISA: `target-cpu=native` would build for whichever CPU the job lands on, and a -# cached build then dies of SIGILL on an older one (see rust.yml). +# The testbeds are the self-hosted runners `size-attester7870-x64` and `size-attester7870-arm64` +# (with `self-hosted-ghr`, an ephemeral VM per job; x86-64 an Intel Xeon Gold 6548N, 32 vCPUs, +# 64 GB; arm64 an Arm Neoverse-V2, 16 vCPUs, 64 GB): base and PR build and run on one machine, so +# each is built for `target-cpu=native` (on x86-64 AVX-512, VPCLMULQDQ and GFNI included), and +# its cache is keyed on the CPU model so a native build never lands on another CPU. A fork has no +# such runners, so in a fork's own repository the testbeds fall back to GitHub-hosted runners with +# a pinned ISA, x86-64 built for haswell and arm64 for neoverse-n2 (a fork's PR to this repository +# runs here, on the self-hosted runners): `target-cpu=native` would build for whichever CPU the +# job lands on, and a cached build then dies of SIGILL on an older one (see rust.yml). # # Run from the Actions tab (`workflow_dispatch`), it compares any two refs the same way: `head` # (default: the branch it is run on) against `base` (default: the head's first parent). Such a @@ -59,11 +59,14 @@ jobs: - name: x86-64 runner: ${{ github.repository == 'leanEthereum/leanVM' && fromJSON('["size-attester7870-x64", "self-hosted-ghr"]') || 'ubuntu-latest' }} rustflags: ${{ github.repository == 'leanEthereum/leanVM' && '-C target-cpu=native' || '-C target-cpu=haswell' }} - # The NEON kernels, which are chosen at compile time on `aes` (PMULL) and `sha3` - # (EOR3), neither of which a generic aarch64 target enables. + # The NEON kernels are chosen at compile time on `aes` (PMULL) and `sha3` (EOR3), which + # a generic aarch64 target does not enable: the native build enables them where the CPU + # has them, the fork's neoverse-n2 pin names them, and the job fails early without them. - name: arm64 - runner: ubuntu-24.04-arm - rustflags: -C target-cpu=neoverse-n2 -C target-feature=+aes,+sha3 + runner: ${{ github.repository == 'leanEthereum/leanVM' && fromJSON('["size-attester7870-arm64", "self-hosted-ghr"]') || 'ubuntu-24.04-arm' }} + rustflags: ${{ github.repository == 'leanEthereum/leanVM' && '-C target-cpu=native' || '-C target-cpu=neoverse-n2 -C target-feature=+aes,+sha3' }} + # What the CPU must have to run that build. + cpu: ${{ github.repository == 'leanEthereum/leanVM' && 'aes pmull sha3' || 'aes pmull sha3 sve2' }} # The cases of `proven()` and `proven_trees()` in bins/leanvm/src/tracked.rs, run by the # CLI (`-1thread`, one thread: the CLI proves it, a tree's leaf included, with # `LEANVM_NUM_THREADS=1`; a tree's case, `Nto1` for N proofs combined into 1, prints two @@ -97,26 +100,37 @@ jobs: # The head (a PR's merge commit) and its first parent, the default base. fetch-depth: 2 - uses: dtolnay/rust-toolchain@stable - # What the x86 build can use, so the PR comment's numbers can be read against the hardware. + # What the build can use, so the PR comment's numbers can be read against the hardware. # The model goes into the cache key: a native build is only reused on the CPU it was built for. - name: CPU id: cpu - if: runner.arch == 'X64' run: | - lscpu | grep -E '^(Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per socket|L2 cache|L3 cache):' + lscpu | grep -E '^(Vendor ID|Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per (socket|cluster)|L1d cache|L2 cache|L3 cache):' free -g | grep -E '^Mem:' - flags=$(grep -m1 -E '^flags' /proc/cpuinfo) - for f in avx2 avx512f avx512bw avx512vl avx512vbmi avx512ifma vpclmulqdq gfni; do + if [ "$RUNNER_ARCH" = ARM64 ]; then + flags=$(grep -m1 -E '^Features' /proc/cpuinfo) + wanted="asimd aes pmull sha3 sve sve2 svepmull svebitperm i8mm bf16" + else + flags=$(grep -m1 -E '^flags' /proc/cpuinfo) + wanted="avx2 avx512f avx512bw avx512vl avx512vbmi avx512ifma vpclmulqdq gfni" + fi + for f in $wanted; do if grep -qw "$f" <<<"$flags"; then echo "$f: yes"; else echo "$f: no"; fi done echo "model=-$(lscpu | sed -n 's/^Model name: *//p' | head -n1 | tr -c 'A-Za-z0-9\n' '-')" >> "$GITHUB_OUTPUT" - # The CPU the testbed's ISA is pinned for must have what it enables, or it dies of SIGILL. - - name: CPU features - if: runner.os == 'Linux' && runner.arch == 'ARM64' + # The build must enable the NEON kernels' features, and the CPU must have what the build + # enables, or the comparison measures the fallback or dies of SIGILL. + - name: NEON kernels + if: runner.arch == 'ARM64' + env: + CPU_FEATURES: ${{ matrix.testbed.cpu }} run: | - grep -m1 -E '^Features' /proc/cpuinfo - for f in aes pmull sha3 sve2; do grep -qw "$f" <(grep -m1 -E '^Features' /proc/cpuinfo) || { echo "::error::no $f on this runner"; exit 1; }; done - # One cache per testbed, and on x86 per CPU model; RUSTFLAGS is part of its key. + read -ra flags <<<"$RUSTFLAGS" + cfg=$(rustc --print cfg "${flags[@]}") + for f in aes sha3; do grep -qx "target_feature=\"$f\"" <<<"$cfg" || { echo "::error::$RUSTFLAGS does not enable $f"; exit 1; }; done + features=$(grep -m1 -E '^Features' /proc/cpuinfo) + for f in $CPU_FEATURES; do grep -qw "$f" <<<"$features" || { echo "::error::no $f on this runner"; exit 1; }; done + # One cache per testbed and CPU model; RUSTFLAGS is part of its key. - uses: Swatinem/rust-cache@v2 with: key: ${{ matrix.testbed.name }}${{ steps.cpu.outputs.model }} diff --git a/AGENTS.md b/AGENTS.md index d68ec0493..b6cef040a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -189,7 +189,7 @@ Addition with carries, comparisons, shifts, AND/OR, multiplication and division - **Witness generation is word arithmetic wherever a circuit has a generator, and the generic walk of the gate list, bit by bit, elsewhere**: ALU, LOAD, STORE, LD, SD, SHIFT, MULH and HASH fill by `Fill::Instance` (`ClassCircuit::fill`), MUL by `Fill::Batch8` on AVX2 only, and every clock circuit has one (`ClassSpec::clock_witness`: `Clock::witness`, and for EXT's, whose ports past the clock's split the flags and add 8 and 16 to the pointers, `Ext::clock_witness` on top of `Clock::witness_with`); DIV and MUL off AVX2 walk. Each generator computes the circuit's values natively and writes its rows in the order the circuit makes its products (all but the multipliers' through `rv::circuits::Products`), so a change to a circuit's gate list must change its generator too. Each is pinned to the walk by a test that covers edge words (zero, all ones, the sign bits, every shift amount, every misalignment) and random instances: `semantics::tests::word_witness_is_the_walk` for ALU, the memory classes and SHIFT, and their own for MUL, MULH, HASH and the clock circuits (`clock::tests::the_word_witness_is_the_gate_walk`, every table's, EXT's included). A new generator needs such a test. HASH's generator is unrolled, so every carry run's place is a constant. - **Circuit sizes are structure, not measurements**: `k_log` (its `ClassCircuit`'s) and `clock_k_log` per class are pinned in its `ClassSpec` and asserted against the built circuits; product counts are in `doc/leanvm` Annex C. A product can be committed straight at an output port's position (`flock::circuit::Builder::and_output`), which saves the copy; the clock circuit's step uses it. -CI compares every PR with its base, never with a history. The base is the merge commit's first parent; a workflow builds and runs both on one runner and saves both as an artifact, and a `workflow_run` workflow on the default branch (`main`, the only branch GitHub runs those from) posts them on the PR as one comment, edited on every run and deleted once nothing changed. The posting workflows never run the PR's code: they take only the PR number from a run's artifact, a fork's included, post nothing unless GitHub's API says that PR's head is the commit the run measured, and check every name and number before it reaches the comment. **Exact counts**: `counts.yml` runs `cargo leanvm bench --cycles-only`, every program at the sizes above without a proof (`cycles`, the program's own instructions; `proven-rows`, table heights after padding; `committed`, witness words) and the two aggregation trees above, one case per tree and kind of node (`aggregate-leanxmss-400-2to1-first` and `-node`, `aggregate-leanxmss-400-4to1-first` and `-node`, from `counted_trees()`: `rows`, the circuit's own; `proven-rows` and `committed` as for a program), puts the tables in each run's summary from the same pass (`--markdown-file "$GITHUB_STEP_SUMMARY"`, which appends what `--markdown` prints), saves the artifact `counts` and fails on any increase; `counts-comment.yml` posts the counts that changed. A tree's name says its shape, `Nto1` for N proofs combined into 1 at every node; `-first` is a first-level node (the RISC-V verifier in rows over N leaf proofs) and `-node` a higher node (the recursion verifier in rows over N child proofs). **Proving**: `bench.yml` runs one job per testbed and benchmark: each case of `proven()` in `bins/leanvm/src/tracked.rs` (at the sizes a 16 GB runner holds: leanXMSS, leanSPHINCS, Falcon, the state proofs and the shielded transfers at a quarter, no leanDA; leanXMSS also on one thread, `leanxmss-100-1thread`, which the CLI proves in a process of its own with `LEANVM_NUM_THREADS=1` since the pool is sized once per process), each of `proven_trees()` (the two aggregation trees above over leanXMSS at a quarter, `aggregate-leanxmss-100-2to1` and `aggregate-leanxmss-100-4to1`: the leaf proven once, then one first-level node and one node over copies of it, the two benchmarks `-first` and `-node`, each node's stages the children of a `Prove` span around it and the node's peak memory including the first-level node's; the 2-to-1 tree also on one thread, `aggregate-leanxmss-100-2to1-1thread`, proven whole, leaf included, in a process of its own like `leanxmss-100-1thread`), and each `benches/` target above with every benchmark it prints (flock's two at `FLOCK_N_LOG=18`, pcs's `throughput` at its default size). Each job builds both sides and runs them in turns, base, PR, PR, base, for `ROUNDS` rounds: `cargo leanvm bench --only ` gives the proving time, the proof's size in bytes (exact: the inputs are seeded and grinding takes the smallest nonce), the time of each of the proof's stages (`stage.`, the `Prove` span's direct children, `--tracing`'s first level, timed by a `tracing` layer that enables spans only), the process's peak resident memory (`peak-memory`, that case's alone since each job proves one), the proving passes' heap as the global allocator counts it (`heap-peak`, the most bytes live at once above those live when a pass began, and `allocations`, its allocations and reallocations, each the largest over the measured passes; host-side setup is outside them), and the verifying time, and a target's `--json` its own measures: `latency` for a whole operation (flock's proving times with the witness excluded, pcs's commit and open), `per-op` in nanoseconds per call for `kernels`, a hashed block of `hash_throughput` and one NTT encode. `bench-comment.yml` posts a size that changed and a time that moved the same way in every round with a median ratio at least 1% from one, every result collapsed beneath; it gates nothing. Run in turns on one runner, the comparison does not depend on which machine the job landed on, which a history across GitHub's runners does (one runner label spans CPU models of different speeds). The testbeds are `x86-64`, the self-hosted runner `size-attester7870-x64` (an Intel Xeon Gold 6548N, 32 vCPUs, 64 GB), built for `target-cpu=native` since base and PR build and run on that one machine, so it measures the AVX-512, VPCLMULQDQ and GFNI arms (the job prints the CPU and those flags, and keys its cache on the CPU model so a native build is never reused on another CPU; in a fork's own repository, which has no such runner, the testbed falls back to `ubuntu-latest` built for `target-cpu=haswell`), and `arm64`, GitHub-hosted (`ubuntu-24.04-arm`, 4 vCPUs, `target-cpu=neoverse-n2` plus `+aes,+sha3`, since the NEON field kernels are chosen at compile time on those features and a generic aarch64 target enables neither; the job fails early if the CPU lacks them), its ISA pinned because a cached native build dies of SIGILL on an older CPU, so a change to how the prover scales across many cores shows there only in part. A benchmark's name is what base and PR are matched by, so renaming one or changing its input shows it as new, as does one the base cannot run (it cannot build the target, or its first base run fails or prints no JSON, and the job carries on with the PR's runs alone). +CI compares every PR with its base, never with a history. The base is the merge commit's first parent; a workflow builds and runs both on one runner and saves both as an artifact, and a `workflow_run` workflow on the default branch (`main`, the only branch GitHub runs those from) posts them on the PR as one comment, edited on every run and deleted once nothing changed. The posting workflows never run the PR's code: they take only the PR number from a run's artifact, a fork's included, post nothing unless GitHub's API says that PR's head is the commit the run measured, and check every name and number before it reaches the comment. **Exact counts**: `counts.yml` runs `cargo leanvm bench --cycles-only`, every program at the sizes above without a proof (`cycles`, the program's own instructions; `proven-rows`, table heights after padding; `committed`, witness words) and the two aggregation trees above, one case per tree and kind of node (`aggregate-leanxmss-400-2to1-first` and `-node`, `aggregate-leanxmss-400-4to1-first` and `-node`, from `counted_trees()`: `rows`, the circuit's own; `proven-rows` and `committed` as for a program), puts the tables in each run's summary from the same pass (`--markdown-file "$GITHUB_STEP_SUMMARY"`, which appends what `--markdown` prints), saves the artifact `counts` and fails on any increase; `counts-comment.yml` posts the counts that changed. A tree's name says its shape, `Nto1` for N proofs combined into 1 at every node; `-first` is a first-level node (the RISC-V verifier in rows over N leaf proofs) and `-node` a higher node (the recursion verifier in rows over N child proofs). **Proving**: `bench.yml` runs one job per testbed and benchmark: each case of `proven()` in `bins/leanvm/src/tracked.rs` (at the sizes a 16 GB runner holds: leanXMSS, leanSPHINCS, Falcon, the state proofs and the shielded transfers at a quarter, no leanDA; leanXMSS also on one thread, `leanxmss-100-1thread`, which the CLI proves in a process of its own with `LEANVM_NUM_THREADS=1` since the pool is sized once per process), each of `proven_trees()` (the two aggregation trees above over leanXMSS at a quarter, `aggregate-leanxmss-100-2to1` and `aggregate-leanxmss-100-4to1`: the leaf proven once, then one first-level node and one node over copies of it, the two benchmarks `-first` and `-node`, each node's stages the children of a `Prove` span around it and the node's peak memory including the first-level node's; the 2-to-1 tree also on one thread, `aggregate-leanxmss-100-2to1-1thread`, proven whole, leaf included, in a process of its own like `leanxmss-100-1thread`), and each `benches/` target above with every benchmark it prints (flock's two at `FLOCK_N_LOG=18`, pcs's `throughput` at its default size). Each job builds both sides and runs them in turns, base, PR, PR, base, for `ROUNDS` rounds: `cargo leanvm bench --only ` gives the proving time, the proof's size in bytes (exact: the inputs are seeded and grinding takes the smallest nonce), the time of each of the proof's stages (`stage.`, the `Prove` span's direct children, `--tracing`'s first level, timed by a `tracing` layer that enables spans only), the process's peak resident memory (`peak-memory`, that case's alone since each job proves one), the proving passes' heap as the global allocator counts it (`heap-peak`, the most bytes live at once above those live when a pass began, and `allocations`, its allocations and reallocations, each the largest over the measured passes; host-side setup is outside them), and the verifying time, and a target's `--json` its own measures: `latency` for a whole operation (flock's proving times with the witness excluded, pcs's commit and open), `per-op` in nanoseconds per call for `kernels`, a hashed block of `hash_throughput` and one NTT encode. `bench-comment.yml` posts a size that changed and a time that moved the same way in every round with a median ratio at least 1% from one, every result collapsed beneath; it gates nothing. Run in turns on one runner, the comparison does not depend on which machine the job landed on, which a history across GitHub's runners does (one runner label spans CPU models of different speeds). The testbeds are self-hosted runners, `x86-64` on `size-attester7870-x64` (an Intel Xeon Gold 6548N, 32 vCPUs, 64 GB) and `arm64` on `size-attester7870-arm64` (an Arm Neoverse-V2, 16 vCPUs, 64 GB), each built for `target-cpu=native` since base and PR build and run on that one machine: x86-64 measures the AVX-512, VPCLMULQDQ and GFNI arms, and arm64 the NEON field kernels, which are chosen at compile time on `aes` and `sha3` (a generic aarch64 target enables neither, so the arm64 job fails early if its build does not enable them or the CPU lacks what the build enables). Each job prints the CPU and its relevant features, and keys its cache on the CPU model so a native build is never reused on another CPU. In a fork's own repository, which has no such runners, the testbeds fall back to GitHub-hosted runners with a pinned ISA, since a cached native build dies of SIGILL on an older CPU: `ubuntu-latest` built for `target-cpu=haswell`, and `ubuntu-24.04-arm` built for `target-cpu=neoverse-n2` plus `+aes,+sha3`. A benchmark's name is what base and PR are matched by, so renaming one or changing its input shows it as new, as does one the base cannot run (it cannot build the target, or its first base run fails or prints no JSON, and the job carries on with the PR's runs alone). `bench.yml`'s jobs each run `scripts/ab.sh --ci` on their benchmark, and the script owns the whole comparison (which benchmarks are `benches/` targets, the settings they run with, the order of the rounds, the tolerance of a base that cannot run, `ab.json`), so a local run of it makes the comparison CI makes. `counts.yml` keeps its own steps; `scripts/ab.sh counts` is the same comparison. Run from the Actions tab (`workflow_dispatch`), `bench.yml` compares any two refs on the same testbeds (inputs `head`, default the branch it runs on, and `base`, default the head's first parent, passed to `ab.sh --base`) and leaves its artifacts only, since `bench-comment.yml` posts for PRs alone.