diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 2798ac628..000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Deploy Site - -on: - workflow_dispatch: - push: - branches: [main] - paths: - - 'site/**' - - 'scripts/gen_leaderboard_data.py' - - 'results/**' - -permissions: - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - build: - # Don't deploy from forks (e.g. when syncing main from upstream). The - # GitHub Pages deployment isn't available on forks and the run would fail. - if: github.event.repository.fork == false - runs-on: ubuntu-latest - steps: - # Full history: the sitemap dates each page from the last commit that - # touched its source, and a shallow clone can only say "today" for - # everything. Without it the generator drops lastmod rather than guess. - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - # Only the og:image cards need it. The generator runs fine without it and - # simply leaves the cards (and the tags pointing at them) out, so a local - # run still produces the whole site. - # - # continue-on-error keeps that true here: `run:` executes under `bash -e`, - # so a PyPI hiccup or a runner image that starts enforcing PEP 668 - # (externally-managed-environment) would otherwise fail the job before the - # generator runs and stop the site updating at all. Losing the cards for - # one deploy is the smaller failure. Pinned so the deploy job, which holds - # pages:write and id-token:write, does not pull an unreviewed release. - - name: Install Pillow (social cards) - continue-on-error: true - run: python3 -m pip install --quiet --disable-pip-version-check 'pillow==11.3.0' - - - name: Generate leaderboard data - run: python3 scripts/gen_leaderboard_data.py - - # The README badges carry a rank computed by a Python port of the board's - # own scoring. Publishing a badge that disagrees with the page it links to - # is worse than publishing none, so a drifted port fails the deploy. - - name: Check badge ranks against the board - run: node scripts/check_badge_parity.js - - # The layout lives in the script, which publish-genhttp.yml lays out the same site with. - - name: Assemble site — standalone leaderboard at root - run: scripts/assemble_site.sh - - - name: Upload artifact - uses: actions/upload-pages-artifact@v4 - with: - path: site/public - - deploy: - needs: build - if: github.event.repository.fork == false - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index bd803e383..101939f87 100644 --- a/README.md +++ b/README.md @@ -1,180 +1,180 @@ -# HttpArena - -[![Discord](https://discordapp.com/api/guilds/1177529388229734410/widget.png?style=shield)](https://discord.com/invite/H84B5ZqDXR) - -## Hardware Upgrade -Hi, thank you for visiting or contributing to our project, we are always looking to improve this benchmark platform, if you wish to help us by sponsoring or donating, all the money is invested into infrastructure costs, we are currently aiming for hardware upgrades that would substantially improve our benchmarks. - -HTTP framework benchmark platform. - -30 test profiles. 64-core dedicated hardware. Same conditions for every framework. - -[View Leaderboard](https://www.http-arena.com/) | [Documentation](https://www.http-arena.com/#doc=) | [Add a Framework](https://www.http-arena.com/#doc=add-framework) - ---- - -## PR Commands - -| Command | Description | -|---------|-------------| -| `/benchmark -f ` | Run every test the framework subscribes to | -| `/benchmark -f -t ` | Run one test only | -| `/benchmark -f --save` | Run and save results (updates the leaderboard on merge) | -| `/benchmark -f -t --save` | Run one test and save results | -| `/benchmark -f --compare ` | Measure the deltas against another framework instead of this one | -| `/benchmark-multiple -f ,,...` | Benchmark several frameworks in one run; takes `-t` and `--save` too, and saved results land in a single commit | -| `/benchmark-multiple --save` | No `-f` needed: benchmark and save every framework the PR touches | -| `/benchmark-test -t ` | Benchmark **all** enabled frameworks subscribed to `` and save the results | - -For `/benchmark`, always specify `-f `; the flags combine in any order. Results come back as a comment with a per-profile table of RPS, p99, CPU and memory, one table per framework on multi runs. A new benchmark comment while a run is in flight queues behind it (one deep) instead of cancelling it. For multi-framework PRs (dependency bumps, same-language refactors) prefer `/benchmark-multiple`, which runs everything in a single job and commits all saved results together, so no run overwrites another. `--compare` works on single-framework runs only. - -**What the deltas are measured against.** By default, this framework's own results published on `main` - answering *"did this change help?"*. When you are tuning a variant or a successor entry, `--compare` re-bases them on another entry instead: - -``` -/benchmark -f genhttp --compare genhttp-kestrel -``` - -The reply states which baseline it used, and profiles the other framework does not run show `n/a` rather than a delta. - ---- - -## Test Profiles - -| Category | Profiles | Description | -|----------|----------|-------------| -| Connection | `baseline`, `pipelined` *, `limited-conn` | Mixed GET/POST with query parsing (512/4K conns), 16× batched pipelining (reference-only, shown faded, excluded from the composite score), short-lived connections that close after 10 requests | -| Concurrency | `async` | `GET /delay/{ms}` with the wait named in the route, 32K held connections. Isolates what a framework does while a request is pending, with no database or I/O in the way. Scored for frameworks and engines, not for infrastructure, since the spec settled at a flat 10 ms over 32K connections | -| Efficiency | `latency-1m`, `latency-10k`, `latency-500k-8cpu` * | One million req/s offered at a fixed rate with zrk; the metric is CPU spent per request, read exactly from the container's cgroup rather than sampled. Everyone who finishes serves the same million, so what separates them is the bill. Scored on its own 0-100 formula rather than on rps, since the rate is pinned. `latency-10k` is the identical setup - same cores, connections and endpoint - at a near-idle 10K req/s, so what it reports is the standing cost of being a running server rather than of serving load, and it is scored the same way. Both score CPU at 0.50, p99 at 0.25 and mean latency at 0.25, the mean being the queueing signal: ten times the best mean is worth nothing. `latency-500k-8cpu` pins 500K req/s with the server on four cores plus their SMT siblings, so the load sits near saturation and the queue a server builds shows in its mean; reference-only while the first entries are measured | -| Workload | `json-comp`, `json-tls`, `8gbit`, `static-tls` | JSON serialization, gzip/brotli compression, HTTP/1.1 over TLS, a 10 KB body echoed over TLS at a pinned 50K req/s so ingest and egress load together - scored on the same 0-100 formula as the latency profiles, since its rate is pinned too, 20-file static asset serving over TLS | -| Database | `async-db` * | Async Postgres sequential scan. Reference-only: the database and its driver dominate it far more than the framework does, so it is measured and shown but no longer decides the ranking | -| Templates | `fortunes` * | DB query + HTML template render (TechEmpower-style Fortunes). Reference-only: measures template-engine throughput, and is not part of the composite score | -| H/2 | `baseline-h2`, `static-h2`, `baseline-h2c`, `json-h2c` | Baseline + static over TLS with h2 stream multiplexing; baseline + JSON over cleartext h2 (prior-knowledge, port 8082) | -| H/3 | `baseline-h3`, `static-h3` | Baseline and static over QUIC with TLS 1.3 | -| gRPC | `unary-grpc`, `unary-grpc-tls` | Unary gRPC over plaintext HTTP/2 and TLS | -| Gateway | `gateway-64`, `gateway-h3` | Reverse proxy + server stack over HTTP/2 and HTTP/3 with mixed workload | -| Production Stack | `production-stack` | Four-service architecture: edge + Redis + JWT auth sidecar + server, 10K-item cache-aside, concurrent reads + writes | -| WebSocket | `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` | Echo throughput across connection counts; 16x batched echo; echo with each connection closed after 10 messages (upgrade-handshake cost) | - -## Run Locally - -```bash -git clone https://github.com/MDA2AV/HttpArena.git -cd HttpArena - -./scripts/validate.sh # correctness check -./scripts/benchmark.sh # all profiles -./scripts/benchmark.sh baseline # specific profile -./scripts/benchmark.sh --save # save results -``` - -## Contributing - -- [Add a new framework](https://www.http-arena.com/#doc=add-framework) -- Improve an existing implementation by opening a PR that modifies files under `frameworks//` -- [Open an issue](https://github.com/MDA2AV/HttpArena/issues) -- Comment on any open issue or PR - -### Framework Maintainers - -Add your GitHub username to the `maintainers` array in your framework's `meta.json` to get notified when someone opens a PR that touches your framework: - -```json -"maintainers": ["your-github-username"] -``` - -## Add the badge - -Benchmarked here? Put your rank in your own README. It re-renders itself every -time the board is republished, so you paste it once and never touch it again: - -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=0) - -```md -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=0) -``` - -Swap `actix` for your entry and `h1` for the family you want: - -| Family | `` | Composite it reports | -|---|---|---| -| HTTP/1.1 | `h1` | Connection, Workload, Database and Multi-endpoint profiles | -| HTTP/2 | `h2` | the HTTP/2 profiles | -| HTTP/3 | `h3` | the HTTP/3 profiles | -| Gateway | `gw` | reverse-proxy and production-stack profiles | -| gRPC | `grpc` | unary and streaming, plaintext and TLS | -| WebSocket | `ws` | the echo profiles | - -`` is your entry's name on the board, lowercased, with anything -outside `A-Z a-z 0-9 . _ -` turned into `-`, the same name as your file in -[`site/data/results/`](site/data/results). The full list of what is published, -with every rank in it, is at -[`/badge/index.json`](https://www.http-arena.com/badge/index.json). - -### Ranked among your own language - -Optional. Append `-` to the family for a badge reading -**`#1 of 6 (Rust)`** gives the same scores and the same order, counting only -entries in your language: - -```md -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1-rust.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=0&lang=Rust) -``` - -The language is lowercased, with `#` spelled `sharp` and `++` spelled `pp`, so -`C#` is `csharp`, `C++` is `cpp`, `TS` is `ts`. It links to the board filtered to -that language, so the field is the one you can count there. Your line is in -`index.json` under `scopes..byLanguage.markdown`. - -### Counting tuned entries in - -Also optional. By default the field is **standard configurations only**, so the -number compares like for like. Append `-with-tuned` to count tuned entries too: - -```md -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1-with-tuned.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=1) -``` - -It combines with the language suffix, as in `h1-rust-with-tuned.json`. Every -variant is in `index.json` under `scopes.`, keyed `default`, -`withTuned`, `byLanguage` and `byLanguageWithTuned`. - -If your own entry is tuned, its default URL already counts tuned entries, since -it cannot be ranked in a field it is excluded from, so `h1.json` and -`h1-with-tuned.json` give it the same number. - -The two halves say different things. The right half is how you placed: gold for -first, then shading down by how far into the field you are. The left half is -which tier you competed in, in the same colours the board uses for the little -square next to every framework name: - -| | Tier | Ranked against | -|---|---|---| -| 🟩 | `flagship` | flagship + emerging, as one field | -| 🟦 | `emerging` | flagship + emerging, as one field | -| 🟧 | `experimental` | other experimental entries | -| 🟥 | `engine` | other engines, on the profiles engines are scored on | -| 🟦 | `infrastructure` | other proxies / static-file servers, on the profiles infrastructure is scored on | - -A few things worth knowing before you paste it: - -- **The rank is against your own tier.** `flagship` and `emerging` rank together - as one field, because that is the board's default view. `engine` entries rank - among engines, over the profile subset engines are scored on. That way a - framework's ceiling is never set by an engine's result, or the reverse. -- **It only exists where you scored.** No result in a family means no badge for - it, and a family with a single entry in your tier publishes nothing, because - a rank is worth showing once there was somebody to beat. -- **The field is every row the board lists.** `of 31` is what you get counting - engine rows on the linked page, including any entry sitting at 0 because - nothing it ran scores for its tier. Those occupy a place in the field but get - no badge of their own. -- **It follows the default board.** Same scoring as the page it links to, with - the memory-efficiency and rescale toggles off. Follow the link and you land on - the field the rank was measured against, meaning the whole league rather than - your row on its own, so `#6 of 83` arrives with the other 82 around it. -- **It updates on deploy, then when the caches let it.** shields holds a rank - for 5 minutes, and GitHub proxies README images through Camo, which holds - one for a few hours. Nothing to re-paste either way. - -Shields' usual styling works. Append `&style=flat-square`, `&logo=rust`, and so -on to the `img.shields.io` URL. +# HttpArena + +[![Discord](https://discordapp.com/api/guilds/1177529388229734410/widget.png?style=shield)](https://discord.com/invite/H84B5ZqDXR) + +## Hardware Upgrade +Hi, thank you for visiting or contributing to our project, we are always looking to improve this benchmark platform, if you wish to help us by sponsoring or donating, all the money is invested into infrastructure costs, we are currently aiming for hardware upgrades that would substantially improve our benchmarks. + +HTTP framework benchmark platform. + +30 test profiles. 64-core dedicated hardware. Same conditions for every framework. + +[View Leaderboard](https://http-arena.com/) | [Documentation](https://http-arena.com/#doc=) | [Add a Framework](https://http-arena.com/#doc=add-framework) + +--- + +## PR Commands + +| Command | Description | +|---------|-------------| +| `/benchmark -f ` | Run every test the framework subscribes to | +| `/benchmark -f -t ` | Run one test only | +| `/benchmark -f --save` | Run and save results (updates the leaderboard on merge) | +| `/benchmark -f -t --save` | Run one test and save results | +| `/benchmark -f --compare ` | Measure the deltas against another framework instead of this one | +| `/benchmark-multiple -f ,,...` | Benchmark several frameworks in one run; takes `-t` and `--save` too, and saved results land in a single commit | +| `/benchmark-multiple --save` | No `-f` needed: benchmark and save every framework the PR touches | +| `/benchmark-test -t ` | Benchmark **all** enabled frameworks subscribed to `` and save the results | + +For `/benchmark`, always specify `-f `; the flags combine in any order. Results come back as a comment with a per-profile table of RPS, p99, CPU and memory, one table per framework on multi runs. A new benchmark comment while a run is in flight queues behind it (one deep) instead of cancelling it. For multi-framework PRs (dependency bumps, same-language refactors) prefer `/benchmark-multiple`, which runs everything in a single job and commits all saved results together, so no run overwrites another. `--compare` works on single-framework runs only. + +**What the deltas are measured against.** By default, this framework's own results published on `main` - answering *"did this change help?"*. When you are tuning a variant or a successor entry, `--compare` re-bases them on another entry instead: + +``` +/benchmark -f genhttp --compare genhttp-kestrel +``` + +The reply states which baseline it used, and profiles the other framework does not run show `n/a` rather than a delta. + +--- + +## Test Profiles + +| Category | Profiles | Description | +|----------|----------|-------------| +| Connection | `baseline`, `pipelined` *, `limited-conn` | Mixed GET/POST with query parsing (512/4K conns), 16× batched pipelining (reference-only, shown faded, excluded from the composite score), short-lived connections that close after 10 requests | +| Concurrency | `async` | `GET /delay/{ms}` with the wait named in the route, 32K held connections. Isolates what a framework does while a request is pending, with no database or I/O in the way. Scored for frameworks and engines, not for infrastructure, since the spec settled at a flat 10 ms over 32K connections | +| Efficiency | `latency-1m`, `latency-10k`, `latency-500k-8cpu` * | One million req/s offered at a fixed rate with zrk; the metric is CPU spent per request, read exactly from the container's cgroup rather than sampled. Everyone who finishes serves the same million, so what separates them is the bill. Scored on its own 0-100 formula rather than on rps, since the rate is pinned. `latency-10k` is the identical setup - same cores, connections and endpoint - at a near-idle 10K req/s, so what it reports is the standing cost of being a running server rather than of serving load, and it is scored the same way. Both score CPU at 0.50, p99 at 0.25 and mean latency at 0.25, the mean being the queueing signal: ten times the best mean is worth nothing. `latency-500k-8cpu` pins 500K req/s with the server on four cores plus their SMT siblings, so the load sits near saturation and the queue a server builds shows in its mean; reference-only while the first entries are measured | +| Workload | `json-comp`, `json-tls`, `8gbit`, `static-tls` | JSON serialization, gzip/brotli compression, HTTP/1.1 over TLS, a 10 KB body echoed over TLS at a pinned 50K req/s so ingest and egress load together - scored on the same 0-100 formula as the latency profiles, since its rate is pinned too, 20-file static asset serving over TLS | +| Database | `async-db` * | Async Postgres sequential scan. Reference-only: the database and its driver dominate it far more than the framework does, so it is measured and shown but no longer decides the ranking | +| Templates | `fortunes` * | DB query + HTML template render (TechEmpower-style Fortunes). Reference-only: measures template-engine throughput, and is not part of the composite score | +| H/2 | `baseline-h2`, `static-h2`, `baseline-h2c`, `json-h2c` | Baseline + static over TLS with h2 stream multiplexing; baseline + JSON over cleartext h2 (prior-knowledge, port 8082) | +| H/3 | `baseline-h3`, `static-h3` | Baseline and static over QUIC with TLS 1.3 | +| gRPC | `unary-grpc`, `unary-grpc-tls` | Unary gRPC over plaintext HTTP/2 and TLS | +| Gateway | `gateway-64`, `gateway-h3` | Reverse proxy + server stack over HTTP/2 and HTTP/3 with mixed workload | +| Production Stack | `production-stack` | Four-service architecture: edge + Redis + JWT auth sidecar + server, 10K-item cache-aside, concurrent reads + writes | +| WebSocket | `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` | Echo throughput across connection counts; 16x batched echo; echo with each connection closed after 10 messages (upgrade-handshake cost) | + +## Run Locally + +```bash +git clone https://github.com/MDA2AV/HttpArena.git +cd HttpArena + +./scripts/validate.sh # correctness check +./scripts/benchmark.sh # all profiles +./scripts/benchmark.sh baseline # specific profile +./scripts/benchmark.sh --save # save results +``` + +## Contributing + +- [Add a new framework](https://http-arena.com/#doc=add-framework) +- Improve an existing implementation by opening a PR that modifies files under `frameworks//` +- [Open an issue](https://github.com/MDA2AV/HttpArena/issues) +- Comment on any open issue or PR + +### Framework Maintainers + +Add your GitHub username to the `maintainers` array in your framework's `meta.json` to get notified when someone opens a PR that touches your framework: + +```json +"maintainers": ["your-github-username"] +``` + +## Add the badge + +Benchmarked here? Put your rank in your own README. It re-renders itself every +time the board is republished, so you paste it once and never touch it again: + +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1.json)](https://http-arena.com/#type=emerging,flagship&tuned=0) + +```md +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1.json)](https://http-arena.com/#type=emerging,flagship&tuned=0) +``` + +Swap `actix` for your entry and `h1` for the family you want: + +| Family | `` | Composite it reports | +|---|---|---| +| HTTP/1.1 | `h1` | Connection, Workload, Database and Multi-endpoint profiles | +| HTTP/2 | `h2` | the HTTP/2 profiles | +| HTTP/3 | `h3` | the HTTP/3 profiles | +| Gateway | `gw` | reverse-proxy and production-stack profiles | +| gRPC | `grpc` | unary and streaming, plaintext and TLS | +| WebSocket | `ws` | the echo profiles | + +`` is your entry's name on the board, lowercased, with anything +outside `A-Z a-z 0-9 . _ -` turned into `-`, the same name as your file in +[`site/data/results/`](site/data/results). The full list of what is published, +with every rank in it, is at +[`/badge/index.json`](https://http-arena.com/badge/index.json). + +### Ranked among your own language + +Optional. Append `-` to the family for a badge reading +**`#1 of 6 (Rust)`** gives the same scores and the same order, counting only +entries in your language: + +```md +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1-rust.json)](https://http-arena.com/#type=emerging,flagship&tuned=0&lang=Rust) +``` + +The language is lowercased, with `#` spelled `sharp` and `++` spelled `pp`, so +`C#` is `csharp`, `C++` is `cpp`, `TS` is `ts`. It links to the board filtered to +that language, so the field is the one you can count there. Your line is in +`index.json` under `scopes..byLanguage.markdown`. + +### Counting tuned entries in + +Also optional. By default the field is **standard configurations only**, so the +number compares like for like. Append `-with-tuned` to count tuned entries too: + +```md +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1-with-tuned.json)](https://http-arena.com/#type=emerging,flagship&tuned=1) +``` + +It combines with the language suffix, as in `h1-rust-with-tuned.json`. Every +variant is in `index.json` under `scopes.`, keyed `default`, +`withTuned`, `byLanguage` and `byLanguageWithTuned`. + +If your own entry is tuned, its default URL already counts tuned entries, since +it cannot be ranked in a field it is excluded from, so `h1.json` and +`h1-with-tuned.json` give it the same number. + +The two halves say different things. The right half is how you placed: gold for +first, then shading down by how far into the field you are. The left half is +which tier you competed in, in the same colours the board uses for the little +square next to every framework name: + +| | Tier | Ranked against | +|---|---|---| +| 🟩 | `flagship` | flagship + emerging, as one field | +| 🟦 | `emerging` | flagship + emerging, as one field | +| 🟧 | `experimental` | other experimental entries | +| 🟥 | `engine` | other engines, on the profiles engines are scored on | +| 🟦 | `infrastructure` | other proxies / static-file servers, on the profiles infrastructure is scored on | + +A few things worth knowing before you paste it: + +- **The rank is against your own tier.** `flagship` and `emerging` rank together + as one field, because that is the board's default view. `engine` entries rank + among engines, over the profile subset engines are scored on. That way a + framework's ceiling is never set by an engine's result, or the reverse. +- **It only exists where you scored.** No result in a family means no badge for + it, and a family with a single entry in your tier publishes nothing, because + a rank is worth showing once there was somebody to beat. +- **The field is every row the board lists.** `of 31` is what you get counting + engine rows on the linked page, including any entry sitting at 0 because + nothing it ran scores for its tier. Those occupy a place in the field but get + no badge of their own. +- **It follows the default board.** Same scoring as the page it links to, with + the memory-efficiency and rescale toggles off. Follow the link and you land on + the field the rank was measured against, meaning the whole league rather than + your row on its own, so `#6 of 83` arrives with the other 82 around it. +- **It updates on deploy, then when the caches let it.** shields holds a rank + for 5 minutes, and GitHub proxies README images through Camo, which holds + one for a few hours. Nothing to re-paste either way. + +Shields' usual styling works. Append `&style=flat-square`, `&logo=rust`, and so +on to the `img.shields.io` URL. diff --git a/scripts/assemble_site.sh b/scripts/assemble_site.sh index a3938891b..cdcc26f43 100755 --- a/scripts/assemble_site.sh +++ b/scripts/assemble_site.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Lays out the site that gets served, from what scripts/gen_leaderboard_data.py generated: -# site/public/ for GitHub Pages (deploy.yml), and the same tree without the run logs for -# the genhttp.dev lambda (publish_genhttp.py), which sends the site's log links on to the -# logs in this repository instead. +# without the run logs for the genhttp.dev lambda (publish_genhttp.py), which sends the site's +# log links on to the logs in this repository instead, and with them in site/public/ to look +# at the whole site locally (python3 -m http.server -d site/public). # # scripts/assemble_site.sh [--no-logs] [out-dir] out-dir defaults to site/public set -euo pipefail diff --git a/scripts/gen_leaderboard_data.py b/scripts/gen_leaderboard_data.py index 67e3bfbdf..a302547d9 100644 --- a/scripts/gen_leaderboard_data.py +++ b/scripts/gen_leaderboard_data.py @@ -653,7 +653,7 @@ def build_rounds(): # without this every doc collapses into the single "/" URL. Content is the exact # HTML build_docs() already produces; only the link scheme differs (#doc=x -> /docs/x/). -SITE = "https://www.http-arena.com" +SITE = "https://http-arena.com" GEN = ROOT / "site" / "generated" DOCS_OUT = GEN / "docs" @@ -3353,7 +3353,7 @@ def build_compare_pages(profiles, results, lang_rows, members_by_lang, compare_l _og_card(dest / "og.png", "Head to head · " + SCOPE_NAME[DEFAULT_SCOPE], lang + " web frameworks, compared", [(r[0], "%.0f" % r[3]) for r in rows[:5]], - "%d entries · %s · www.http-arena.com" % (len(members), round_name)) + "%d entries · %s · http-arena.com" % (len(members), round_name)) cards += 1 written.append((lang, len(members))) @@ -3374,7 +3374,7 @@ def build_compare_pages(profiles, results, lang_rows, members_by_lang, compare_l _og_card(dest / "og.png", "Head to head · every language", "Compare any two entries, whatever they are written in", [(r[0], "%.0f" % r[3]) for r in all_rows[:5]], - "%d entries · %s · www.http-arena.com" % (len(all_members), round_name)) + "%d entries · %s · http-arena.com" % (len(all_members), round_name)) cards += 1 written.insert(0, (None, len(all_members))) @@ -3383,7 +3383,7 @@ def build_compare_pages(profiles, results, lang_rows, members_by_lang, compare_l "Compare two web frameworks, profile by profile", [(lang or "Every language", "%d entries" % n) for lang, n in sorted(written, key=lambda x: -x[1])[:5]], - "%d languages · %s · www.http-arena.com" % (len(written), round_name)) + "%d languages · %s · http-arena.com" % (len(written), round_name)) cards += 1 (COMPARE_OUT / "index.html").write_text( _compare_index_page(written, updated, "/compare/og.png" if with_og else ""), @@ -3483,7 +3483,7 @@ def build_fw_pages(profiles, results, meta, fw_lang, badge_index, achievements, [(SCOPE_NAME[s], "#%d of %d" % (rank, field)) for s, rank, field, _, _link in ranks], " · ".join(x for x in [lang, TYPE_LABEL.get(kind, kind), - round_name, "www.http-arena.com"] if x), + round_name, "http-arena.com"] if x), blurb=m.get("desc", "")) cards += 1 entries.append((fw, lang, kind)) @@ -3518,14 +3518,14 @@ def build_og_images(content, trails, rows, fw_lang, round_name): "Composite ranking · " + SCOPE_NAME[DEFAULT_SCOPE], "Which web framework is actually fastest?", top, - f"{len(rows)} entries · {round_name} · www.http-arena.com") + f"{len(rows)} entries · {round_name} · http-arena.com") written = 1 for did, d in content.items(): crumb = " › ".join(t for t, _ in trails.get(did, [])[:-1]) or "Knowledge Base" _og_card(DOCS_OUT / did / "og.png" if did else DOCS_OUT / "og.png", crumb, d["t"] or "Knowledge Base", [], - "HttpArena Knowledge Base · www.http-arena.com", + "HttpArena Knowledge Base · http-arena.com", blurb=d.get("d") or _meta_desc(d["html"])) written += 1 return "/og.png", written diff --git a/scripts/validate.sh b/scripts/validate.sh index d08cf8616..64c913ccf 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -461,7 +461,7 @@ fi # ───── Helpers ───── -DOCS_BASE="https://www.http-arena.com/#doc=test-profiles" +DOCS_BASE="https://http-arena.com/#doc=test-profiles" # Every parameter the profiles below send is drawn fresh per run. The suite used # to ask for a fixed set - /json/12?m=3, /json/50?m=1, min=10&max=50&limit=50 - diff --git a/site/content/docs/add-framework/badges.md b/site/content/docs/add-framework/badges.md index 845e5bdeb..094477b46 100644 --- a/site/content/docs/add-framework/badges.md +++ b/site/content/docs/add-framework/badges.md @@ -14,12 +14,12 @@ itself each time the leaderboard is republished. You paste it once. ## Paste it ```md -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=0) +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1.json)](https://http-arena.com/#type=emerging,flagship&tuned=0) ``` Replace `actix` with your entry and `h1` with the family you want. The exact line for your framework, already assembled and deep-linked to the right view of -the board, is in [`/badge/index.json`](https://www.http-arena.com/badge/index.json) +the board, is in [`/badge/index.json`](https://http-arena.com/badge/index.json) under `scopes..default.markdown`. ## Families @@ -44,13 +44,13 @@ Optional, and a second badge rather than a replacement. Append `-` to the family and the badge reads **`#1 of 6 (Rust)`**: ```md -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1-rust.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=0&lang=Rust) +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1-rust.json)](https://http-arena.com/#type=emerging,flagship&tuned=0&lang=Rust) ``` The language segment is the board's own language name, lowercased, with `#` spelled out as `sharp` and `++` as `pp`: `C#` → `csharp`, `C++` → `cpp`, `TS` → `ts`, `JS` → `js`. Your ready-made line is in -[`/badge/index.json`](https://www.http-arena.com/badge/index.json) under +[`/badge/index.json`](https://http-arena.com/badge/index.json) under `scopes..byLanguage.markdown`. ## Counting tuned entries in @@ -61,7 +61,7 @@ someone else's hand-tuned build. Append `-with-tuned` to count tuned entries as well: ```md -[![HTTP Arena](https://img.shields.io/endpoint?url=https://www.http-arena.com/badge/actix/h1-with-tuned.json)](https://www.http-arena.com/#type=emerging,flagship&tuned=1) +[![HTTP Arena](https://img.shields.io/endpoint?url=https://http-arena.com/badge/actix/h1-with-tuned.json)](https://http-arena.com/#type=emerging,flagship&tuned=1) ``` The suffixes combine, so the full set for one family is: @@ -73,7 +73,7 @@ The suffixes combine, so the full set for one family is: | `h1-rust.json` | standard, one language | | `h1-rust-with-tuned.json` | standard + tuned, one language | -Each is in [`/badge/index.json`](https://www.http-arena.com/badge/index.json) +Each is in [`/badge/index.json`](https://http-arena.com/badge/index.json) under `scopes.`, keyed `default`, `withTuned`, `byLanguage` and `byLanguageWithTuned`. @@ -102,7 +102,7 @@ type swatch beside every framework name: | 🟧 | `experimental` | other experimental entries | | 🟥 | `engine` | other engines, on the profiles engines are scored on | -The tier is also in [`/badge/index.json`](https://www.http-arena.com/badge/index.json) +The tier is also in [`/badge/index.json`](https://http-arena.com/badge/index.json) as `type`, next to each entry's ranks. ## What the number means diff --git a/site/content/docs/load-generators/ws/_index.md b/site/content/docs/load-generators/ws/_index.md index 3961795e8..c90d64bce 100644 --- a/site/content/docs/load-generators/ws/_index.md +++ b/site/content/docs/load-generators/ws/_index.md @@ -51,7 +51,7 @@ gcannon reports WebSocket results with the same layout as HTTP requests, except WS frames: 2400000 ``` -The parser (`gcannon_parse ws-echo`) records `frames received` as the `status_2xx` equivalent and divides by the measured duration to produce the headline RPS number shown on the [WebSocket leaderboard](https://www.http-arena.com/#scope=ws). One echo round-trip counts as one unit - the frames-received count from the client side, not frames-sent, because the metric is "how many echoes the framework completed," not "how many messages the benchmarker pushed into the socket." +The parser (`gcannon_parse ws-echo`) records `frames received` as the `status_2xx` equivalent and divides by the measured duration to produce the headline RPS number shown on the [WebSocket leaderboard](https://http-arena.com/#scope=ws). One echo round-trip counts as one unit - the frames-received count from the client side, not frames-sent, because the metric is "how many echoes the framework completed," not "how many messages the benchmarker pushed into the socket." ## Why not a dedicated WebSocket tool diff --git a/site/content/docs/test-profiles/_index.md b/site/content/docs/test-profiles/_index.md index 661e71844..374cc9c42 100644 --- a/site/content/docs/test-profiles/_index.md +++ b/site/content/docs/test-profiles/_index.md @@ -20,8 +20,8 @@ The endpoints are split across two files: | Spec | Profiles covered | | |------|-----------------|---| -| [openapi.json](/openapi.json) | All HTTP/1.1, HTTP/2, HTTP/3, gRPC, and WebSocket profiles | [Browse](https://redocly.github.io/redoc/?url=https://www.http-arena.com/openapi.json) | -| [openapi-gateway.json](/openapi-gateway.json) | Gateway profiles (production-stack) | [Browse](https://redocly.github.io/redoc/?url=https://www.http-arena.com/openapi-gateway.json) | +| [openapi.json](/openapi.json) | All HTTP/1.1, HTTP/2, HTTP/3, gRPC, and WebSocket profiles | [Browse](https://redocly.github.io/redoc/?url=https://http-arena.com/openapi.json) | +| [openapi-gateway.json](/openapi-gateway.json) | Gateway profiles (production-stack) | [Browse](https://redocly.github.io/redoc/?url=https://http-arena.com/openapi-gateway.json) | ## Benchmark parameters diff --git a/site/content/docs/test-profiles/h1/json-compressed/implementation.md b/site/content/docs/test-profiles/h1/json-compressed/implementation.md index b5a80dd4a..fe69afaa5 100644 --- a/site/content/docs/test-profiles/h1/json-compressed/implementation.md +++ b/site/content/docs/test-profiles/h1/json-compressed/implementation.md @@ -99,7 +99,7 @@ On the JSON Compressed tab of the main leaderboard, the Score column is computed ### Composite score column -On the [composite leaderboard](https://www.http-arena.com/), the JSON Compressed column follows a different aggregation path: +On the [composite leaderboard](https://http-arena.com/), the JSON Compressed column follows a different aggregation path: 1. Average `rps` and `bandwidth` across the 3 conn counts (arithmetic mean, equal weight). 2. Compute `myBpr = avgBw / avgRps` from those averages. diff --git a/site/lambda/Site.cs b/site/lambda/Site.cs index 1f5730bad..81ef000a4 100644 --- a/site/lambda/Site.cs +++ b/site/lambda/Site.cs @@ -1,5 +1,5 @@ // The site: what scripts/gen_leaderboard_data.py generated and scripts/assemble_site.sh laid out -// at a commit of main - the tree GitHub Pages serves, without the logs. It is data, like the +// at a commit of main - the tree GitHub Pages served, without the logs. It is data, like the // results beside it: the HttpArena repository sends it through api/ingest/site/ when its publish // workflow is run, no version is saved for it, and a deploy leaves it as it is. // @@ -11,8 +11,9 @@ // changes finds its file. // // A folder answers with its index.html (docs/scoring/ is docs/scoring/index.html), a folder -// asked for without its slash is sent on to it as GitHub Pages does, and a path that is nothing -// answers 404 with the site's own 404.html. +// asked for without its slash is sent on to it as GitHub Pages did, and a path that is nothing +// answers 404 with the site's own 404.html. Any origin may read it, as on Pages: the docs open +// openapi.json in Redoc, on an origin of its own. using System.Security.Cryptography; @@ -291,7 +292,7 @@ private static async Task ReadAsync(Stream body) } } -/// Answers a path of the site the way GitHub Pages does. +/// Answers a path of the site the way GitHub Pages did. public sealed class SiteHandler(SiteStore site) : IHandler { private static readonly ContentType Html = new("text/html; charset=utf-8"); @@ -304,7 +305,7 @@ public async ValueTask HandleAsync(IRequest request) { if (!request.HasType(RequestMethod.Get, RequestMethod.Head)) { - return request.Respond().Status(ResponseStatus.MethodNotAllowed).Header("Allow", "GET, HEAD").Build(); + return Respond(request).Status(ResponseStatus.MethodNotAllowed).Header("Allow", "GET, HEAD").Build(); } var target = request.Header.Target; @@ -315,12 +316,12 @@ public async ValueTask HandleAsync(IRequest request) if (await site.OpenAsync(name) is (var resource, var type)) { - return request.Respond().Content(resource, type).Build(); + return Respond(request).Content(resource, type).Build(); } if (!site.Published) { - return request.Respond() + return Respond(request) .Status(ResponseStatus.ServiceUnavailable) .Content("The site was not published here yet: the HttpArena repository sends it with scripts/publish_genhttp.py.", Text) .Build(); @@ -329,14 +330,17 @@ public async ValueTask HandleAsync(IRequest request) if (path.Length > 0 && !target.HasTrailingSlash && site.Has(path + "/index.html")) { // relative, as every address here is: "scoring/" from docs/scoring - return request.Respond() + return Respond(request) .Status(ResponseStatus.MovedPermanently) .Header("Location", path[(path.LastIndexOf('/') + 1)..] + "/") .Build(); } - var notFound = request.Respond().Status(ResponseStatus.NotFound); + var notFound = Respond(request).Status(ResponseStatus.NotFound); return (await site.OpenAsync("404.html") is (var page, _) ? notFound.Content(page, Html) : notFound.Content("Not found.", Text)).Build(); } + + /// Every answer of the site, readable from any origin. + private static IResponseBuilder Respond(IRequest request) => request.Respond().Header("Access-Control-Allow-Origin", "*"); } diff --git a/site/lambda/docs/decisions.md b/site/lambda/docs/decisions.md index 097f17e4a..453d4b326 100644 --- a/site/lambda/docs/decisions.md +++ b/site/lambda/docs/decisions.md @@ -14,7 +14,7 @@ the editor, or pushed to this lambda's own git repository, is replaced by the ne ## The site is data, not a version The site is what `scripts/gen_leaderboard_data.py` generated and `scripts/assemble_site.sh ---no-logs` laid out: the same tree GitHub Pages serves, without the logs. Nearly every page of it +--no-logs` laid out: the same tree GitHub Pages served, without the logs. Nearly every page of it is generated from the results, so it changes whenever they do, and a version for every such change is what the owner did not want: a version is the program. So the site is sent like the results, through `api/ingest/site/`, and kept in the workspace; a deploy leaves it alone, and a @@ -27,10 +27,12 @@ of a ~1,770-file site are sent the first time and only what changed after that. neither the new paths nor the ones before them is deleted then, so a page in the middle of being sent while the site changes still finds its file. -`Site.cs` serves it the way GitHub Pages does: a folder answers with its `index.html`, a folder +`Site.cs` serves it the way GitHub Pages did: a folder answers with its `index.html`, a folder asked for without its slash is sent on to it with a 301, and a path that is nothing answers 404 with the site's own `404.html` - never the `index.html` of a folder above it. Types come from the -names, and ETags from the files, so a browser that has a page is told it is unchanged. +names, and ETags from the files, so a browser that has a page is told it is unchanged. Every +answer carries `Access-Control-Allow-Origin: *`, as on Pages: the docs' "Browse" links open +`openapi.json` in Redoc, which reads it from an origin of its own. The pages link absolutely (`/docs/...`, `/data.js`): they are generated for the root of a domain. That works at the lambda's own address and on a domain of its own, and not in a feature's preview @@ -54,7 +56,7 @@ are what everything that reads results already understands. They are built per r The container log of every run is kept where `benchmark.sh` writes it, in the repository under `site/static/logs/`, as the owner wants, and this lambda keeps no copy. The board links a log at -`logs/{profile}/{conns}/{framework dir}.log`, as it does on GitHub Pages; here that answers 302 to +`logs/{profile}/{conns}/{framework dir}.log`, as it did on GitHub Pages; here that answers 302 to the same file on `raw.githubusercontent.com`, as it is on main. The redirect is temporary so that browsers don't remember it, and the links work wherever the board is served from. The lambda kept gzipped copies until `V3__Drop_logs.sql` dropped their table. diff --git a/site/lambda/tests/README.md b/site/lambda/tests/README.md index 81f400424..ad2444d81 100644 --- a/site/lambda/tests/README.md +++ b/site/lambda/tests/README.md @@ -5,7 +5,7 @@ - The site: `/` is the board, a folder answers with its `index.html` (`docs/scoring/`), a folder without its slash is sent on to it (`docs` answers 301 to `docs/`), and a path that is nothing answers 404 with the site's `404.html`. Every page answers exactly what `assemble_site.sh` laid - out at the commit `api/site` names. + out at the commit `api/site` names, with `Access-Control-Allow-Origin: *`. - Publishing the same site twice sends nothing the second time; content no longer named by the site or the one before it is deleted. - `api/frameworks`, `api/results/{slug}` and `api/host` answer exactly what the repository's diff --git a/site/lambda/tests/smoke.mjs b/site/lambda/tests/smoke.mjs index 194f891ae..aff233600 100644 --- a/site/lambda/tests/smoke.mjs +++ b/site/lambda/tests/smoke.mjs @@ -24,6 +24,7 @@ await check("the site is published", "api/site", (r, body) => r.status !== 200 ? await check("a folder answers with its index", "docs/", status(200)); await check("a folder without its slash is sent on", "docs", (r) => r.status !== 301 ? `answered ${r.status}` : r.headers.get("location") === "docs/" ? null : `sent to ${r.headers.get("location")}`); await check("a path that is nothing is 404", "no/such/page/", status(404)); +await check("any origin may read the site", "openapi.json", (r) => r.status !== 200 ? `answered ${r.status}` : r.headers.get("access-control-allow-origin") === "*" ? null : "no Access-Control-Allow-Origin: *"); await check("badges for shields.io", "badge/index.json", (r, body) => r.status !== 200 ? `answered ${r.status}` : JSON.parse(body) && null); await check("the entries", "api/frameworks", (r, body) => r.status !== 200 ? `answered ${r.status}` : typeof JSON.parse(body) === "object" ? null : "not an object"); await check("the publishes", "api/publishes", (r, body) => r.status !== 200 ? `answered ${r.status}` : Array.isArray(JSON.parse(body)) ? null : "not a list"); diff --git a/site/leaderboard/index.html b/site/leaderboard/index.html index 572b592af..d449c4df8 100644 --- a/site/leaderboard/index.html +++ b/site/leaderboard/index.html @@ -3,7 +3,7 @@ - + HTTP Web Server Benchmarks – HttpArena diff --git a/site/root/404.html b/site/root/404.html index 46a620373..891972d9a 100644 --- a/site/root/404.html +++ b/site/root/404.html @@ -6,7 +6,7 @@