From 26f7dede70f69cc66516c6bf7b4f6886f6adf0f9 Mon Sep 17 00:00:00 2001 From: Jordan Gonzalez <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:48:48 -0400 Subject: [PATCH 01/30] [SVLS-9974] ci: replace cargo-audit with cargo-deny --- .github/workflows/rs_ci.yml | 27 +++---- .trivyignore.yaml | 4 +- bottlecap/.cargo/audit.toml | 22 ------ bottlecap/Cargo.toml | 6 +- bottlecap/deny.toml | 140 ++++++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 39 deletions(-) delete mode 100644 bottlecap/.cargo/audit.toml create mode 100644 bottlecap/deny.toml diff --git a/.github/workflows/rs_ci.yml b/.github/workflows/rs_ci.yml index ffd15acf8..cd48a5a54 100644 --- a/.github/workflows/rs_ci.yml +++ b/.github/workflows/rs_ci.yml @@ -140,7 +140,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 with: cache: false - - uses: taiki-e/install-action@328a871ad8f62ecac78390391f463ccabc974b72 # v2.69.9 + - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 with: tool: nextest@0.9 - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 @@ -176,23 +176,26 @@ jobs: - run: dd-rust-license-tool check working-directory: bottlecap - cargo-audit: + # cargo-deny: advisories, duplicate versions, git sources and licenses, with + # one config and one ignore list in bottlecap/deny.toml. It replaces the + # cargo-audit job (same RustSec database) and keeps the `Audit` check name, + # which branch protection on main requires. The Trivy lockfile scan below + # stays as the second advisory source (GHSA ids, .trivyignore.yaml). + cargo-deny: name: Audit runs-on: ubuntu-22.04 permissions: contents: read - checks: write steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Scan Rust dependencies with cargo-audit - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 with: - token: ${{ secrets.GITHUB_TOKEN }} - working-directory: bottlecap - # Mirrors bottlecap/.cargo/audit.toml. This action only reads ./.cargo/audit.toml - # relative to its cwd (repo root), so the crate-local ignore list is never loaded in CI. - ignore: RUSTSEC-2026-0285,RUSTSEC-2026-0346,RUSTSEC-2026-0348,RUSTSEC-2026-0349,RUSTSEC-2026-0381,RUSTSEC-2026-0382 + cache: false + - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 + with: + tool: cargo-deny@0.20.2 + - working-directory: bottlecap + run: cargo deny --locked check cargo-lockfile-scan: name: Lockfile Scan diff --git a/.trivyignore.yaml b/.trivyignore.yaml index 0d1dde4a9..6caff7390 100644 --- a/.trivyignore.yaml +++ b/.trivyignore.yaml @@ -13,7 +13,7 @@ vulnerabilities: reports nothing, while `-e build,normal` reports the build-script edge. - id: GHSA-2mjx-qc3c-rqvc statement: | - Same advisory as RUSTSEC-2026-0285, ignored for cargo-audit in - `bottlecap/.cargo/audit.toml`. The fix is in rustls >= 0.23.45, which + Same advisory as RUSTSEC-2026-0285, ignored for cargo-deny in + `bottlecap/deny.toml`. The fix is in rustls >= 0.23.45, which requires aws-lc-rs >= 1.18 and so AWS-LC-FIPS 4.x, pending FIPS 140-3 certification. Remove this entry together with that ignore. diff --git a/bottlecap/.cargo/audit.toml b/bottlecap/.cargo/audit.toml deleted file mode 100644 index 1e6371600..000000000 --- a/bottlecap/.cargo/audit.toml +++ /dev/null @@ -1,22 +0,0 @@ -# RUSTSEC-2026-0285 is fixed in rustls >= 0.23.45, and RUSTSEC-2026-0346, -# RUSTSEC-2026-0348, and RUSTSEC-2026-0349 in >= 0.23.46, but those versions -# require aws-lc-rs >= 1.18, which pulls in AWS-LC-FIPS 4.x, pending FIPS -# 140-3 certification. RUSTSEC-2026-0382 (rustls-webpki) is likewise only -# patched in 0.103.16, which requires aws-lc-rs ^1.18. Remove these ignores -# once a FIPS-certified aws-lc-rs is available. Trivy reports -# RUSTSEC-2026-0285 as GHSA-2mjx-qc3c-rqvc, ignored in .trivyignore.yaml. -# -# RUSTSEC-2026-0381 (webpki-roots) is a false positive: the locked 0.26.11 is -# a semver-trick shim whose only dependency is webpki-roots ^1 (resolved to -# 1.0.10, which removes the distrusted emSign root), and no 0.26.x release -# exists that updates the shim itself. It enters the lock via -# hyper-http-proxy's rustls-tls-webpki-roots feature. -[advisories] -ignore = [ - "RUSTSEC-2026-0285", - "RUSTSEC-2026-0346", - "RUSTSEC-2026-0348", - "RUSTSEC-2026-0349", - "RUSTSEC-2026-0381", - "RUSTSEC-2026-0382", -] diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 67f7bc860..26ec8582a 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -33,7 +33,7 @@ serde_bytes = { version = "0.11", default-features = false, features = ["std"] } rmp-serde = { version = "1.3.1", default-features = false } flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } thiserror = { version = "1.0", default-features = false } -# Transitive dependency (pulled in via cookie). Pinned to >=0.3.47 so cargo audit / CI passes (RUSTSEC-2026-0009). +# Transitive dependency (pulled in via cookie). Pinned to >=0.3.47 so cargo deny / CI passes (RUSTSEC-2026-0009). time = { version = "0.3.47", default-features = false } tokio = { version = "1.47", default-features = false, features = ["macros", "rt-multi-thread", "time"] } tokio-util = { version = "0.7", default-features = false } @@ -48,10 +48,10 @@ base64 = { version = "0.22", default-features = false } rustls = { version = ">=0.23.18, <0.23.45", default-features = false } # Constrain the transitive FIPS crypto backend: aws-lc-rs 1.18 switches to # AWS-LC-FIPS 4.x, pending FIPS 140-3 certification. Lift this bound only -# after 4.x certification is verified. See .cargo/audit.toml and +# after 4.x certification is verified. See deny.toml and # DataDog/serverless-components#173. aws-lc-rs = { version = ">=1, <1.18", default-features = false, optional = true } -# Transitive via rustls. Pinned to >=0.103.13 so cargo audit passes (RUSTSEC-2026-0098, RUSTSEC-2026-0099, RUSTSEC-2026-0104). +# Transitive via rustls. Pinned to >=0.103.13 so cargo deny passes (RUSTSEC-2026-0098, RUSTSEC-2026-0099, RUSTSEC-2026-0104). rustls-webpki = { version = "0.103.13", default-features = false } rustls-pemfile = { version = "2.0", default-features = false, features = ["std"] } rustls-pki-types = { version = "1.0", default-features = false } diff --git a/bottlecap/deny.toml b/bottlecap/deny.toml new file mode 100644 index 000000000..6f0ee7270 --- /dev/null +++ b/bottlecap/deny.toml @@ -0,0 +1,140 @@ +# cargo-deny configuration for bottlecap. Run from this directory: +# +# cargo deny check # all four checks +# cargo deny check advisories # RustSec only (replaces cargo-audit) +# +# CI runs this in `.github/workflows/rs_ci.yml` (job `cargo-deny`). The +# advisory ignore list lives only here; `.trivyignore.yaml` carries the same +# advisories under their GHSA ids for the Trivy lockfile scan. + +[graph] +# The default and fips features are compile-time exclusive, but they resolve +# together, so one graph covers the FIPS-only crates (aws-lc-rs, +# rustls-native-certs) as well as the test-mode binary. +all-features = true +# Only the targets the extension ships on. This drops windows-*, wasm and +# apple-only crates from the duplicate and license checks. +targets = [ + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", +] + +[advisories] +db-urls = ["https://github.com/rustsec/advisory-db"] +# A yank upstream shows as a warning and does not block an unrelated PR. +yanked = "warn" +# Unmaintained and unsound notices for transitive crates are not actionable +# here; one on a direct dependency fails the check until it is listed below. +# cargo-audit reported both classes as warnings only; set them to "none" for +# strict parity. +unmaintained = "workspace" +unsound = "workspace" +# RUSTSEC-2026-0285 is fixed in rustls >= 0.23.45, and RUSTSEC-2026-0346, +# RUSTSEC-2026-0348, and RUSTSEC-2026-0349 in >= 0.23.46, but those versions +# require aws-lc-rs >= 1.18, which pulls in AWS-LC-FIPS 4.x, pending FIPS +# 140-3 certification. RUSTSEC-2026-0382 (rustls-webpki) is likewise only +# patched in 0.103.16, which requires aws-lc-rs ^1.18. Remove these ignores +# once a FIPS-certified aws-lc-rs is available. Trivy reports +# RUSTSEC-2026-0285 as GHSA-2mjx-qc3c-rqvc, ignored in .trivyignore.yaml. +# +# The zstd advisories cover APIs this graph does not call: bottlecap, the +# serverless-components crates and libdd-trace-utils compress through +# `stream::write::Encoder` / the bulk API; the only decoding is +# `stream::read::Decoder` on the embedded AppSec ruleset and `decode_all` in +# a test helper, both excluded by RUSTSEC-2026-0403. Only libddwaf-sys's build +# script enables zstd's default (legacy) features; build-dependency features +# resolve separately, so the shipped binary has no legacy-frame support. +# zstd-safe 7.2.4 also has unsound notices RUSTSEC-2026-0395 to 0406 that the +# `workspace` scope does not report; the same zstd 0.14 / zstd-safe 8.2 bump +# fixes them (needs the same bump in serverless-components and libdatadog); +# drop these ignores with it. +ignore = [ + { id = "RUSTSEC-2026-0285", reason = "rustls fix needs aws-lc-rs >= 1.18 (AWS-LC-FIPS 4.x, uncertified)" }, + { id = "RUSTSEC-2026-0346", reason = "rustls fix needs aws-lc-rs >= 1.18 (AWS-LC-FIPS 4.x, uncertified)" }, + { id = "RUSTSEC-2026-0348", reason = "rustls fix needs aws-lc-rs >= 1.18 (AWS-LC-FIPS 4.x, uncertified)" }, + { id = "RUSTSEC-2026-0349", reason = "rustls fix needs aws-lc-rs >= 1.18 (AWS-LC-FIPS 4.x, uncertified)" }, + { id = "RUSTSEC-2026-0382", reason = "rustls-webpki fix needs aws-lc-rs ^1.18 (AWS-LC-FIPS 4.x, uncertified)" }, + { id = "RUSTSEC-2026-0391", reason = "zstd `with_context` is not used anywhere in the graph; fix is zstd 0.14" }, + { id = "RUSTSEC-2026-0398", reason = "zstd-safe `seekable` feature is not enabled; fix is zstd 0.14" }, + { id = "RUSTSEC-2026-0403", reason = "only `stream::read::Decoder` and `decode_all` are used, which the advisory excludes; runtime zstd-sys is built without the `legacy` feature" }, + { id = "RUSTSEC-2026-0410", reason = "wasm32 only; the extension ships on Linux" }, + { id = "RUSTSEC-2023-0050", reason = "multipart 0.18 is unmaintained; it parses AppSec multipart request bodies; no maintained replacement is chosen yet" }, + { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained; replace with rustls-pki-types PEM parsing (code change, not a version bump)" }, + { id = "RUSTSEC-2026-0386", reason = "tokio AsyncFd unsoundness; bottlecap does not construct AsyncFd. Fix: cargo update -p tokio (>= 1.53.3)" }, +] + +[bans] +# Duplicates cost binary size and cold-start relocations, but a new one is a +# review topic, not a failed PR. Known duplicates are listed in `skip` so the +# warning only shows new ones. +multiple-versions = "warn" +wildcards = "allow" +highlight = "all" +# Today's duplicates, oldest version(s) listed, with the crate that holds it +# back. Remove an entry when the holder moves. +skip = [ + { crate = "bitflags@1", reason = "nix 0.26 (bottlecap)" }, + { crate = "foldhash@0.1", reason = "hashbrown 0.15 via datadog-opentelemetry, dogstatsd, libdd-*" }, + { crate = "getrandom@0.2", reason = "rand 0.8 and ring" }, + { crate = "getrandom@0.3", reason = "rand_core 0.9 (rand 0.9 via opentelemetry_sdk) and jobserver (build only)" }, + { crate = "hashbrown@0.13", reason = "ordered_hash_map 0.4 (bottlecap)" }, + { crate = "hashbrown@0.15", reason = "datadog-opentelemetry, dogstatsd, libdd-remote-config, libdd-telemetry" }, + { crate = "itertools@0.11", reason = "bindgen and lalrpop (build only)" }, + { crate = "linux-raw-sys@0.4", reason = "rustix 0.38 via which (build only)" }, + { crate = "nix@0.26", reason = "bottlecap pins 0.26; libdd-common is on 0.29" }, + { crate = "opentelemetry-semantic-conventions@0.30", reason = "bottlecap pins 0.30; datadog-opentelemetry is on 0.32" }, + { crate = "ordered-float@2", reason = "serde-value via serde-aux" }, + { crate = "prost@0.13", reason = "datadog-protos (saluki pin) and tonic 0.13" }, + { crate = "prost-derive@0.13", reason = "prost 0.13" }, + { crate = "prost-types@0.13", reason = "datadog-protos (saluki pin)" }, + { crate = "rand@0.8", reason = "bottlecap, datadog-opentelemetry, libdd-trace-utils; opentelemetry_sdk is on 0.9" }, + { crate = "rand_chacha@0.3", reason = "rand 0.8" }, + { crate = "rand_core@0.6", reason = "rand 0.8" }, + { crate = "rustix@0.38", reason = "which 4 (build only)" }, + { crate = "thiserror@1", reason = "bottlecap, datadog-opentelemetry, dogstatsd, libdd-capabilities; libdd-remote-config and opentelemetry are on 2" }, + { crate = "thiserror-impl@1", reason = "thiserror 1" }, + { crate = "tonic@0.13", reason = "datadog-protos (saluki pin)" }, + { crate = "webpki-roots@0.26", reason = "hyper-http-proxy; 0.26.11 is a shim over 1.x" }, +] +skip-tree = [] +deny = [] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +allow-git = [] + +[sources.allow-org] +# serverless-components, saluki, libddwaf-rust. +github = ["DataDog"] + +[licenses] +# The set of licenses in today's graph. A crate with a new license fails the +# check until the license is reviewed and added here. +allow = [ + "0BSD", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-2-Clause-Patent", + "BSD-3-Clause", + "BSL-1.0", + "CC0-1.0", + "CDLA-Permissive-2.0", + "ISC", + "MIT", + "MIT-0", + "Unicode-3.0", + "Unlicense", + "Zlib", +] +exceptions = [ + # The AWS-LC FIPS module carries the OpenSSL license (fips feature only). + { crate = "aws-lc-fips-sys", allow = ["OpenSSL"] }, +] +confidence-threshold = 0.8 +# bottlecap itself has no license field (publish = false). +private = { ignore = true } From 5bf1e877f1d1a85255484294f7bd63303073585f Mon Sep 17 00:00:00 2001 From: Jordan Gonzalez <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:49:01 -0400 Subject: [PATCH 02/30] [SVLS-9975] chore(deps): remove 6 unused dependencies --- bottlecap/Cargo.lock | 50 ++-------------------------------- bottlecap/Cargo.toml | 15 ++++++---- bottlecap/LICENSE-3rdparty.csv | 4 --- 3 files changed, 11 insertions(+), 58 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 3475b27e9..cd689a4b0 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -420,16 +420,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ - "bit-vec 0.6.3", -] - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec 0.8.0", + "bit-vec", ] [[package]] @@ -438,12 +429,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" version = "1.3.2" @@ -501,7 +486,6 @@ dependencies = [ "datadog-fips", "datadog-opentelemetry", "datadog-protos", - "ddsketch-agent", "dogstatsd", "figment", "flate2", @@ -517,9 +501,7 @@ dependencies = [ "hyper-http-proxy", "hyper-rustls", "hyper-util", - "indexmap 2.13.0", "itertools 0.14.0", - "lazy_static", "libdd-capabilities", "libdd-common", "libdd-trace-normalization", @@ -528,7 +510,6 @@ dependencies = [ "libdd-trace-stats", "libdd-trace-utils", "libddwaf", - "log", "mime", "mock_instant", "multipart", @@ -536,7 +517,6 @@ dependencies = [ "opentelemetry-proto", "opentelemetry-semantic-conventions 0.30.0", "ordered_hash_map", - "proptest", "prost 0.14.3", "rand 0.8.6", "regex", @@ -548,7 +528,6 @@ dependencies = [ "rustls-pki-types", "rustls-webpki", "serde", - "serde-aux", "serde_bytes", "serde_html_form", "serde_json", @@ -2131,7 +2110,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" dependencies = [ "ascii-canvas", - "bit-set 0.5.3", + "bit-set", "ena", "itertools 0.11.0", "lalrpop-util", @@ -3129,16 +3108,12 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ - "bit-set 0.8.0", - "bit-vec 0.8.0", "bitflags 2.11.0", "num-traits", "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", - "rusty-fork", - "tempfile", "unarray", ] @@ -3760,18 +3735,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "rusty-fork" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - [[package]] name = "ryu" version = "1.0.23" @@ -4838,15 +4801,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wait-timeout" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" -dependencies = [ - "libc", -] - [[package]] name = "walkdir" version = "2.5.0" diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 26ec8582a..4e71339c9 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -4,6 +4,15 @@ version = "0.1.0" edition = "2024" publish = false +# CI runs `cargo machete` (rs_ci.yml). These entries are version bounds on +# transitive crates, a crate reached only through a re-export (aws-lc-rs via +# `rustls::crypto`), or a use cargo-machete cannot see (serde_bytes in a +# `#[serde(with = "...")]` string). Keep the reason next to the dependency. +# `time` is also only a version bound, but cargo-machete counts `std::time::` +# as a use, so listing it here triggers an "ignored but used" warning. +[package.metadata.cargo-machete] +ignored = ["aws-lc-rs", "rustls-native-certs", "rustls-webpki", "serde_bytes"] + [dependencies] anyhow = { version = "1.0", default-features = false } async-trait = { version = "0.1", default-features = false } @@ -18,8 +27,6 @@ hyper-util = { version = "0.1.16", features = ["http1", "client", "client-legacy http-body = { version = "1", default-features = false } http-body-util = { version = "0.1", default-features = false } itertools = { version = "0.14", default-features = false, features = ["use_std"] } -lazy_static = { version = "1.5", default-features = false } -log = { version = "0.4", default-features = false } mime = { version = "0.3", default-features = false } multipart = { version = "0.18", default-features = false, features = ["server"] } nix = { version = "0.26", default-features = false, features = ["feature", "fs"] } @@ -62,7 +69,6 @@ tonic = { version = "0.14", features = ["transport", "codegen", "server", "chann tonic-types = { version = "0.14", default-features = false } zstd = { version = "0.13.3", default-features = false } futures = { version = "0.3.31", default-features = false } -serde-aux = { version = "4.7", default-features = false } serde_html_form = { version = "0.2", default-features = false } opentelemetry-proto = { version = "0.32.0", features = ["trace", "with-serde", "gen-tonic"] } opentelemetry-semantic-conventions = { version = "0.30", features = ["semconv_experimental"] } @@ -76,14 +82,12 @@ tower-http = { version = "0.6.6", default-features = false, features = ["limit"] hyper-http-proxy = { version = "1.1.0", default-features = false, features = [ "rustls-tls-webpki-roots", ] } -indexmap = {version = "2.11.0", default-features = false} # If you are adding or updating a datadog-owned code dependency, please ensure # that it has a clippy.toml rule for disallowing the reqwest::Client::builder # method in favor of our # datadog_fips::reqwest_adapter::create_reqwest_client_builder. An example can # be found in the clippy.toml file adjacent to this Cargo.toml. datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "f863626dbfe3c59bb390985fa6530b0621c2a0a2"} -ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "f863626dbfe3c59bb390985fa6530b0621c2a0a2"} # These are pinned to the same crates.io versions that dd-trace-rs's datadog-opentelemetry # depends on, so cargo unifies them instead of compiling duplicate copies. Re-check with # `cargo tree --duplicates | grep ^libdd-` when bumping either side. @@ -103,7 +107,6 @@ libddwaf = { version = "1.28.1", git = "https://github.com/DataDog/libddwaf-rust [dev-dependencies] figment = { version = "0.10", default-features = false, features = ["yaml", "env", "test"] } -proptest = "1.4" httpmock = "0.7" tower = { version = "0.5", features = ["util"] } mock_instant = "0.6" diff --git a/bottlecap/LICENSE-3rdparty.csv b/bottlecap/LICENSE-3rdparty.csv index 07ec6ff60..a29e02387 100644 --- a/bottlecap/LICENSE-3rdparty.csv +++ b/bottlecap/LICENSE-3rdparty.csv @@ -15,8 +15,6 @@ aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND Apac axum,https://github.com/tokio-rs/axum,MIT,The axum Authors axum-core,https://github.com/tokio-rs/axum,MIT,The axum-core Authors base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce -bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner -bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye @@ -242,7 +240,6 @@ rustls-platform-verifier,https://github.com/rustls/rustls-platform-verifier,MIT rustls-platform-verifier-android,https://github.com/rustls/rustls-platform-verifier,MIT OR Apache-2.0,The rustls-platform-verifier-android Authors rustls-webpki,https://github.com/rustls/webpki,ISC,The rustls-webpki Authors rustversion,https://github.com/dtolnay/rustversion,MIT OR Apache-2.0,David Tolnay -rusty-fork,https://github.com/altsysrq/rusty-fork,MIT OR Apache-2.0,Jason Lingle ryu,https://github.com/dtolnay/ryu,Apache-2.0 OR BSL-1.0,David Tolnay safemem,https://github.com/abonander/safemem,MIT OR Apache-2.0,Austin Bonander schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " @@ -330,7 +327,6 @@ utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors value-bag,https://github.com/sval-rs/value-bag,Apache-2.0 OR MIT,Ashley Mannix -wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton want,https://github.com/seanmonstar/want,MIT,Sean McArthur wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers wasip2,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip2 Authors From b1575de24cc8443f5ee20bb9dad5f75b35028b4c Mon Sep 17 00:00:00 2001 From: Jordan Gonzalez <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:49:02 -0400 Subject: [PATCH 03/30] [SVLS-9975] ci: add a cargo-machete gate for unused dependencies --- .github/workflows/rs_ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/rs_ci.yml b/.github/workflows/rs_ci.yml index cd48a5a54..00a647cb7 100644 --- a/.github/workflows/rs_ci.yml +++ b/.github/workflows/rs_ci.yml @@ -228,3 +228,20 @@ jobs: with: cache: false - run: ./scripts/verify_tls_root_features.sh + + # Unused dependencies. Static (no build), so it is deterministic. A crate + # used only through a string path or kept as a version bound goes in + # `[package.metadata.cargo-machete] ignored` in bottlecap/Cargo.toml. The + # binary is called directly: it needs no cargo and no toolchain. + cargo-machete: + name: Unused Dependencies + runs-on: ubuntu-22.04 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 + with: + tool: cargo-machete@0.9.2 + - working-directory: bottlecap + run: cargo-machete From fa7a94d02a715705c489da62eb04bb2d275adf0d Mon Sep 17 00:00:00 2001 From: Jordan Gonzalez <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:49:03 -0400 Subject: [PATCH 04/30] [SVLS-9977] test: shared fixture corpus for tests and benches --- .gitignore | 4 +- bottlecap/src/extension/telemetry/mod.rs | 3 +- bottlecap/src/extension/telemetry/stitch.rs | 6 +- bottlecap/tests/fixtures/README.md | 70 +++++ bottlecap/tests/fixtures/dogstatsd.rs | 69 +++++ bottlecap/tests/fixtures/events.rs | 137 ++++++++++ bottlecap/tests/fixtures/logs.rs | 73 +++++ bottlecap/tests/fixtures/mod.rs | 156 +++++++++++ bottlecap/tests/fixtures/telemetry.rs | 278 ++++++++++++++++++++ bottlecap/tests/fixtures/traces.rs | 161 ++++++++++++ bottlecap/tests/fixtures_corpus.rs | 101 +++++++ 11 files changed, 1053 insertions(+), 5 deletions(-) create mode 100644 bottlecap/tests/fixtures/README.md create mode 100644 bottlecap/tests/fixtures/dogstatsd.rs create mode 100644 bottlecap/tests/fixtures/events.rs create mode 100644 bottlecap/tests/fixtures/logs.rs create mode 100644 bottlecap/tests/fixtures/mod.rs create mode 100644 bottlecap/tests/fixtures/telemetry.rs create mode 100644 bottlecap/tests/fixtures/traces.rs create mode 100644 bottlecap/tests/fixtures_corpus.rs diff --git a/.gitignore b/.gitignore index 78015339d..06fcc1e10 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,6 @@ bottlecap/proptest-regressions integration-tests/cdk.context.json .gitlab/pipeline* -/CLAUDE_PERSONAL.md \ No newline at end of file +/CLAUDE_PERSONAL.md +# Generated benchmark fixture corpus (see bottlecap/tests/fixtures/README.md) +bottlecap/tests/fixtures/generated diff --git a/bottlecap/src/extension/telemetry/mod.rs b/bottlecap/src/extension/telemetry/mod.rs index 93b362917..a7b6d2ebb 100644 --- a/bottlecap/src/extension/telemetry/mod.rs +++ b/bottlecap/src/extension/telemetry/mod.rs @@ -5,7 +5,8 @@ use crate::extension::{EXTENSION_ID_HEADER, base_url}; pub mod events; pub mod listener; -mod stitch; +#[doc(hidden)] +pub mod stitch; pub const TELEMETRY_SUBSCRIPTION_ROUTE: &str = "2022-07-01/telemetry"; diff --git a/bottlecap/src/extension/telemetry/stitch.rs b/bottlecap/src/extension/telemetry/stitch.rs index 0807ff5fc..bdf2eed3f 100644 --- a/bottlecap/src/extension/telemetry/stitch.rs +++ b/bottlecap/src/extension/telemetry/stitch.rs @@ -67,7 +67,7 @@ const MAX_SEAMS: usize = 64; /// What came of pairing an unparseable body with a held fragment. #[derive(Debug)] -pub(crate) enum Stitch { +pub enum Stitch { /// The body completed a split payload. Complete(Vec), /// The body opens a split payload, and is held for its continuation. @@ -78,13 +78,13 @@ pub(crate) enum Stitch { /// Holds the leading half of a split payload until its continuation arrives. #[derive(Clone, Default)] -pub(crate) struct FragmentBuffer { +pub struct FragmentBuffer { held: Arc>>, } impl FragmentBuffer { /// Joins `body` onto the held fragment, or holds `body` if it opens a split payload. - pub(crate) fn stitch(&self, body: &[u8]) -> Stitch { + pub fn stitch(&self, body: &[u8]) -> Stitch { let mut slot = self.held.lock().unwrap_or_else(PoisonError::into_inner); if let Some(stale) = slot.take_if(|held| held.received.elapsed() > FRAGMENT_TTL) { diff --git a/bottlecap/tests/fixtures/README.md b/bottlecap/tests/fixtures/README.md new file mode 100644 index 000000000..d5c1efe52 --- /dev/null +++ b/bottlecap/tests/fixtures/README.md @@ -0,0 +1,70 @@ +# Fixture corpus + +Shared inputs for the criterion benches (`bottlecap/benches/`), the fault-injection harness and +the Lambda simulator. Everything is generated by the Rust module in this directory, so no large +blob is checked in. + +## Use from Rust + +```rust +#[path = "../tests/fixtures/mod.rs"] // from benches/; integration tests use `mod fixtures;` +mod fixtures; +use fixtures::{dogstatsd, events, logs, telemetry, traces}; +``` + +| Module | Generator | What it returns | +|---|---|---| +| `telemetry` | `platform_batch(request_id) -> Vec` | One cold-start invocation: `initStart`, `initRuntimeDone`, `initReport`, `start`, `runtimeDone`, `report` | +| | `log_batch(LogKind, n) -> Vec` | One POST body of `n` function log events (`LogKind::{Text, JsonObject, JsonString}`) | +| | `log_batches(LogKind, n, per_post) -> Vec>` | The same, split into POSTs of `per_post` events (Lambda sends at most 1000) | +| | `invocation_batch(LogKind, n) -> Vec` | `platform.start`, `n` log lines, `runtimeDone`, `report` | +| | `split_record_posts(total, piece, per_post) -> Vec>` | A record of `total` bytes cut into `piece`-byte pieces, spread over POSTs; the last carries `runtimeDone`. Feed to `FragmentBuffer::stitch` in order | +| | `split_record_single_post(total, piece) -> Vec` | All pieces of one split record in one body (seams only) | +| | `runtime_done_with(id, status, with_metrics)`, `logs_dropped(n)`, `unknown_event()` | A `runtimeDone` of any status with or without `metrics` (Lambda omits it on a timeout), a `platform.logsDropped`, an event of an unknown type | +| | `event(millis, type, record_json)`, `init_start()`, `platform_start(id)`, `runtime_done(id)`, `report(id)`, `function_event(kind, i)`, `extension_event(i)`, `batch(&[String])` | Building blocks | +| | `REQUEST_ID`, `FUNCTION_ARN` | Constants used everywhere | +| `logs` | `text_line(i)`, `json_object(i) -> Value`, `json_line(i)`, `text_line_of_len(i, len)`, `OOM_LINES` | Raw log lines: Lambda text format (~200 B), Powertools-like JSON (~2 KB), one OOM line per runtime | +| `traces` | `make_traces(n_traces, spans_per_trace, with_sql) -> Vec>` | Synthetic traces: `aws.lambda` root, SQL and HTTP clients, internal spans | +| | `trace_2000_spans()`, `v04_2000_spans() -> Vec` | The reference payload: 100 x 20 spans, ~1 MB v0.4 msgpack | +| | `v04_msgpack(&traces)`, `header_tags()`, `TRACE_HEADERS` | Encoding and the matching tracer headers | +| `dogstatsd` | `line(Kind, i)`, `packet(Kind, first, n)`, `packets(Kind, total, per_packet)`, `unique_tag_line(i)` | Datagrams: 50 metric names x 100 endpoints; `Kind::{Count, Gauge, Distribution}` | +| | `ENRICH_TAGS`, `DIST_8_TAGS`, `COUNT_2_TAGS`, `SKIPPED_LINES` | Constants | +| `events` | `payload(name)`, `payload_names()`, `all_payloads()` | The 38 trigger events in `tests/payloads/` | +| | `sqs_event(records, body_bytes)`, `sqs_220kb()` | SQS batch with `_datadog` trace context; reference 10 x 22 KB | +| | `dynamodb_event(records, attrs)`, `dynamodb_100_records()` | DynamoDB stream batch; reference 100 x 30 attributes | +| | `lwa_next_headers(request_id)` | Headers of the `/invocation/next` response LWA intercepts | +| root | `write_corpus(&dir) -> Vec<(PathBuf, usize)>`, `default_corpus_dir()` | Writes the files below | + +## Files on disk + +Generated on demand. Run from `bottlecap/`: + +```sh +cargo test --test fixtures_corpus write_fixture_corpus -- --ignored +``` + +Output goes to `bottlecap/tests/fixtures/generated/` (gitignored), or to `$BOTTLECAP_FIXTURES_DIR`. +`MANIFEST.txt` lists every file with its size. + +``` +telemetry/platform_batch.json platform events of one cold start +telemetry/function_text_1000.json 1000 text log events, one POST +telemetry/function_json_object_1000.json 1000 JSON-object log events, one POST +telemetry/function_json_string_1000.json 1000 JSON-in-string log events, one POST +telemetry/invocation_batch_text_200.json start + 200 lines + runtimeDone + report +telemetry/split_record_1mib_post_{0..1}.json a 1 MiB record in 256 KiB pieces, 3 per POST +telemetry/split_record_1mib_single_post.json the same pieces in one body +traces/v04_2000_spans.msgpack 100 traces x 20 spans +traces/v04_100_spans.msgpack 5 traces x 20 spans +logs/text_lines_1000.txt raw text lines +logs/json_lines_1000.jsonl raw JSON lines +logs/oom_lines.txt one OOM line per runtime +dogstatsd/count_packet_{0..9}.txt 20 count lines per datagram +dogstatsd/distribution_packet_{0..9}.txt 20 distribution lines per datagram +dogstatsd/enrich_tags.txt the tags bottlecap adds to every metric +events/sqs_220kb.json 10 SQS records x 22 KB +events/dynamodb_100_records.json 100 DynamoDB records x 30 attributes +events/payloads/.json a copy of the 38 trigger events in tests/payloads/ +``` + +The checked-in trigger events stay in `bottlecap/tests/payloads/`. diff --git a/bottlecap/tests/fixtures/dogstatsd.rs b/bottlecap/tests/fixtures/dogstatsd.rs new file mode 100644 index 000000000..aa6698fba --- /dev/null +++ b/bottlecap/tests/fixtures/dogstatsd.rs @@ -0,0 +1,69 @@ +//! DogStatsD datagrams as a client library sends them to UDP 8125. + +/// The tags bottlecap adds to every metric (`SortedTags::parse(ENRICH_TAGS)`). +pub const ENRICH_TAGS: &str = "function_arn:arn:aws:lambda:us-east-1:123456789012:function:my-checkout-fn,region:us-east-1,account_id:123456789012,aws_account:123456789012,functionname:my-checkout-fn,resource:my-checkout-fn,executedversion:$LATEST,memorysize:128,architecture:arm64,runtime:nodejs22.x,_dd.runtime_family:nodejs,env:prod,service:checkout,version:1.2.3,dd_extension_version:90,_dd.compute_stats:1"; + +/// A distribution with 8 tags, the heaviest common line. +pub const DIST_8_TAGS: &str = "app.checkout.latency:12.5|d|#env:prod,service:checkout,version:1.2.3,region:us-east-1,team:payments,endpoint:/api/v1/pay,status:200,method:POST"; + +/// A count with 2 tags, the lightest common line. +pub const COUNT_2_TAGS: &str = "my.metric:1|c|#tag1:a,tag2:b"; + +/// Lines bottlecap drops before parsing: service checks, events, and its own enhanced metric. +pub const SKIPPED_LINES: [&str; 3] = [ + "_sc|my.check|0|#env:prod", + "_e{5,4}:title|text", + "aws.lambda.enhanced.invocations:1|c", +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + Count, + Gauge, + Distribution, +} + +/// One metric line. The name cycles over 50 values and the `endpoint` tag over 100, so a +/// stream of `n` lines has at most 5000 contexts. +pub fn line(kind: Kind, i: usize) -> String { + match kind { + Kind::Count => format!( + "app.request.count{}:1|c|#endpoint:/api/v{},status:200,method:GET", + i % 50, + i % 100 + ), + Kind::Gauge => format!( + "app.queue.depth{}:{}|g|#endpoint:/api/v{},status:200,method:GET", + i % 50, + i % 900, + i % 100 + ), + Kind::Distribution => format!( + "app.request.latency{}:{}|d|#endpoint:/api/v{},status:200,method:GET", + i % 50, + (i % 900) + 1, + i % 100 + ), + } +} + +/// A line whose tag value is unique per `i` (one new interned string per line). +pub fn unique_tag_line(i: usize) -> String { + format!("my.metric:1|c|#env:prod,service:api,request_id:{i:08x}-0000-4000-8000-{i:012x}") +} + +/// `n` lines joined by `\n`, one UDP datagram (keep it under 8 KiB). +pub fn packet(kind: Kind, first: usize, n: usize) -> String { + (first..first + n) + .map(|i| line(kind, i)) + .collect::>() + .join("\n") +} + +/// `total` lines split into datagrams of `per_packet` lines. +pub fn packets(kind: Kind, total: usize, per_packet: usize) -> Vec { + (0..total) + .step_by(per_packet.max(1)) + .map(|first| packet(kind, first, per_packet.min(total - first))) + .collect() +} diff --git a/bottlecap/tests/fixtures/events.rs b/bottlecap/tests/fixtures/events.rs new file mode 100644 index 000000000..3741ffbe7 --- /dev/null +++ b/bottlecap/tests/fixtures/events.rs @@ -0,0 +1,137 @@ +//! Invocation payloads: the 38 trigger events in `tests/payloads/`, plus large generated +//! SQS and DynamoDB events. + +use std::path::{Path, PathBuf}; + +use serde_json::{Value, json}; + +fn payloads_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("payloads") +} + +/// Reads `tests/payloads/.json`. +pub fn payload(name: &str) -> String { + let path = payloads_dir().join(format!("{name}.json")); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +/// Names of every file in `tests/payloads/`, sorted, without the `.json` suffix. +pub fn payload_names() -> Vec { + let mut names: Vec = std::fs::read_dir(payloads_dir()) + .expect("list tests/payloads") + .filter_map(Result::ok) + .filter_map(|e| { + let name = e.file_name().to_str()?.to_string(); + name.strip_suffix(".json").map(str::to_string) + }) + .collect(); + names.sort(); + names +} + +/// Every payload, parsed, with its name. +pub fn all_payloads() -> Vec<(String, Value)> { + payload_names() + .into_iter() + .map(|name| { + let value: Value = serde_json::from_str(&payload(&name)) + .unwrap_or_else(|e| panic!("parse payload {name}: {e}")); + (name, value) + }) + .collect() +} + +/// An SQS batch of `records` messages, each with a `body_bytes` body and a `_datadog` +/// message attribute carrying trace context. +pub fn sqs_event(records: usize, body_bytes: usize) -> Value { + let body: String = (0..body_bytes / 40) + .map(|i| format!("{{\"k{i}\":\"v{i}xxxxxxxxxxxxxxxxxxxxxx\"}},")) + .collect(); + let body = format!("[{}]", body.trim_end_matches(',')); + let recs: Vec = (0..records) + .map(|i| { + json!({ + "messageId": format!("059f36b4-87a3-44ab-83d2-66197{i:07}"), + "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", + "body": body, + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1545082649183", + "SenderId": "AIDAIENQZJOLO23YVJ4VO", + "ApproximateFirstReceiveTimestamp": "1545082649185" + }, + "messageAttributes": { + "_datadog": { + "stringValue": "{\"x-datadog-trace-id\":\"2684756524522091840\",\"x-datadog-parent-id\":\"7431398482019833808\",\"x-datadog-sampling-priority\":\"1\"}", + "stringListValues": [], + "binaryListValues": [], + "dataType": "String" + } + }, + "md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:my-queue", + "awsRegion": "us-east-1" + }) + }) + .collect(); + json!({ "Records": recs }) +} + +/// The reference SQS payload: 10 records of 22 KB, about 220 KB serialized. +pub fn sqs_220kb() -> Value { + sqs_event(10, 22_000) +} + +/// A DynamoDB stream batch of `records` INSERT records with `attrs` attributes each. +pub fn dynamodb_event(records: usize, attrs: usize) -> Value { + let recs: Vec = (0..records) + .map(|i| { + let mut img = serde_json::Map::new(); + for a in 0..attrs { + img.insert(format!("attr{a}"), json!({"S": format!("value-{i}-{a}")})); + } + json!({ + "eventID": format!("{i}"), + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "us-east-1", + "dynamodb": { + "ApproximateCreationDateTime": 1_428_537_600, + "Keys": {"Id": {"N": "101"}}, + "NewImage": Value::Object(img), + "SequenceNumber": format!("{i}00000000000000000"), + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "eventSourceARN": "arn:aws:dynamodb:us-east-1:123456789012:table/ExampleTableWithStream/stream/2015-06-27T00:48:05.899" + }) + }) + .collect(); + json!({ "Records": recs }) +} + +/// The reference DynamoDB payload: 100 records of 30 attributes, about 100 KB serialized. +pub fn dynamodb_100_records() -> Value { + dynamodb_event(100, 30) +} + +/// Headers of the `GET /runtime/invocation/next` response that LWA intercepts. +pub fn lwa_next_headers(request_id: &str) -> Vec<(&'static str, String)> { + vec![ + ("content-type", "application/json".to_string()), + ("lambda-runtime-aws-request-id", request_id.to_string()), + ("lambda-runtime-deadline-ms", "1741965555094".to_string()), + ( + "lambda-runtime-invoked-function-arn", + crate::fixtures::telemetry::FUNCTION_ARN.to_string(), + ), + ( + "lambda-runtime-trace-id", + "Root=1-67d448e8-3ae320be0e2cdf2f53ccdaba;Lineage=1:73f724a8:0".to_string(), + ), + ] +} diff --git a/bottlecap/tests/fixtures/logs.rs b/bottlecap/tests/fixtures/logs.rs new file mode 100644 index 000000000..162c37d59 --- /dev/null +++ b/bottlecap/tests/fixtures/logs.rs @@ -0,0 +1,73 @@ +//! Raw log lines, as a function writes them to stdout. + +use serde_json::Value; + +use crate::fixtures::telemetry::REQUEST_ID; + +/// Lambda text-format line (about 200 bytes): `