diff --git a/.cargo/config.toml b/.cargo/config.toml index aea0df2d59..0c81704c8b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -28,8 +28,8 @@ test_details = "test --target aarch64-apple-darwin" # (native) are simply not listed. build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. diff --git a/AGENTS.md b/AGENTS.md index d90eda1f2d..d3aafb14a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ crates/ trusted-server-adapter-cloudflare/ # Cloudflare Workers entry point (wasm32-unknown-unknown binary) trusted-server-adapter-spin/ # Fermyon Spin entry point (wasm32-wasip1 component) trusted-server-cli/ # Host-target `ts` operator CLI + trusted-server-identity-providers/ # Server-side identity provider kinds (used by the Fastly adapter) trusted-server-js/ # TypeScript/JS build — per-integration IIFE bundles lib/ # TS source, Vitest tests, esbuild pipeline ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d5d0b9ea8..c1dc4349f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 measured from dispatch, including delayed headers and trickling bodies on all adapters. Complete available responses survive a slow publisher origin; unfinished providers time out and mediation is skipped after budget exhaustion. - Provider bids that are already complete at the final nonblocking snapshot remain + Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. + On Fastly the final snapshot may wait up to 1 ms per partially received body so + the host can surface bytes that already arrived, and keeps a response whose + headers arrived by the cutoff. +- Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, pull sync skips them, and browser IDs never fill them. Set `tcf_vendor_id` to gate both lookups and the provider's stored IDs in bidstream and `/identify` on TCF vendor consent. See [Identity providers](docs/guide/configuration.md#identity-providers). +- Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its size limit, and continues with the remaining groups. +- **Breaking:** An EC is now minted, and the EC row written, only on a top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, integration proxies and `/auction` no longer mint one, and prefetches and prerenders never mint or write. Each navigation makes at most one conditional page-load write; a consent withdrawal's tombstone is still written on any request that carries the EC cookie. +- **Breaking:** Browser EIDs from the `/auction` request body are no longer stored, and `/auction` and `/_ts/page-bids` no longer write browser EIDs from `ts-eids` or `sharedId`. Those cookies are ingested on the next top-level navigation. +- **Breaking:** An EC minted on the current navigation is no longer sent as `user.id` to bidders, and identity providers are not called for it, until a later navigation finds its row. +- **Breaking:** Axum, Cloudflare and Spin refuse to start when an enabled identity provider is configured, and Fastly refuses to start when an enabled provider names a `kind` this build does not include. +- **Breaking (rollback):** Stored provider UIDs may be up to 1024 bytes (browser, batch and pull ingest stay at 512). A binary from before this release cannot read an EC row holding a longer UID, so roll back only after such rows expire or are rewritten. A config blob that sets `ec.identity_providers` or a non-default identity budget cannot be loaded by an older binary; restore the defaults and re-push before rolling back. - Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. - Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/Cargo.lock b/Cargo.lock index 70316712ac..d9cf35f9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6604,6 +6604,7 @@ dependencies = [ "serde_json", "toml", "trusted-server-core", + "trusted-server-identity-providers", "url", "urlencoding", "web-time", @@ -6737,6 +6738,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "trusted-server-identity-providers" +version = "0.1.0" +dependencies = [ + "error-stack", + "toml", + "trusted-server-core", +] + [[package]] name = "trusted-server-integration-tests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 139cf4f610..f431012308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/trusted-server-adapter-spin", "crates/trusted-server-cli", "crates/trusted-server-core", + "crates/trusted-server-identity-providers", "crates/trusted-server-integration-tests", "crates/trusted-server-js", "crates/trusted-server-openrtb", @@ -121,6 +122,7 @@ toml_edit = "0.23.10" tower = "0.4" tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } +trusted-server-identity-providers = { path = "crates/trusted-server-identity-providers" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } url = "2.5.8" diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 0b1858e208..8e3143bdaa 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -91,6 +91,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Axum")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -224,15 +225,12 @@ async fn dispatch_fallback( } if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); return state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services, req, }) @@ -499,8 +497,7 @@ fn named_route_handler( Ok(admin_ec_lookup_not_supported()) } NamedRouteHandler::AdminEidsLookup => { - let partner_registry = - PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 668b025ccb..3f96bc9217 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -204,6 +204,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Cloudflare")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -535,15 +536,12 @@ fn build_router(state: &Arc) -> RouterService { EdgeCacheHeader::CloudflareCdnCacheControl, ) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) @@ -655,7 +653,7 @@ fn build_router(state: &Arc) -> RouterService { .get( "/_ts/admin/eids", make_handler(Arc::clone(&state), |s, _services, req| async move { - let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&s.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) }), ) diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 6c7121d0d6..0e5d6aaceb 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -37,6 +37,7 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-identity-providers = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } web-time = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ce06ce7e30..82e9aee082 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -86,6 +86,8 @@ //! [`load_settings_from_config_store`]; if settings loading itself fails, they //! are returned without geo or TS headers. +use std::cell::RefCell; +use std::net::IpAddr; use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; @@ -97,7 +99,7 @@ use edgezero_core::http::{ HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, }; use edgezero_core::router::RouterService; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -111,11 +113,18 @@ use trusted_server_core::ec::admin::{ }; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, PendingLookups, ProviderCoordinator, +}; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::ec::{EcContext, EidSyncSource}; +use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ @@ -123,7 +132,8 @@ use trusted_server_core::integrations::{ RequestFilterRegistryOutcome, }; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, PlatformPendingRequest, + RuntimeServices, StoreName, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, @@ -142,6 +152,9 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; +use trusted_server_identity_providers::{ + build_after_response_coordinator, build_coordinator_with, partner_registry, +}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -190,14 +203,28 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, + /// Partner registry, built with the state. + pub(crate) partner_registry: Arc, + /// Coordinator for providers collected before the response, if any. + pub(crate) page_identity: Option>, + /// Coordinator for providers collected after the response, if any. + pub(crate) after_response_identity: Option>, +} + +impl AppState { + /// Returns the partner registry. + pub(crate) fn partner_registry(&self) -> Arc { + Arc::clone(&self.partner_registry) + } } /// Build the application state, loading settings and constructing all per-application components. /// /// # Errors /// -/// Returns an error when settings, the auction orchestrator, or the integration -/// registry fail to initialise. +/// Returns an error when settings, the auction orchestrator, the integration +/// registry, the partner registry or an enabled identity provider fail to +/// initialise. pub(crate) fn build_state( stores: &RuntimeStoreConfig, ) -> Result, Report> { @@ -229,12 +256,30 @@ pub(crate) fn build_state_from_settings( let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + // A provider that cannot run must stop startup: skipping it would quietly + // hand its sources back to browser ingest. + let partner_registry = Arc::new(partner_registry(&settings.ec)?); + let page_identity = build_coordinator_with(&settings, Arc::clone(&partner_registry)) + .change_context(TrustedServerError::Configuration { + message: "identity provider configuration is invalid".to_owned(), + })? + .map(Arc::new); + let after_response_identity = + build_after_response_coordinator(&settings, Arc::clone(&partner_registry)) + .change_context(TrustedServerError::Configuration { + message: "identity provider configuration is invalid".to_owned(), + })? + .map(Arc::new); + Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, + partner_registry, + page_identity, + after_response_identity, })) } @@ -334,6 +379,141 @@ pub(crate) struct EcFinalizeState { pub(crate) services: RuntimeServices, } +/// Identity lookups dispatched for this navigation, collected before finalize. +pub(crate) struct IdentityLookups { + pub(crate) coordinator: Arc, + pub(crate) pending: PendingLookups, +} + +/// What a navigation's after-response identity lookups need once the response +/// has been sent. +/// +/// The router owns the request, so the request signals the providers read are +/// copied here. The lookups are dispatched only after the response, so they +/// never delay the page. +pub(crate) struct AfterResponseLookups { + pub(crate) ec_id: String, + pub(crate) client_ip: Option, + pub(crate) user_agent: Option, +} + +/// Identity work a navigation hands to the entry point. +/// +/// In-flight platform requests are not `Send`, so they cannot ride in +/// response extensions with [`EcFinalizeState`]. They travel through one +/// request-local slot instead: the entry point clears it before routing and +/// takes it after the router returns. +#[derive(Default)] +pub(crate) struct NavigationLookups { + /// Page-phase lookups, collected before EC finalization. + pub(crate) page: Option, + /// After-response lookups, dispatched once the response has been sent. + pub(crate) after_response: Option, +} + +thread_local! { + static NAVIGATION_LOOKUPS: RefCell = RefCell::new(NavigationLookups::default()); +} + +/// Takes the identity work stored for the current request. +/// +/// The entry point calls this before routing (discarding anything a prior +/// request left behind) and after the router returns. +pub(crate) fn take_navigation_lookups() -> NavigationLookups { + NAVIGATION_LOOKUPS.with(|slot| std::mem::take(&mut *slot.borrow_mut())) +} + +/// Request signals lent to identity providers. +struct ProviderRequestSignals<'a> { + client_ip: Option, + user_agent: Option<&'a str>, +} + +impl<'a> ProviderRequestSignals<'a> { + fn read(req: &'a Request, ec: &EcRequestState) -> Self { + Self { + client_ip: ec + .ec_context + .client_ip() + .and_then(|ip| ip.parse::().ok()), + user_agent: req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()), + } + } +} + +/// Captures the request signals for after-response identity lookups on a +/// permitted publisher navigation. +/// +/// Returns `None` when no after-response provider is enabled, the request has +/// no EC, or the identity graph is unavailable. +fn prepare_after_response_lookups( + state: &AppState, + req: &Request, + ec: &EcRequestState, +) -> Option { + state.after_response_identity.as_ref()?; + ec.kv_graph.as_ref()?; + let ec_id = ec.ec_context.ec_value()?.to_owned(); + let signals = ProviderRequestSignals::read(req, ec); + Some(AfterResponseLookups { + ec_id, + client_ip: signals.client_ip, + user_agent: signals.user_agent.map(str::to_owned), + }) +} + +/// Starts identity-provider lookups for a permitted publisher navigation. +/// +/// Returns `None` when no provider is enabled, the request has no EC or one +/// minted on this request, or the identity graph is unavailable. +async fn dispatch_identity_lookups( + state: &AppState, + services: &RuntimeServices, + req: &Request, + ec: &mut EcRequestState, +) -> Option { + let coordinator = Arc::clone(state.page_identity.as_ref()?); + let graph = ec.kv_graph.as_ref()?; + // An EC minted on this request has no row yet and may never be issued, + // so it is not sent to vendors; a later navigation looks it up. + if ec.ec_context.ec_generated() { + return None; + } + let ec_id = ec.ec_context.ec_value()?.to_owned(); + if matches!(ec.ec_context.kv_snapshot(), EcKvSnapshot::NotRead) { + let snapshot = graph.load_snapshot(&ec_id); + ec.ec_context.set_kv_snapshot(snapshot); + } + let transport = PlatformLookupTransport::new(services); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let signals = ProviderRequestSignals::read(req, ec); + let now = trusted_server_core::ec::current_timestamp(); + let pending = coordinator + .dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: ec.ec_context.kv_snapshot(), + consent: ec.ec_context.consent(), + client: ClientSignals { + ip: signals.client_ip, + user_agent: signals.user_agent, + }, + publisher_domain: &state.settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + ) + .await; + Some(IdentityLookups { + coordinator, + pending, + }) +} + /// Per-request EC identity state built before dispatch, mirroring the /// pre-routing prelude of the legacy `route_request` (device signals, bot /// gate, cookie capture, consent/geo-aware [`EcContext`], and KV graph @@ -579,19 +759,19 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = PartnerRegistry::from_config(&state.settings.ec.partners) - .and_then(|registry| match handler { - NamedRouteHandler::AdminEcLookup => { - // Deliberately do not use an EC request-state graph: that - // copy is bot-gated, while operators use curl for this - // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); - handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) - } - NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), - _ => unreachable!("admin diagnostics should use early dispatch"), - }) - .unwrap_or_else(|error| http_error(&error)); + let registry = state.partner_registry(); + let response = match handler { + NamedRouteHandler::AdminEcLookup => { + // Deliberately do not use an EC request-state graph: that + // copy is bot-gated, while operators use curl for this + // authenticated diagnostic. + let kv = crate::maybe_identity_graph(&state.settings); + handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) + } + NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), + _ => unreachable!("admin diagnostics should use early dispatch"), + } + .unwrap_or_else(|error| http_error(&error)); return Ok(response); } @@ -660,7 +840,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); handle_identify( &state.settings, &kv, @@ -674,11 +854,11 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; handle_auction( &state.settings, @@ -698,11 +878,11 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; let auction = AuctionDispatch { integration_registry: &state.registry, @@ -745,7 +925,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -820,8 +1000,6 @@ async fn dispatch_fallback( method: &method, path: &path, settings: &state.settings, - kv: ec.kv_graph.as_ref(), - ec_context: &mut ec.ec_context, services, req, }) @@ -848,11 +1026,21 @@ async fn dispatch_fallback( // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. let is_navigation = is_navigation_request(&req); + let mut has_permit = false; if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); + // The page-load write permit uses the stricter Fetch Metadata test + // (spec §3.1); requests without it never write KV. + if let Some(permit) = NavigationPermit::classify(&req) { + ec.ec_context.set_navigation_permit(permit); + has_permit = true; + } } let is_publisher_navigation = ec.is_real_browser && is_navigation; + // A new EC is minted only when this request may also write its row, + // so a client without Fetch Metadata never receives a throwaway EC. if is_publisher_navigation + && has_permit && let Err(err) = ec .ec_context .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) @@ -860,6 +1048,23 @@ async fn dispatch_fallback( log::warn!("EC generation failed for publisher proxy: {err:?}"); } + // Identity lookups start before the origin fetch so they overlap it; + // the entry point collects them before EC finalization. + if is_publisher_navigation + && has_permit + && let Some(lookups) = dispatch_identity_lookups(state, services, &req, &mut ec).await + { + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().page = Some(lookups)); + } + // After-response providers only capture request signals here; they + // are built and dispatched once the response has been sent. + if is_publisher_navigation + && has_permit + && let Some(lookups) = prepare_after_response_lookups(state, &req, &ec) + { + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().after_response = Some(lookups)); + } + // Run the server-side auction with the configured creative- // opportunity slots and collect dispatched bids from the lazy // publisher body stream. `handle_publisher_request` matches the @@ -867,44 +1072,40 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_config(&state.settings.ec.partners) { - Ok(partner_registry) => { - let auction = AuctionDispatch { - integration_registry: &state.registry, - orchestrator: &state.orchestrator, - slots, - registry: Some(&partner_registry), - }; - match handle_publisher_request( - &state.settings, - services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, - EdgeCacheHeader::SurrogateControl, + let partner_registry = state.partner_registry(); + let auction = AuctionDispatch { + integration_registry: &state.registry, + orchestrator: &state.orchestrator, + slots, + registry: Some(&*partner_registry), + }; + match handle_publisher_request( + &state.settings, + services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, + EdgeCacheHeader::SurrogateControl, + ) + .await + { + Ok(pub_response) => { + // Origin start succeeded on the sole publisher-page + // path. Authorize orphan recovery only for real-browser + // document navigations so named routes, integration + // proxies, and filter short circuits cannot rotate an + // identity. + ec.ec_context.set_recovery_eligible(is_publisher_navigation); + publisher_response_into_streaming_response( + pub_response, + &method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + services.clone(), ) .await - { - Ok(pub_response) => { - // Origin start succeeded on the sole publisher-page - // path. Authorize orphan recovery only for real-browser - // document navigations so named routes, integration - // proxies, and filter short circuits cannot rotate an - // identity. - ec.ec_context.set_recovery_eligible(is_publisher_navigation); - publisher_response_into_streaming_response( - pub_response, - &method, - Arc::clone(&state.settings), - state.registry.as_ref(), - Arc::clone(&state.orchestrator), - services.clone(), - ) - .await - } - Err(e) => Err(e), - } } Err(e) => Err(e), } @@ -1421,8 +1622,24 @@ mod tests { use error_stack::Report; use futures::StreamExt as _; use futures::executor::block_on; + use std::borrow::Cow; + use std::num::NonZeroUsize; + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::consent::jurisdiction::Jurisdiction; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; + use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, ProviderCoordinator, RegisteredProvider, + }; use trusted_server_core::ec::device::DeviceSignals; + use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; + use trusted_server_core::ec::provider::{ + ClientSignals, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, + ProviderResponse, ProviderUid, SourceCapability, SourceInput, + }; + use trusted_server_core::ec::rate_limiter::RateLimiter; + use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, @@ -1435,7 +1652,7 @@ mod tests { RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; - use trusted_server_core::settings::Settings; + use trusted_server_core::settings::{EcPartner, IdentityProviderConfig, Settings}; use trusted_server_core::test_support::nextjs_auction; #[test] @@ -1615,6 +1832,9 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + partner_registry: Arc::new(PartnerRegistry::empty()), + page_identity: None, + after_response_identity: None, }); TrustedServerApp::routes_for_state(&state) } @@ -2578,6 +2798,395 @@ mod tests { } } + fn navigation_request(path: &str) -> Request { + let mut request = browser_request(Method::GET, path, "document"); + request + .headers_mut() + .insert("sec-fetch-mode", HeaderValue::from_static("navigate")); + request + } + + fn take_finalize_state(response: &mut Response) -> super::EcFinalizeState { + response + .extensions_mut() + .remove::() + .expect("should attach finalize state") + } + + #[test] + fn publisher_navigation_receives_navigation_permit() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_some(), + "should issue a permit" + ); + } + + fn consented_navigation(mut request: Request) -> Request { + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + request + } + + #[test] + fn enabled_provider_with_unregistered_kind_fails_startup() { + let mut settings = test_settings(); + settings.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "Example IDs", + "source_domain": "ids.example.com" + })) + .expect("should parse partner"), + ); + settings.ec.identity_providers.push( + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider"), + ); + + let result = build_state_from_settings(settings); + + assert!( + result.is_err(), + "should refuse to start with a provider this build cannot run" + ); + } + + #[test] + fn navigation_without_fetch_metadata_mints_no_ec() { + let router = test_router(); + let mut request = consented_navigation(empty_request(Method::GET, "/article")); + request + .headers_mut() + .insert(header::ACCEPT, HeaderValue::from_static("text/html")); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + + let mut response = route(&router, request); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_none(), + "should not mint an EC that no row can back" + ); + } + + #[test] + fn navigation_with_fetch_metadata_mints_an_ec() { + let router = test_router(); + + let mut response = route( + &router, + consented_navigation(navigation_request("/article")), + ); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_some(), + "should mint an EC on a page-load navigation" + ); + } + + #[test] + fn publisher_navigation_without_providers_dispatches_nothing() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let _state = take_finalize_state(&mut response); + assert!( + super::take_navigation_lookups().page.is_none(), + "should dispatch no identity lookups without enabled providers" + ); + } + + /// Returns `p-1` for the lookup it was sent, then reports nothing left. + struct LookupHttpClient; + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for LookupHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + Ok(PlatformPendingRequest::new(()).with_backend_name(request.backend_name)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn select_until( + &self, + mut pending_requests: Vec, + _deadline: web_time::Instant, + ) -> Result, Report> { + let Some(ready) = pending_requests.pop() else { + return Ok(None); + }; + let backend_name = ready + .backend_name() + .expect("should carry the backend name") + .to_owned(); + let response = edgezero_core::http::response_builder() + .status(StatusCode::OK) + .body(Body::from("p-1")) + .map_err(|_| Report::new(PlatformError::HttpClient))?; + Ok(Some(PlatformSelectResult { + ready: Ok(PlatformResponse::new(response).with_backend_name(backend_name)), + remaining: pending_requests, + failed_backend_name: None, + })) + } + } + + /// Looks up `ids.example.com` and returns the response body as its UID. + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: Method::GET, + url: url::Url::parse("https://ids.example.com/lookup").expect("should parse"), + headers: edgezero_core::http::HeaderMap::new(), + body: Bytes::new(), + context: None, + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }])) + } + } + + struct AllowAll; + + impl RateLimiter for AllowAll { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(false) + } + } + + #[test] + fn collected_results_reach_the_page_load_write() { + let provider_config: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider config"); + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + let mut settings = test_settings(); + settings.ec.ec_store = Some("ec_identity_store".to_owned()); + settings.ec.partners = vec![partner]; + // This build registers no kind, so the app state is built without the + // provider and the test supplies its registry and coordinator. + let router = TrustedServerApp::routes_for_state(&app_state_for_settings(settings.clone())); + let partners = { + let mut with_provider = settings.ec.clone(); + with_provider.identity_providers = vec![provider_config.clone()]; + Arc::new( + PartnerRegistry::from_ec_with_kinds(&with_provider, |_| true) + .expect("should build registry"), + ) + }; + let app_state = AppState { + partner_registry: Arc::clone(&partners), + ..Arc::into_inner(app_state_for_settings(settings.clone())) + .expect("should own the app state") + }; + let mut request = navigation_request("/article"); + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + let mut response = route(&router, request); + let mut state = take_finalize_state(&mut response); + let ec_id = state + .ec_context + .ec_value() + .expect("should generate an EC") + .to_owned(); + state.services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(LookupHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let provider: Box = Box::new(EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse source"), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + })); + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config, provider)], + partners, + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = PlatformLookupTransport::new(&state.services); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }; + let pending = block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: state.ec_context.kv_snapshot(), + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "test-publisher.com", + rate_limiter: &AllowAll, + now: UnixSeconds::new(1_700_000_000), + }, + )); + + let result = crate::apply_edgezero_ec_finalize( + &settings, + Some(&app_state), + &mut state, + Some(super::IdentityLookups { + coordinator: Arc::new(coordinator), + pending, + }), + &mut response, + ); + + result.expect("should finalize"); + assert!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| value.to_str().is_ok_and(|cookie| cookie.contains(&ec_id))), + "should set the EC cookie after the page-load write" + ); + let graph = crate::require_identity_graph(&settings).expect("should open identity graph"); + let snapshot = graph.load_snapshot(&ec_id); + let entry = snapshot.entry_for(&ec_id).expect("should store the row"); + let row = serde_json::to_string(entry).expect("should serialize row"); + assert!( + row.contains("ids.example.com") && row.contains("p-1"), + "should write the provider EID into the row: {row}" + ); + } + + #[test] + fn publisher_request_without_fetch_metadata_navigation_gets_no_permit() { + let router = test_router(); + let mut prefetch = navigation_request("/article"); + prefetch + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for request in [ + browser_request(Method::GET, "/article", "document"), + prefetch, + ] { + let mut response = route(&router, request); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit without a top-level navigation" + ); + } + } + + #[test] + fn integration_proxy_never_generates_ec() { + let router = test_router(); + + let mut response = route( + &router, + navigation_request("/integrations/prebid/bundle.js"), + ); + + let mut state = take_finalize_state(&mut response); + assert!( + !state.ec_context.ec_generated(), + "should not mint an EC on an integration proxy" + ); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit to an integration proxy" + ); + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 3cc13fdfa8..cc8ba854e8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,9 +13,19 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::after_response::{ + after_response_lookups_allowed, commit_after_response_lookups, +}; +use trusted_server_core::ec::coordinator::IdentityCoordinator as _; +use trusted_server_core::ec::coordinator::LookupContext; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::current_timestamp; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -28,6 +38,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; +use trusted_server_identity_providers::partner_registry; mod app; mod backend; @@ -44,8 +55,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, + AfterResponseLookups, AppState, EcFinalizeState, IdentityLookups, NavigationLookups, + RuntimeStoreConfig, TrustedServerApp, load_settings_from_config_store, take_navigation_lookups, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -491,6 +502,8 @@ fn edgezero_main( core_req.extensions_mut().insert(device_signals); } core_req.extensions_mut().insert(client_info); + // Drop lookups a previous request on this sandbox left behind. + drop(take_navigation_lookups()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -517,6 +530,12 @@ fn edgezero_main( // Pop response extensions before the Fastly conversion, which drops them. let ec_state = response.extensions_mut().remove::(); + // Identity lookups dispatched on a publisher navigation are collected in + // EC finalization, after the router's executor has returned. + let NavigationLookups { + page: identity, + after_response, + } = take_navigation_lookups(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); @@ -544,7 +563,13 @@ fn edgezero_main( if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + settings, + app_state.as_deref(), + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -552,6 +577,13 @@ fn edgezero_main( counters.as_ref(), ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + run_after_response_lookups( + settings, + app_state.as_deref(), + &partner_registry, + &mut ec_state, + after_response, + ); return; } Err(e) => { @@ -563,7 +595,13 @@ fn edgezero_main( } else { match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + None, + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -575,6 +613,13 @@ fn edgezero_main( &partner_registry, &ec_state, ); + run_after_response_lookups( + &settings, + None, + &partner_registry, + &mut ec_state, + after_response, + ); return; } Err(e) => { @@ -636,12 +681,39 @@ fn apply_entry_point_finalize_headers( apply_finalize_headers(settings, geo_info.as_ref(), response); } +/// Runs EC finalization for a routed response. +/// +/// Collects `identity` lookups first. This must run outside the router's +/// `block_on`: collection drives its own executor, and nesting executors +/// panics. fn apply_edgezero_ec_finalize( settings: &Settings, + state: Option<&AppState>, ec_state: &mut EcFinalizeState, + identity: Option, response: &mut HttpResponse, -) -> Result> { - let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; +) -> Result, Report> { + // A redirect skips the write, so do not wait for lookups it would discard. + let identity = identity.filter(|_| { + let redirect = NavigationPermit::is_redirect(response.status()); + if redirect { + log::debug!("Identity lookups dropped: redirect response skips the write"); + } + !redirect + }); + if let Some(IdentityLookups { + coordinator, + pending, + }) = identity + { + let transport = PlatformLookupTransport::new(&ec_state.services); + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + ec_state.ec_context.set_lookup_results(results); + } + let partner_registry = match state { + Some(state) => state.partner_registry(), + None => Arc::new(partner_registry(&settings.ec)?), + }; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { @@ -686,6 +758,74 @@ fn run_edgezero_pull_sync_after_send( ); } +/// Runs after-response identity lookups and applies their follow-up write. +/// +/// Called once the response has been sent, so provider latency never delays +/// the page. Skipped unless finalize confirmed the navigation (it issues the +/// follow-up permit) and the EC is still the one the lookups were prepared +/// for. +fn run_after_response_lookups( + settings: &Settings, + state: Option<&AppState>, + partner_registry: &Arc, + ec_state: &mut EcFinalizeState, + lookups: Option, +) { + let Some(lookups) = lookups else { + return; + }; + let Some(permit) = ec_state.ec_context.take_after_response_permit() else { + log::debug!( + "After-response lookups skipped: no permit (consent, redirect or failed page-load write)" + ); + return; + }; + if ec_state.ec_context.ec_value() != Some(lookups.ec_id.as_str()) { + log::debug!("After-response lookups skipped: EC changed during the request"); + return; + } + let Some(coordinator) = state.and_then(|state| state.after_response_identity.as_ref()) else { + return; + }; + let Some(graph) = maybe_identity_graph(settings) else { + return; + }; + let snapshot = graph.load_snapshot(&lookups.ec_id); + if !after_response_lookups_allowed( + &snapshot, + &lookups.ec_id, + ec_state.ec_context.ec_generated(), + ) { + log::debug!("After-response lookups skipped: new EC or no live row"); + return; + } + let transport = PlatformLookupTransport::new(&ec_state.services); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let now = current_timestamp(); + let pending = futures::executor::block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &lookups.ec_id, + snapshot: &snapshot, + consent: ec_state.ec_context.consent(), + client: ClientSignals { + ip: lookups.client_ip, + user_agent: lookups.user_agent.as_deref(), + }, + publisher_domain: &settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + )); + if pending.is_empty() { + return; + } + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + let outcome = + commit_after_response_lookups(&graph, permit, partner_registry, &lookups.ec_id, &results); + log::info!("After-response lookups: write outcome={outcome}"); +} + fn prepare_pull_sync_after_send( context: Option, graph_factory: F, diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index 6a8842d180..6b074f385c 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -16,6 +16,9 @@ use super::{ // Bound payload retained across all partial bodies, as well as each response. const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; +/// Longest host wait an expired snapshot spends surfacing one arrived body. +const SNAPSHOT_SURFACE_TIMEOUT_MS: u32 = 1; + // Keep partially buffered responses alongside requests still awaiting headers so // an early header with a slow body cannot hold up a completed sibling response. struct DeadlinePending { @@ -215,7 +218,9 @@ fn body_is_ready(body: &BodyHandle) -> Result> { /// Dropping response bodies closes their host handles. Fastly has no pending /// request cancellation ABI: unfinished header requests are abandoned and /// never waited on past the auction deadline. If collection starts after the -/// deadline, one nonblocking snapshot preserves already available responses. +/// deadline, one snapshot preserves already available responses; it may spend +/// at most [`SNAPSHOT_SURFACE_TIMEOUT_MS`] per pending body letting the host +/// surface bytes that already arrived. /// /// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that /// capacity, the largest partial response fails with an attributed HTTP client @@ -260,11 +265,22 @@ pub(super) fn select_until( // Take one nonblocking snapshot. Fully buffered results are cached; // responses that would require a wait are abandoned, never retried. for index in 0..pending.len() { + let mut surfaced = false; loop { let previous = pending[index].buffered_bytes(); + let was_headers = matches!(pending[index].state, Some(PendingState::Headers(_))); let result = pending[index].poll(); - let progressed = - matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; + let progressed = matches!(result, Ok(None)) + && snapshot_progressed(was_headers, previous, &pending[index]); + if matches!(result, Ok(None)) + && !progressed + && !surfaced + && let Some(PendingState::Body { body, .. }) = &pending[index].state + { + surfaced = true; + surface_arrived_body(body); + continue; + } match result { Ok(None) if progressed => {} Ok(None) => { @@ -337,6 +353,44 @@ pub(super) fn select_until( } } +/// Lets the host surface body bytes that have already arrived. +/// +/// A body whose bytes the host holds can still report not ready until the +/// host services async I/O once, so an expired snapshot would abandon a +/// response that fully arrived before the deadline. One select capped at +/// [`SNAPSHOT_SURFACE_TIMEOUT_MS`] runs at most once per pending body; a +/// failure leaves the body not ready. +fn surface_arrived_body(body: &BodyHandle) { + let handles = [ + // SAFETY: the live owned body retains the handle while select borrows it. + unsafe { body.as_u32() }, + ]; + let mut ready_index = 0; + // SAFETY: handles is a live slice, and ready_index is writable. The + // timeout is positive: the host interprets zero as an unbounded wait. + let status = unsafe { + fastly_sys::fastly_async_io::select( + handles.as_ptr(), + handles.len(), + SNAPSHOT_SURFACE_TIMEOUT_MS, + &mut ready_index, + ) + }; + if let Err(status) = status.result() { + log::debug!("Expired snapshot: body surface select failed: {status:?}"); + } +} + +/// Whether a nonblocking poll that returned no response advanced `item`. +/// +/// Arriving headers move a request into [`PendingState::Body`] without +/// reading any bytes; that is progress, or a response that fully arrived +/// before an expired collection would be abandoned unread. +fn snapshot_progressed(was_headers: bool, previous_bytes: usize, item: &DeadlinePending) -> bool { + let entered_body = was_headers && matches!(item.state, Some(PendingState::Body { .. })); + entered_body || item.buffered_bytes() > previous_bytes +} + fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { return; @@ -543,6 +597,20 @@ mod tests { ); } + #[test] + fn snapshot_counts_arriving_headers_as_progress() { + let pending = DeadlinePending::from_platform(buffered_pending(Vec::new(), "example")) + .expect("should resume a buffered pending request"); + assert!( + super::snapshot_progressed(true, 0, &pending), + "should keep polling a request whose headers just arrived" + ); + assert!( + !super::snapshot_progressed(false, 0, &pending), + "should stop polling a body that read nothing" + ); + } + #[test] fn expired_deadline_preserves_an_already_ready_body() { let pending = buffered_pending(b"ready".to_vec(), "example"); diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9edce69fa3..a9f4e031d2 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -133,6 +133,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Spin")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -653,7 +654,7 @@ fn build_router(state: &Arc) -> RouterService { let s = Arc::clone(&s); async move { let req = ctx.into_request(); - let result = PartnerRegistry::from_config(&s.settings.ec.partners) + let result = PartnerRegistry::from_ec(&s.settings.ec) .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); Ok::(result.unwrap_or_else(|e| http_error(&e))) } @@ -817,15 +818,12 @@ fn build_router(state: &Arc) -> RouterService { let result = if method == Method::GET && path.starts_with("/static/tsjs=") { handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 086515fa20..1f5a8932b8 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -14,9 +14,10 @@ use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; -use crate::ec::eids::{resolve_partner_ids, to_eids}; +use crate::ec::current_timestamp; +use crate::ec::eid_types::UnixSeconds; use crate::ec::kv::KvIdentityGraph; -use crate::ec::kv_types::MAX_UID_LENGTH; +use crate::ec::kv_types::{MAX_STORED_UID_LENGTH, MAX_UID_LENGTH}; use crate::ec::prebid_eids::parse_prebid_eids_cookie; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; @@ -535,8 +536,13 @@ pub(crate) fn resolve_auction_eids( return Some(Vec::new()); } - let resolved = resolve_partner_ids(registry, entry); - Some(to_eids(&resolved)) + let now = UnixSeconds::new(current_timestamp()); + Some( + entry + .live_eids(registry, ec_context.consent(), now) + .map(Eid::from) + .collect(), + ) } pub(crate) fn resolve_client_auction_eids( @@ -600,7 +606,7 @@ fn parse_client_auction_eids(raw: Option<&JsonValue>) -> Option> { continue; } - eids.push(Eid { source, uids }); + eids.push(Eid::new(source, uids)); } if eids.is_empty() { None } else { Some(eids) } @@ -635,8 +641,9 @@ pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { - let mut merged = Vec::new(); + let mut merged: Vec = Vec::new(); + // Server-resolved entries come first so their fields win on duplicates. for eid in resolved_eids .into_iter() .flatten() @@ -645,50 +652,67 @@ pub(crate) fn merge_auction_eids( if eid.source.is_empty() { continue; } - - let source_index = match merged - .iter() - .position(|existing: &Eid| existing.source == eid.source) - { - Some(index) => index, - None => { - merged.push(Eid { - source: eid.source.clone(), - uids: Vec::new(), - }); - merged.len() - 1 - } - }; - - for uid in eid.uids { - if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + let Eid { + source, + inserter, + matcher, + mm, + ext, + uids, + } = eid; + let mut kept = Vec::with_capacity(uids.len()); + for uid in uids { + // Stored provider UIDs may reach the stored limit; client UIDs + // were already capped at `MAX_UID_LENGTH` when parsed. + if uid.id.trim().is_empty() || uid.id.len() > MAX_STORED_UID_LENGTH { continue; } - - if let Some(existing_uid) = merged[source_index] - .uids + // A duplicate keeps the earlier UID's fields but fills its + // missing `atype` and `ext` from the later one. + let duplicate = merged .iter_mut() - .find(|existing| existing.id == uid.id) - { - if existing_uid.atype.is_none() { - existing_uid.atype = uid.atype; - } - if existing_uid.ext.is_none() { - existing_uid.ext = uid.ext; + .filter(|existing| existing.source == source) + .flat_map(|existing| existing.uids.iter_mut()) + .chain(kept.iter_mut()) + .find(|existing: &&mut Uid| existing.id == uid.id); + match duplicate { + Some(existing) => { + if existing.atype.is_none() { + existing.atype = uid.atype; + } + if existing.ext.is_none() { + existing.ext = uid.ext; + } } - } else { - merged[source_index].uids.push(uid); + None => kept.push(uid), } } + if kept.is_empty() { + continue; + } + // Entries are distinct when any EID-level field differs, including + // `ext` (compared by JSON value), so no entry's fields are lost. + let same_entry = merged.iter_mut().find(|existing| { + existing.source == source + && existing.inserter == inserter + && existing.matcher == matcher + && existing.mm == mm + && existing.ext == ext + }); + match same_entry { + Some(existing) => existing.uids.extend(kept), + None => merged.push(Eid { + source, + inserter, + matcher, + mm, + ext, + uids: kept, + }), + } } - merged.retain(|eid| !eid.uids.is_empty()); - - if merged.is_empty() { - None - } else { - Some(merged) - } + (!merged.is_empty()).then_some(merged) } #[cfg(test)] @@ -782,10 +806,9 @@ mod tests { #[tokio::test] async fn auction_endpoint_snapshot_is_reused_by_response_finalization() { // `/auction` loads the identity-graph row to resolve server-side EIDs. - // Finalization runs afterwards on the same EC context and ingests - // `ts-eids`/`sharedId` updates. Both must be served by a single billable - // read: before the snapshot was shared, finalization saw `NotRead` and - // paid for a second lookup. + // Finalization runs afterwards on the same EC context. It must not pay + // for a second lookup, and it never writes KV: only the page load's + // navigation may write (spec D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -869,21 +892,17 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "the sharedId update must still be ingested from the shared snapshot" + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[tokio::test] - async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { - // Regression test for #1184: `/auction` sends every EID it has in the - // request body, but response finalization used to ingest identity - // graph updates only from the `ts-eids` cookie — which the browser - // caps in size and may not have sent at all. `handle_auction` must - // hand its parsed body EIDs to `ec_context` so finalization ingests - // them regardless of what the cookie carried. + async fn auction_body_eids_reach_context_but_never_kv() { + // `handle_auction` hands its parsed body EIDs to `ec_context`, but body + // EIDs go to bidders on that request only: finalization of an + // `/auction` request never writes them to KV (spec §5.1, D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -962,10 +981,9 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the body's EID must be ingested into KV without a ts-eids cookie" + assert!( + stored.ids.is_empty(), + "the body's EIDs must not be persisted from an auction request" ); } @@ -2058,33 +2076,152 @@ mod tests { ); } + #[test] + fn merge_auction_eids_keeps_distinct_eid_level_entries_per_source() { + let resolved = Some(vec![Eid { + inserter: Some("a.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![ + Eid { + inserter: Some("b.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }, + Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(2), + ext: None, + }], + ), + ]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!( + merged.len(), + 2, + "should keep entries with different inserters" + ); + assert_eq!( + merged[0].uids.len(), + 1, + "should drop duplicate u1 from client" + ); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should keep server fields for duplicate" + ); + assert_eq!( + merged[1].uids[0].id, "u2", + "should keep distinct client uid" + ); + } + + #[test] + fn merge_auction_eids_keeps_entries_with_different_ext_separate() { + let resolved = Some(vec![Eid { + ext: Some(json!({ "k": "server" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![Eid { + ext: Some(json!({ "k": "client" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!(merged.len(), 2, "should keep entries with different ext"); + assert_eq!( + merged[0].ext, + Some(json!({ "k": "server" })), + "should keep server entry ext" + ); + assert_eq!( + merged[1].ext, + Some(json!({ "k": "client" })), + "should keep client entry ext" + ); + assert_eq!(merged[1].uids[0].id, "u2", "should keep client uid"); + } + + #[test] + fn merge_auction_eids_keeps_a_stored_provider_uid_past_the_ingest_limit() { + let long = "p".repeat(MAX_UID_LENGTH + 100); + let resolved_eids = Some(vec![Eid::new( + "ids.example.com".to_string(), + vec![Uid { + id: long.clone(), + atype: Some(1), + ext: None, + }], + )]); + + let merged = merge_auction_eids(None, resolved_eids).expect("should merge EIDs"); + + assert_eq!( + merged[0].uids[0].id, long, + "should send a stored UID within the stored limit" + ); + } + #[test] fn merge_auction_eids_deduplicates_client_and_resolved_ids() { - let client_eids = Some(vec![Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }]); + )]); let resolved_eids = Some(vec![ - Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_string(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_string(), + vec![Uid { id: "LR_xyz".to_string(), atype: Some(3), ext: None, }], - }, + ), ]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2098,22 +2235,22 @@ mod tests { #[test] fn merge_auction_eids_preserves_multiple_uids_per_source() { - let client_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_client".to_string(), atype: None, ext: None, }], - }]); - let resolved_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_server".to_string(), atype: Some(3), ext: None, }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2125,22 +2262,22 @@ mod tests { #[test] fn merge_auction_eids_prefers_server_resolved_metadata_on_conflict() { - let client_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(1), ext: Some(json!({ "provider": "client" })), }], - }]); - let resolved_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(3), ext: Some(json!({ "provider": "server" })), }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2158,6 +2295,40 @@ mod tests { ); } + #[test] + fn merge_auction_eids_fills_missing_atype_and_ext_from_duplicate() { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: Some(1), + ext: Some(json!({ "provider": "client" })), + }], + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: None, + ext: None, + }], + )]); + + let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); + + assert_eq!(merged[0].uids.len(), 1, "should deduplicate duplicate uid"); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should fill missing atype from the duplicate" + ); + assert_eq!( + merged[0].uids[0].ext, + Some(json!({ "provider": "client" })), + "should fill missing ext from the duplicate" + ); + } + #[test] fn auction_rejects_oversized_body() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index b684c80dde..15076adb6a 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -850,14 +850,14 @@ mod tests { #[test] fn response_includes_eid_headers_when_eids_present() { let mut request = make_auction_request(); - request.user.eids = Some(vec![Eid { - source: "ssp.com".to_owned(), - uids: vec![Uid { + request.user.eids = Some(vec![Eid::new( + "ssp.com".to_owned(), + vec![Uid { id: "uid-1".to_owned(), atype: Some(3), ext: None, }], - }]); + )]); let settings = make_settings(); let result = make_empty_result(); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..c839a582eb 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -92,14 +92,14 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { raw_ac_string: Some("fictional-ac".to_string()), ..Default::default() }), - eids: Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { + eids: Some(vec![Eid::new( + "identity.example".to_string(), + vec![Uid { id: "fictional-uid".to_string(), atype: Some(1), ext: None, }], - }]), + )]), }, device: Some(DeviceInfo { user_agent: Some("Fictional Browser".to_string()), diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 059d500092..cb98956130 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -13,6 +13,7 @@ use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; +use crate::ec::provider::validate_enabled_configs; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ @@ -32,7 +33,8 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; +use crate::redacted::Redacted; +use crate::settings::{AssetOriginAuth, Ec, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] @@ -167,6 +169,16 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { ], true, ), + field( + vec![ + object("ec"), + optional_object("identity_providers"), + SecretPathSegment::ArrayEach, + optional_object("secrets"), + object("token"), + ], + true, + ), field( vec![ object("handlers"), @@ -268,6 +280,7 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Ec { + let mut ec = ec.clone(); + for (index, partner) in ec.partners.iter_mut().enumerate() { + if partner.api_token.is_some() { + partner.api_token = Some(Redacted::new(format!("{index:0>64}"))); + } + if partner.ts_pull_token.is_some() { + partner.ts_pull_token = Some(Redacted::new(format!("{index:1>64}"))); + } + } + ec +} + +/// Checks the identity-provider rules core owns. +/// +/// Unknown kinds and capability checks need the provider crate; the adapter +/// builds the coordinators at startup and fails to start on either. +fn validate_identity_providers(ec: &Ec) -> Result<(), Report> { + if ec.identity_providers.is_empty() { + return Ok(()); + } + let partners = PartnerRegistry::from_ec(ec)?; + validate_enabled_configs(&ec.identity_providers, &partners).map_err(|report| { + let detail = report + .frames() + .filter_map(|frame| frame.downcast_ref::()) + .map(String::as_str) + .collect::>() + .join("; "); + report.change_context(TrustedServerError::Configuration { + message: format!("ec.identity_providers: {detail}"), + }) + }) +} + fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { let Some(raw_config) = settings.integrations.get(JS_ASSET_PROXY_INTEGRATION_ID) else { return Ok(()); @@ -447,6 +501,15 @@ fn validate_secret_key_references(settings: &Settings) -> Result<(), Report Settings { + let mut settings = valid_settings(); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ]; + let mut config = serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + }); + if let (Some(base), serde_json::Value::Object(overrides)) = + (config.as_object_mut(), provider) + { + base.extend(overrides); + } + settings.ec.identity_providers = + vec![serde_json::from_value(config).expect("should build provider config")]; + settings + } + + #[test] + fn runtime_validation_accepts_valid_identity_provider() { + let settings = settings_with_identity_provider(serde_json::json!({})); + + validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); + } + + #[test] + fn deploy_validation_accepts_provider_with_unresolved_partner_token() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + settings.ec.partners[0].api_token = + Some(Redacted::new("example_partner_api_token".to_owned())); + + validate_settings_for_deploy(&settings) + .expect("should not length-check a partner token secret key name"); + } + + #[test] + fn runtime_validation_rejects_provider_endpoint_outside_allowlist() { + let settings = settings_with_identity_provider(serde_json::json!({ + "endpoint": "https://other.example.com/v1/lookup" + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject endpoint outside allowlist"); + + assert!( + err.to_string().contains("allowed_domains"), + "should explain the allowlist rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_provider_source_without_partner() { + let settings = settings_with_identity_provider(serde_json::json!({ + "sources": ["unknown.example.com"] + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject source without partner"); + + assert!( + err.to_string().contains("[[ec.partners]]"), + "should explain the partner rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_source_claimed_by_two_providers() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + let mut second = settings.ec.identity_providers[0].clone(); + second.id = "second-provider".to_owned(); + settings.ec.identity_providers.push(second); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject double-claimed source"); + + assert!( + err.to_string() + .contains("claimed by another enabled provider"), + "should explain the claim rule: {err:?}" + ); + } + + #[test] + fn deploy_validation_rejects_empty_provider_token_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": " " } + })); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject empty provider token reference"); + + assert!( + err.to_string() + .contains("ec.identity_providers[0].secrets.token"), + "should identify the provider token reference: {err:?}" + ); + } + + #[test] + fn app_config_keeps_provider_token_key_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": "example_provider_token" } + })); + + let app_config = + TrustedServerAppConfig::new(settings).expect("should validate provider token key name"); + + let serialized = serde_json::to_string(&app_config).expect("should serialize app config"); + assert!( + serialized.contains("example_provider_token"), + "should keep the provider token key reference" + ); + } + #[test] fn trace_config_requires_enabled_on_both_validation_paths() { for raw in [ diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 34a4143cf2..dd7545198b 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -205,6 +205,7 @@ mod tests { "s3-session-key" => "resolved-session-token", "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", + "provider-token-key" => "resolved-provider-token", "trusted-client-ip-key" => "resolved-trusted-client-ip-secret-32-bytes", "trusted-forwarder-key" => "resolved-trusted-forwarder-secret-32-bytes", _ => key, @@ -332,6 +333,46 @@ mod tests { ); } + #[test] + fn resolves_identity_provider_token_from_the_mapped_default_store() { + let mut original = test_settings(); + original.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ); + original.ec.identity_providers.push( + serde_json::from_value(serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "secrets": { "token": "provider-token-key" }, + })) + .expect("should build provider config"), + ); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve the provider token"); + + assert_eq!( + reconstructed.ec.identity_providers[0] + .secrets + .token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-provider-token"), + "should resolve the provider token key" + ); + } + #[test] fn resolves_all_static_credentials_from_the_mapped_default_store() { let mut original = test_settings(); diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 76aa6ad859..5d232df8a9 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -41,6 +41,7 @@ pub use types::{ ConsentContext, ConsentSource, PrivacyFlag, RawConsentSignals, TcfConsent, UsPrivacy, }; +use std::num::NonZeroU16; use std::sync::atomic::{AtomicBool, Ordering}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -287,6 +288,23 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { }) } +/// Whether the effective TCF string grants vendor consent to `vendor_id`. +/// +/// Returns `false` when no TCF string applies. +#[must_use] +pub fn has_tcf_vendor_consent(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + effective_tcf(ctx).is_some_and(|tcf| tcf.has_vendor_consent(vendor_id.get())) +} + +/// Whether TCF rules apply and withhold vendor consent from `vendor_id`. +/// +/// Returns `false` when no TCF string applies and GDPR does not, so a TCF +/// vendor requirement only gates traffic that TCF governs. +#[must_use] +pub fn tcf_vendor_consent_denied(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + (ctx.gdpr_applies || effective_tcf(ctx).is_some()) && !has_tcf_vendor_consent(ctx, vendor_id) +} + /// Returns whether a server-side auction may be dispatched for this request. /// /// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present @@ -685,6 +703,7 @@ mod tests { ConsentPipelineInput, allows_ec_creation, allows_eid_persistence, apply_expiration_check, apply_tcf_conflict_resolution, build_consent_context, build_context_from_signals, consent_allows_server_side_auction, gate_eids_by_consent, has_explicit_ec_withdrawal, + has_tcf_vendor_consent, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -692,6 +711,7 @@ mod tests { }; use crate::consent_config::{ConflictMode, ConsentConfig, ConsentMode}; use crate::cookies::parse_cookies_to_jar; + use std::num::NonZeroU16; fn build_request() -> Request { Request::builder() @@ -755,6 +775,32 @@ mod tests { } } + #[test] + fn tcf_vendor_consent_reads_the_effective_tcf_string() { + let mut tcf = TcfBuilder::new().with_storage(true).build(); + tcf.vendor_consents = vec![9999]; + let context = ConsentContext { + gdpr_applies: true, + tcf: Some(tcf), + ..ConsentContext::default() + }; + let granted = NonZeroU16::new(9999).expect("should be non-zero"); + let other = NonZeroU16::new(1234).expect("should be non-zero"); + + assert!( + has_tcf_vendor_consent(&context, granted), + "should grant consented vendor" + ); + assert!( + !has_tcf_vendor_consent(&context, other), + "should deny other vendor" + ); + assert!( + !has_tcf_vendor_consent(&ConsentContext::default(), granted), + "should deny when no TCF string applies" + ); + } + fn make_tcf(last_updated_ds: u64, allows_eids: bool) -> TcfConsent { TcfBuilder::new() .with_storage(true) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e21fab702c..7b8493beb2 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -26,12 +26,14 @@ use serde_json::Value as JsonValue; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; +use crate::consent::ConsentContext; use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; use crate::openrtb::Eid; -use super::eids::{resolve_partner_ids, to_eids}; +use super::current_timestamp; +use super::eid_types::UnixSeconds; use super::generation::is_valid_ec_id; use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; @@ -251,8 +253,8 @@ struct AuctionEidsView { struct SkippedPartnerId { /// Partner namespace key in the entry's `ids` map. source_domain: String, - /// Why the auction resolution skips it: `empty_uid`, `not_in_registry`, - /// or `bidstream_disabled`. + /// Why the auction resolution skips it: `empty_uid`, `expired` (provider + /// record with no live UID), `not_in_registry`, or `bidstream_disabled`. reason: &'static str, } @@ -394,6 +396,7 @@ fn build_lookup_response( match serde_json::from_slice::(&lookup.body) { Ok(mut entry_json) => { add_iso_timestamp_companions(&mut entry_json); + redact_refresh_state(&mut entry_json); payload.entry = Some(entry_json); match serde_json::from_slice::(&lookup.body) { @@ -470,6 +473,30 @@ fn add_iso_timestamp_companions(entry_json: &mut JsonValue) { } } +/// Placeholder served in place of a provider's private refresh state. +const REDACTED: &str = "[redacted]"; + +/// Replaces every provider refresh `state` in the entry with [`REDACTED`]. +/// +/// Refresh state can hold a provider credential (for example a refresh +/// token), so the admin view shows only that it exists and when it is due. +fn redact_refresh_state(value: &mut JsonValue) { + match value { + JsonValue::Object(object) => { + if let Some(state) = object + .get_mut("refresh") + .and_then(JsonValue::as_object_mut) + .and_then(|refresh| refresh.get_mut("state")) + { + *state = JsonValue::String(REDACTED.to_owned()); + } + object.values_mut().for_each(redact_refresh_state); + } + JsonValue::Array(items) => items.iter_mut().for_each(redact_refresh_state), + _ => {} + } +} + /// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). /// /// Returns `None` for values outside the representable date range. @@ -479,22 +506,31 @@ fn iso_timestamp(unix_seconds: u64) -> Option { .map(|datetime| datetime.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) } -/// Derives the auction EID view for a valid entry, mirroring the filters in -/// [`resolve_partner_ids`] and reporting why each stored ID was skipped. +/// Derives the auction EID view for a valid entry through the same +/// [`KvEntry::live_eids`] resolution the live auction uses, reporting why each +/// stored ID was skipped. +/// +/// Vendor consent belongs to the visitor's request, which this diagnostic +/// does not have, so no source is withheld for it here. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { - let resolved = resolve_partner_ids(registry, entry); - let eids = to_eids(&resolved); + let now = UnixSeconds::new(current_timestamp()); + let eids = entry + .live_eids(registry, &ConsentContext::default(), now) + .map(Eid::from) + .collect(); let mut skipped = Vec::new(); for (source_domain, partner_uid) in &entry.ids { - let reason = if partner_uid.uid.is_empty() { - "empty_uid" - } else { + let reason = if partner_uid.has_live_uid(now) { match registry.get(source_domain) { None => "not_in_registry", Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", Some(_) => continue, } + } else if partner_uid.is_provider_written() { + "expired" + } else { + "empty_uid" }; skipped.push(SkippedPartnerId { source_domain: source_domain.clone(), @@ -682,9 +718,10 @@ mod tests { use crate::ec::kv::TombstoneOutcome; use super::*; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -778,15 +815,11 @@ mod tests { let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); entry.ids.insert( "disabled.example".to_owned(), - KvPartnerId { - uid: "uid-disabled".to_owned(), - }, + KvPartnerId::single("uid-disabled".to_owned()), ); entry.ids.insert( "unknown.example".to_owned(), - KvPartnerId { - uid: "uid-unknown".to_owned(), - }, + KvPartnerId::single("uid-unknown".to_owned()), ); entry } @@ -1416,7 +1449,7 @@ mod tests { #[test] fn eids_lookup_reports_configured_source_without_valid_uid() { - let oversized_uid = "x".repeat(513); + let oversized_uid = "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1); let cookie = eids_cookie_for(&serde_json::json!([{ "source": "bidstream.example", "uids": [ @@ -1533,4 +1566,81 @@ mod tests { assert_eq!(matched[0]["source_domain"], "sharedid.org"); assert_eq!(matched[0]["uid"], "shared-uid-123"); } + + #[test] + fn auction_view_matches_live_auction_for_provider_records() { + let registry = PartnerRegistry::from_config(&[ + make_test_partner("expired.example", true), + make_test_partner("live.example", true), + ]) + .expect("should build registry"); + let provider_record = |id: &str, expiry: Expiry| { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(0), + ) + .expect("should build record") + }; + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert( + "expired.example".to_owned(), + provider_record("old", Expiry::At(UnixSeconds::new(1))), + ); + entry.ids.insert( + "live.example".to_owned(), + provider_record("new", Expiry::Never), + ); + + let view = build_auction_view(®istry, &entry); + + let view_json = serde_json::to_value(&view.eids).expect("should serialize view eids"); + let live: Vec = entry + .live_eids( + ®istry, + &ConsentContext::default(), + UnixSeconds::new(current_timestamp()), + ) + .map(Eid::from) + .collect(); + assert_eq!( + view_json, + serde_json::to_value(&live).expect("should serialize live eids"), + "should show exactly what the live auction resolves" + ); + assert_eq!(view.eids.len(), 1, "should omit the expired provider UID"); + assert_eq!(view.eids[0].uids[0].id, "new", "should keep the live UID"); + assert_eq!(view.skipped.len(), 1, "should report the expired record"); + assert_eq!(view.skipped[0].source_domain, "expired.example"); + assert_eq!(view.skipped[0].reason, "expired", "should report expiry"); + } + + #[test] + fn redacts_provider_refresh_state() { + let mut entry = serde_json::json!({ + "ids": {"token.example": {"eids": [{"source": "token.example", "uids": [ + {"id": "token", "refresh": {"state": "secret-refresh", "from": 1_000}} + ]}]}} + }); + + redact_refresh_state(&mut entry); + + let refresh = &entry["ids"]["token.example"]["eids"][0]["uids"][0]["refresh"]; + assert_eq!(refresh["state"], REDACTED, "should hide the refresh state"); + assert_eq!( + refresh["from"], 1_000, + "should keep when the refresh is due" + ); + assert_eq!( + entry["ids"]["token.example"]["eids"][0]["uids"][0]["id"], "token", + "should keep the UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs new file mode 100644 index 0000000000..e14de5eb18 --- /dev/null +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -0,0 +1,522 @@ +//! The follow-up write of after-response identity lookups. +//! +//! Providers configured with `collect = "after_response"` are dispatched only +//! after the navigation's response has been sent, so their latency never +//! delays the page. Their validated results reach the row in one follow-up +//! update, guarded by the row's current generation, and are served from the +//! next page load. Browser ingest never runs here: it belongs to the +//! navigation's own write. + +use super::coordinator::results::LookupResults; +use super::eid_types::UnixSeconds; +use super::kv::KvIdentityGraph; +use super::page_load::{AfterResponseWrite, RowChange}; +use super::registry::PartnerRegistry; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, current_timestamp, log_id}; + +/// What the follow-up write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum AfterResponseOutcome { + /// The row was updated with the after-response results. + #[display("written")] + Written, + /// The results would not change the row. + #[display("unchanged")] + Unchanged, + /// The row is missing, withdrawn or unreadable, so nothing was written. + #[display("no_row")] + NoRow, + /// The row changed since it was read; the results are dropped. + #[display("conflict")] + Conflict, + /// The store rejected the write. + #[display("failed")] + Failed, +} + +/// Whether after-response lookups may be dispatched for `ec_id`. +/// +/// They run only for an EC that existed before this navigation and whose row, +/// read after the response was sent, is live and consented. An EC minted on +/// this navigation is skipped: no partner can know it yet, and its new row +/// may not be readable so soon. Its lookups run from the next navigation. +#[must_use] +pub fn after_response_lookups_allowed( + snapshot: &EcKvSnapshot, + ec_id: &str, + ec_generated: bool, +) -> bool { + !ec_generated + && snapshot + .entry_for(ec_id) + .is_some_and(|entry| entry.consent.ok) +} + +/// Applies after-response lookup results to the row of `ec_id`. +/// +/// Re-reads the row just before writing, plans an update from `lookups` +/// alone against it, and writes it once, guarded by that read's generation. +/// Reading here rather than before dispatch keeps the window in which another +/// write can conflict to the gap between this read and the write. The +/// `permit` is consumed whether or not a write is made. +/// +/// # Performance +/// +/// One KV read and at most one conditional KV write. Runs after the response +/// has been sent, so it adds no page latency. +#[must_use] +pub fn commit_after_response_lookups( + graph: &KvIdentityGraph, + permit: AfterResponseWrite, + registry: &PartnerRegistry, + ec_id: &str, + lookups: &LookupResults, +) -> AfterResponseOutcome { + if lookups.is_empty() { + return AfterResponseOutcome::Unchanged; + } + let snapshot = &graph.load_snapshot(ec_id); + if !matches!(snapshot, EcKvSnapshot::Present { .. }) { + log::info!( + "After-response lookups: no row for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + } + let change = plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry: None, + browser_ingest: &[], + lookups, + registry, + now: UnixSeconds::new(current_timestamp()), + }); + let Some(change) = change else { + // A withdrawn row also plans no change. + return match snapshot.entry_for(ec_id) { + Some(entry) if entry.consent.ok => AfterResponseOutcome::Unchanged, + _ => AfterResponseOutcome::NoRow, + }; + }; + let RowChange::Update { entry, expected } = change else { + log::warn!( + "After-response lookups: unexpected row change for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + }; + match graph.commit_after_response(permit, ec_id, &entry, expected) { + Ok(()) => AfterResponseOutcome::Written, + Err(report) => { + log::warn!( + "After-response write failed for '{}': {}", + log_id(ec_id), + report.current_context() + ); + match report.current_context() { + super::page_load::KvWriteError::Conflict => AfterResponseOutcome::Conflict, + _ => AfterResponseOutcome::Failed, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::TombstoneOutcome; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use crate::ec::page_load::{Generation, KvWriteError}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn registry() -> PartnerRegistry { + let partners: Vec = ["browser.example.com", "ids.example.com"] + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, + "source_domain": source, + "bidstream_enabled": true, + })) + .expect("should build partner") + }) + .collect(); + let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + assert!( + registry.mark_provider_owned("ids.example.com", None), + "should mark the provider source as owned" + ); + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn found(ec_id: &str, id: &str) -> LookupResults { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + LookupResults::new( + ec_id, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("ids.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + ) + } + + fn stored_uid(graph: &KvIdentityGraph, source: &str) -> Option { + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .and_then(|entry| entry.ids.get(source).map(|record| record.uid.clone())) + } + + #[test] + fn writes_after_response_results_to_an_existing_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "browser.example.com".to_owned(), + KvPartnerId::single("b-1".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-1"), + "should keep the browser UID written by the page load" + ); + } + + #[test] + fn writes_against_the_row_as_read_just_before_the_write() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-reread"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read_before_dispatch = graph.load_snapshot(EC_ID); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + assert_ne!( + graph.load_snapshot(EC_ID), + read_before_dispatch, + "should change the row after the dispatch read" + ); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should not conflict with a write made while the lookups ran" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + } + + #[test] + fn follow_up_write_conflicts_when_the_row_changed_after_its_read() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-conflict"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + let report = result.expect_err("should refuse a stale generation"); + assert!( + matches!(report.current_context(), KvWriteError::Conflict), + "should report a conflict" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + } + + #[test] + fn follow_up_write_never_revives_a_row_withdrawn_after_its_read() { + // Arrange: scenario 23, the row is tombstoned between read and CAS. + let graph = KvIdentityGraph::in_memory("after-response-withdrawn"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + assert_eq!( + graph + .write_withdrawal_tombstone(EC_ID, drop) + .expect("should withdraw the row"), + TombstoneOutcome::Written, + "should tombstone the row" + ); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + assert!(result.is_err(), "should refuse to overwrite the tombstone"); + assert!( + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .is_none_or(|entry| !entry.consent.ok), + "should leave the row withdrawn" + ); + } + + #[test] + fn replaces_a_browser_fallback_for_an_owned_source() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("from-browser".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should replace the browser fallback with the provider UID" + ); + } + + #[test] + fn skips_the_write_when_the_row_already_holds_the_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let first = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + assert_eq!(first, AfterResponseOutcome::Written, "should write first"); + + let second = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + second, + AfterResponseOutcome::Unchanged, + "should not rewrite identical results" + ); + } + + #[test] + fn writes_nothing_without_a_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not create a row" + ); + assert!( + graph.load_snapshot(EC_ID).entry_for(EC_ID).is_none(), + "should leave the row absent" + ); + } + + #[test] + fn writes_nothing_to_a_withdrawn_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not revive a tombstone" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store the provider UID on a tombstone" + ); + } + + #[test] + fn drops_results_made_for_another_ec() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let other = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ABC123"; + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(other, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should discard results dispatched for another EC" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store results for another EC" + ); + } + + #[test] + fn skips_the_write_without_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &LookupResults::new(EC_ID, Vec::new()), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should make no write without results" + ); + } + + #[test] + fn lookups_run_only_for_an_existing_consented_row() { + // Arrange + let graph = KvIdentityGraph::in_memory("after_response_allowed"); + let missing = graph.load_snapshot(EC_ID); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let present = graph.load_snapshot(EC_ID); + let withdrawn_graph = KvIdentityGraph::in_memory("after_response_withdrawn"); + withdrawn_graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let withdrawn = withdrawn_graph.load_snapshot(EC_ID); + + // Act + Assert + assert!( + after_response_lookups_allowed(&present, EC_ID, false), + "should run for a returning EC with a live row" + ); + assert!( + !after_response_lookups_allowed(&present, EC_ID, true), + "should not run for an EC minted on this navigation" + ); + assert!( + !after_response_lookups_allowed(&missing, EC_ID, false), + "should not run when the row does not exist" + ); + assert!( + !after_response_lookups_allowed(&withdrawn, EC_ID, false), + "should not run for a withdrawn row" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 4903589b29..756cc0f490 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -33,6 +33,8 @@ const REASON_INVALID_EC_ID: &str = "invalid_ec_id"; const REASON_INVALID_PARTNER_UID: &str = "invalid_partner_uid"; const REASON_INELIGIBLE: &str = "ineligible"; const REASON_KV_UNAVAILABLE: &str = "kv_unavailable"; +const REASON_PROVIDER_OWNED: &str = "provider_owned"; +const REASON_ROW_FULL: &str = "row_full"; /// Maximum number of mappings allowed in a single batch request. const MAX_BATCH_SIZE: usize = 1000; @@ -167,7 +169,12 @@ fn handle_batch_sync_with_writer( } // 4. Process mappings with per-item validation and rejection reasons. - let (accepted, errors) = process_mappings(writer, &partner.source_domain, &body.mappings); + let (accepted, errors) = process_mappings_for_partner( + writer, + &partner.source_domain, + registry.is_provider_owned(&partner.source_domain), + &body.mappings, + ); let rejected = errors.len(); let status = if rejected > 0 { @@ -193,6 +200,30 @@ fn content_length_exceeds_limit(req: &Request, max_body_size: usize) - .is_some_and(|content_length| content_length > max_body_size) } +/// Processes mappings for one partner, rejecting all of them when an enabled +/// identity provider owns the partner's source (spec §8.5). +fn process_mappings_for_partner( + writer: &dyn BatchSyncWriter, + partner_id: &str, + provider_owned: bool, + mappings: &[SyncMapping], +) -> (usize, Vec) { + if !provider_owned { + return process_mappings(writer, partner_id, mappings); + } + log::info!( + "Batch sync: rejecting {} mappings for provider-owned source '{partner_id}'", + mappings.len() + ); + let errors = (0..mappings.len()) + .map(|index| MappingError { + index, + reason: REASON_PROVIDER_OWNED, + }) + .collect(); + (0, errors) +} + /// Validates all mappings, then processes each normalized EC ID once. /// /// Successful and eligibility outcomes fan out to every valid input in a @@ -248,6 +279,12 @@ fn process_mappings( Ok(UpsertResult::Written | UpsertResult::Unchanged) => { accepted += group.indexes.len(); } + Ok(UpsertResult::RowFull) => { + errors.extend(group.indexes.iter().map(|&index| MappingError { + index, + reason: REASON_ROW_FULL, + })); + } Ok(UpsertResult::NotFound | UpsertResult::ConsentWithdrawn) => { errors.extend(group.indexes.iter().map(|&index| MappingError { index, @@ -544,6 +581,48 @@ mod tests { assert_eq!(errors[1].reason, REASON_INVALID_PARTNER_UID); } + #[test] + fn provider_owned_source_rejects_every_mapping() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Written)]); + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = + process_mappings_for_partner(&writer, "ids.example.com", true, &mappings); + + assert_eq!(accepted, 0, "should accept nothing"); + assert_eq!(errors.len(), 2, "should reject each mapping"); + assert!( + errors + .iter() + .all(|error| error.reason == REASON_PROVIDER_OWNED), + "should report provider_owned" + ); + } + + #[test] + fn process_mappings_reports_a_full_row_and_continues() { + let writer = MockWriter::new(vec![Ok(UpsertResult::RowFull), Ok(UpsertResult::Written)]); + + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 1, "should accept the mapping after the full row"); + assert_eq!(errors.len(), 1, "should report only the full row"); + assert_eq!(errors[0].index, 0, "should report the full row's index"); + assert_eq!( + errors[0].reason, REASON_ROW_FULL, + "should give a full row its own reason" + ); + assert_eq!(writer.calls().len(), 2, "should continue past a full row"); + } + #[test] fn process_mappings_aborts_on_kv_unavailable() { let writer = MockWriter::new(vec![ diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs new file mode 100644 index 0000000000..c3f6a9a642 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -0,0 +1,1638 @@ +//! Server-side identity lookup coordination (spec §8). +//! +//! [`ProviderCoordinator`] owns eligibility, consent, destination checks, a +//! deadline-bounded fan-out through a [`LookupTransport`], validation and +//! planning of the navigation's single row change. Providers stay pure. + +pub mod results; +pub mod transport; +mod validate; + +use std::collections::HashSet; +use std::num::{NonZeroU16, NonZeroU32}; +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; + +use self::results::{LookupResults, LookupStatus, ProviderLookup, SkipReason}; +use self::transport::{ + Deadline, LookupTransport, ProviderIndex, Tagged, TransportError, TransportResponse, +}; +use self::validate::{ValidatedEntries, ValidationPolicy, validate}; +use crate::consent::ConsentContext; +use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; +use crate::ec::EcKvSnapshot; +use crate::ec::consent::ec_consent_granted; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::generation::ec_hash; +use crate::ec::kv_types::KvEntry; +use crate::ec::provider::{ + ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, + ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, + validate_enabled_configs, +}; +use crate::ec::rate_limiter::RateLimiter; +use crate::ec::registry::PartnerRegistry; + +use crate::settings::IdentityProviderConfig; + +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget and validation. Providers never see KV, cookies, HTTP or the +/// clock. Implementations never write KV: they return the change and the +/// caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when no provider is enabled; skipped + /// providers are recorded and reported by [`Self::collect`]. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; +} + +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read before dispatch. + pub snapshot: &'a EcKvSnapshot, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals lent to providers. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + /// Request time. + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. +/// +/// Only the coordinator module can create a non-empty value. `Default` +/// (empty) and `Debug` (counts only) are implemented by hand so they do not +/// require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// EC ID the lookups were dispatched for. + ec_id: String, + /// Request time, for validating `exp` and resolving relative expiry. + now: UnixSeconds, + /// Providers that finished at dispatch (skipped or failed to send). + early: Vec>, + /// Providers whose IDs were generated at dispatch, validated. + generated: Vec>, + /// Provider state to hand back with each in-flight request's response. + contexts: Vec>, +} + +impl Default for PendingLookups { + fn default() -> Self { + Self { + requests: Vec::new(), + deadline: None, + ec_id: String::new(), + now: UnixSeconds::new(0), + early: Vec::new(), + generated: Vec::new(), + contexts: Vec::new(), + } + } +} + +impl std::fmt::Debug for PendingLookups { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingLookups") + .field("requests", &self.requests.len()) + .field("early", &self.early.len()) + .field("generated", &self.generated.len()) + .finish_non_exhaustive() + } +} + +impl PendingLookups { + /// Whether nothing was dispatched, generated or skipped. + #[must_use] + pub fn is_empty(&self) -> bool { + self.requests.is_empty() && self.early.is_empty() && self.generated.is_empty() + } +} + +/// A configured provider handed to [`ProviderCoordinator::new`]. +pub struct RegisteredProvider

{ + /// The provider's configuration entry. + pub config: IdentityProviderConfig, + /// The built provider. + pub provider: P, +} + +impl

RegisteredProvider

{ + /// Pairs a configuration entry with its built provider. + #[must_use] + pub fn new(config: IdentityProviderConfig, provider: P) -> Self { + Self { config, provider } + } +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated when the coordinator is built. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +/// The production coordinator, built per request from settings. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation. + partners: Arc, + budget: Duration, +} + +impl ProviderCoordinator

{ + /// Validates and registers the enabled providers. + /// + /// Disabled providers are dropped: they make no requests. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when an enabled provider's + /// endpoint is not `https` with an allowlisted host, a source has no + /// partner entry, a source is outside the provider's capabilities, a + /// source is claimed by two enabled providers, or more than `u16::MAX` + /// providers are enabled. + pub fn new( + registered: Vec>, + partners: Arc, + budget: Duration, + ) -> Result> { + let enabled: Vec> = registered + .into_iter() + .filter(|registered| registered.config.enabled) + .collect(); + if u16::try_from(enabled.len()).is_err() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("too many identity providers are enabled")); + } + validate_enabled_configs( + enabled.iter().map(|registered| ®istered.config), + &partners, + )?; + + let providers = enabled + .into_iter() + .map(Self::register) + .collect::, Report>>()?; + + Ok(Self { + providers, + partners, + budget, + }) + } + + fn register( + registered: RegisteredProvider

, + ) -> Result, Report> { + let RegisteredProvider { config, provider } = registered; + let sources = parse_sources(&config)?; + let capable: HashSet<&SourceDomain> = provider + .capabilities() + .sources + .iter() + .map(|capability| &capability.source) + .collect(); + if let Some(source) = sources.iter().find(|source| !capable.contains(source)) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is outside its kind's capabilities", + config.id + ))); + } + Ok(Registered { + id: ProviderId::new(&config.id), + policy: ProviderPolicy { + sources, + allowed_domains: config + .allowed_domains + .iter() + .map(|domain| Box::from(domain.as_str())) + .collect(), + tcf_vendor_id: config.tcf_vendor_id, + rate_limit_per_hour: config.rate_limit_per_hour, + }, + provider, + }) + } + + /// Whether no provider is enabled. + #[must_use] + pub fn is_empty(&self) -> bool { + self.providers.is_empty() + } + + /// Whether the row holds a live provider-written UID for `source`. + fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { + entry + .and_then(|entry| entry.ids.get(source)) + .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)) + } + + /// Claimed sources lacking a live provider-written UID (a browser + /// fallback does not count), minus sources needing input V1 + /// never has; the flag reports whether any source was removed for input. + fn due_sources( + registered: &Registered

, + entry: Option<&KvEntry>, + now: UnixSeconds, + ) -> (Vec, bool) { + let capabilities = registered.provider.capabilities(); + let mut input_blocked = false; + let mut due = Vec::new(); + for source in ®istered.policy.sources { + let record = entry + .and_then(|entry| entry.ids.get(source.as_str())) + .filter(|record| record.is_provider_written()); + // A UID whose provider asked for a refresh makes its source due; + // the stored refresh state is the lookup's input. + if record.is_some_and(|record| record.refresh_due_for(®istered.id, now)) { + due.push(source.clone()); + continue; + } + if record.is_some_and(|record| record.has_live_uid(now)) { + continue; + } + let needs_input = capabilities + .sources + .iter() + .find(|capability| capability.source == *source) + .is_some_and(|capability| capability.input != SourceInput::Anonymous); + if needs_input { + input_blocked = true; + continue; + } + due.push(source.clone()); + } + (due, input_blocked) + } + + /// Whether the provider's hourly limit is reached. Limiter errors fail + /// open, matching pull sync. + /// + /// The bucket is the EC hash (derived from the client IP), not the full + /// EC ID, so ECs behind one IP share it and a client that clears cookies + /// cannot reset it. + fn rate_limited(registered: &Registered

, lookup: &LookupContext<'_>) -> bool { + let Some(limit) = registered.policy.rate_limit_per_hour else { + return false; + }; + let key = format!("identity:{}:{}", registered.id, ec_hash(lookup.ec_id)); + match lookup.rate_limiter.exceeded(&key, limit.get()) { + Ok(exceeded) => exceeded, + Err(report) => { + log::warn!( + "Identity lookup: provider={} rate limiter failed, allowing lookup: {report:?}", + registered.id + ); + false + } + } + } + + fn lookup( + &self, + provider: ProviderIndex, + status: LookupStatus, + entries: ValidatedEntries, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let cleared = if status == LookupStatus::Skipped(SkipReason::OptedOut) { + registered.policy.sources.to_vec() + } else { + Vec::new() + }; + ProviderLookup { + provider: registered.id.clone(), + status, + entries, + cleared, + } + } + + fn interpret( + &self, + provider: ProviderIndex, + response: Result>, + context: Option<&str>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let response = match response { + Ok(response) => response, + Err(report) => { + let status = match report.current_context() { + TransportError::Timeout => LookupStatus::Timeout, + TransportError::Send | TransportError::BodyTooLarge => { + log::debug!( + "Identity lookup: provider={} transport failed: {report:?}", + registered.id + ); + LookupStatus::TransportFailed + } + }; + return self.lookup(provider, status, Vec::new()); + } + }; + if matches!( + response.status, + http::StatusCode::NOT_FOUND | http::StatusCode::NO_CONTENT + ) { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + if !response.status.is_success() { + return self.lookup( + provider, + LookupStatus::HttpError(response.status), + Vec::new(), + ); + } + let parsed = registered.provider.parse_response(ProviderResponse { + status: response.status, + headers: &response.headers, + body: &response.body, + context, + }); + let eids = match parsed { + Ok(LookupOutcome::Found(eids)) => eids, + Ok(LookupOutcome::NoMatch) => { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + Ok(LookupOutcome::OptedOut) => { + return self.lookup( + provider, + LookupStatus::Skipped(SkipReason::OptedOut), + Vec::new(), + ); + } + Err(report) => { + log::debug!( + "Identity lookup: provider={} response not parsed: {}", + registered.id, + report.current_context() + ); + return self.lookup(provider, LookupStatus::InvalidResponse, Vec::new()); + } + }; + self.validated(provider, eids, now) + } + + /// Validates a provider's entries and reports them as one lookup. + fn validated( + &self, + provider: ProviderIndex, + eids: Vec>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let policy = ValidationPolicy { + provider: ®istered.id, + claimed: ®istered.policy.sources, + shape: registered.provider.capabilities().shape, + partners: &self.partners, + now, + }; + let (entries, dropped) = validate(eids, &policy); + let kept = entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + if kept == 0 { + log::info!( + "Identity lookup: provider={} dropped every UID ({} drops)", + registered.id, + dropped.total() + ); + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + let kept = u16::try_from(kept).unwrap_or(u16::MAX); + self.lookup(provider, LookupStatus::Found { kept, dropped }, entries) + } +} + +impl IdentityCoordinator for ProviderCoordinator

{ + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups { + let mut pending = PendingLookups { + ec_id: lookup.ec_id.to_owned(), + now: lookup.now, + ..PendingLookups::default() + }; + if self.providers.is_empty() { + return pending; + } + let entry = lookup.snapshot.entry_for(lookup.ec_id); + // Missing EC or EID consent, a withdrawal tombstone, or an unreadable + // row (which may hide a tombstone) skips every provider: a lookup + // discloses the EC ID before finalize could decline it. + let readable = !matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); + let consented = readable + && ec_consent_granted(lookup.consent) + && allows_eid_persistence(lookup.consent) + && entry.is_none_or(|entry| entry.consent.ok); + let deadline = Deadline::after(self.budget); + pending.deadline = Some(deadline); + for (index, registered) in self.providers.iter().enumerate() { + let provider = ProviderIndex::new(index); + let skip = |reason| Tagged { + provider, + value: LookupStatus::Skipped(reason), + }; + let vendor_denied = registered + .policy + .tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(lookup.consent, vendor)); + if !consented || vendor_denied { + pending.early.push(skip(SkipReason::Consent)); + continue; + } + // A remembered opt-out keeps the provider from being called until + // it lapses. + if entry.is_some_and(|entry| entry.opted_out(®istered.id, lookup.now)) { + pending.early.push(skip(SkipReason::OptedOut)); + continue; + } + let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + if due.is_empty() { + pending.early.push(skip(if input_blocked { + SkipReason::Input + } else { + SkipReason::NotDue + })); + continue; + } + let input = LookupInput { + ec_id: lookup.ec_id, + consent: lookup.consent, + client: lookup.client, + publisher_domain: lookup.publisher_domain, + due_sources: &due, + current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), + hashed_email: None, + }; + if let Some(mut eids) = registered.provider.generate(&input) { + // Only a due source without a live UID may take a generated + // ID: a refresh-due source keeps its UID and goes to lookup. + eids.retain(|eid| { + SourceDomain::try_from(eid.source.as_ref()).is_ok_and(|source| { + due.contains(&source) + && !Self::holds_live_uid(entry, source.as_str(), lookup.now) + }) + }); + if !eids.is_empty() { + pending.generated.push(Tagged { + provider, + value: self.validated(provider, eids, lookup.now), + }); + continue; + } + } + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } + let mut request = match registered.provider.build_request(&input) { + Ok(request) => request, + Err(report) => { + log::warn!( + "Identity lookup: provider={} request not built: {}", + registered.id, + report.current_context() + ); + pending.early.push(Tagged { + provider, + value: LookupStatus::RequestFailed, + }); + continue; + } + }; + if !destination_allowed(&request.url, ®istered.policy.allowed_domains) { + pending.early.push(skip(SkipReason::Destination)); + continue; + } + let context = request.context.take(); + match transport.start(request, deadline).await { + Ok(value) => { + pending.requests.push(Tagged { provider, value }); + if let Some(value) = context { + pending.contexts.push(Tagged { provider, value }); + } + } + Err(report) => { + log::warn!( + "Identity lookup: provider={} send failed: {}", + registered.id, + report.current_context() + ); + pending.early.push(Tagged { + provider, + value: LookupStatus::TransportFailed, + }); + } + } + } + pending + } + + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults { + let PendingLookups { + requests, + deadline, + ec_id, + now, + early, + generated, + contexts, + } = pending; + let responses = match deadline { + Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, + _ => Vec::new(), + }; + let mut slots: Vec> = + (0..self.providers.len()).map(|_| None).collect(); + for Tagged { provider, value } in early { + slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); + } + for Tagged { provider, value } in generated { + slots[provider.get()] = Some(value); + } + for Tagged { provider, value } in responses { + let context = contexts + .iter() + .find(|tagged| tagged.provider == provider) + .map(|tagged| tagged.value.as_str()); + slots[provider.get()] = Some(self.interpret(provider, value, context, now)); + } + let lookups: Vec = slots.into_iter().flatten().collect(); + for lookup in &lookups { + log_outcome(lookup); + } + LookupResults::new(&ec_id, lookups) + } +} + +fn log_outcome(lookup: &ProviderLookup) { + let (kept, dropped) = match lookup.status { + LookupStatus::Found { kept, dropped } => (kept, dropped.total()), + _ => (0, 0), + }; + log::info!( + "Identity lookup: provider={} outcome={} kept={kept} dropped={dropped}", + lookup.provider, + lookup.status.metric_name() + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::jurisdiction::Jurisdiction; + use crate::consent::types::TcfConsent; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; + use crate::ec::provider::{ + EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, + ProviderRequest, ProviderUid, SourceCapability, + }; + use crate::error::TrustedServerError; + use crate::settings::EcPartner; + use std::borrow::Cow; + use std::cell::RefCell; + use std::num::NonZeroUsize; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + type Reply = fn() -> Result>; + + /// Context the echo provider sets on its request and expects back. + const ECHO_CONTEXT: &str = "echo-context"; + + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!( + "https://ids.example.com/lookup?due={}", + lookup.due_sources.len() + )) + .expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + context: Some(ECHO_CONTEXT.to_owned()), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.context != Some(ECHO_CONTEXT) { + return Err(Report::new(ProviderError::InvalidResponse)); + } + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + if id == "optout" { + return Ok(LookupOutcome::OptedOut); + } + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }])) + } + } + + /// Answers every request with a canned result, recording the URLs sent. + struct CannedTransport { + reply: Reply, + sent: RefCell>, + } + + impl CannedTransport { + fn new(reply: Reply) -> Self { + Self { + reply, + sent: RefCell::default(), + } + } + } + + impl LookupTransport for CannedTransport { + type Pending = (); + + async fn start( + &self, + request: ProviderRequest, + _deadline: Deadline, + ) -> Result<(), Report> { + self.sent.borrow_mut().push(request.url.to_string()); + Ok(()) + } + + async fn gather( + &self, + pending: Vec>, + _deadline: Deadline, + ) -> Vec>>> { + pending + .into_iter() + .map(|tagged| Tagged { + provider: tagged.provider, + value: (self.reply)(), + }) + .collect() + } + } + + struct Limiter(bool); + + impl RateLimiter for Limiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(self.0) + } + } + + fn reply(status: http::StatusCode, body: &'static [u8]) -> TransportResponse { + TransportResponse { + status, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from_static(body), + } + } + + fn ok_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"p-1")) + } + + fn timeout_reply() -> Result> { + Err(Report::new(TransportError::Timeout)) + } + + fn not_found_reply() -> Result> { + Ok(reply(http::StatusCode::NOT_FOUND, b"")) + } + + fn server_error_reply() -> Result> { + Ok(reply(http::StatusCode::BAD_GATEWAY, b"")) + } + + fn invalid_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"\xff")) + } + + fn partners() -> Arc { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + Arc::new(PartnerRegistry::from_config(&[partner]).expect("should build registry")) + } + + fn provider_config(extra: &str) -> IdentityProviderConfig { + toml::from_str(&format!( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]\n{extra}" + )) + .expect("should parse config") + } + + fn echo_provider(input: SourceInput) -> EchoProvider { + EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + }) + } + + fn coordinator_with( + registered: RegisteredProvider, + ) -> ProviderCoordinator { + ProviderCoordinator::new(vec![registered], partners(), Duration::from_millis(200)) + .expect("should build coordinator") + } + + fn coordinator(extra: &str) -> ProviderCoordinator { + coordinator_with(RegisteredProvider::new( + provider_config(extra), + echo_provider(SourceInput::Anonymous), + )) + } + + fn granting() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + } + } + + fn missing() -> EcKvSnapshot { + EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + } + } + + fn run_with( + coordinator: &ProviderCoordinator

, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + limiter: &dyn RateLimiter, + ) -> LookupResults { + let context = LookupContext { + ec_id: EC_ID, + snapshot, + consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + rate_limiter: limiter, + now: UnixSeconds::new(1000), + }; + futures::executor::block_on(async { + let pending = coordinator.dispatch(transport, &context).await; + coordinator.collect(transport, pending).await + }) + } + + fn run( + coordinator: &ProviderCoordinator

, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + ) -> LookupResults { + run_with(coordinator, transport, snapshot, consent, &Limiter(false)) + } + + fn only_status(results: &LookupResults) -> LookupStatus { + results + .iter() + .next() + .expect("should report the provider") + .status + } + + fn seeded_snapshot(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("coord"); + graph.create(EC_ID, entry).expect("should seed"); + graph.load_snapshot(EC_ID) + } + + #[test] + fn due_source_is_looked_up_and_validated() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep one UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0] + .owner() + .map(ProviderId::as_str), + Some("example-provider"), + "should record the owner" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + fn provider_record(id: &str) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + #[test] + fn source_with_browser_fallback_uid_is_still_due() { + // Arrange + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("fallback"), + ); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up a source holding only a browser fallback" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + #[test] + fn source_with_live_uid_is_not_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), provider_record("p-1")); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn tombstoned_row_skips_for_consent() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&KvEntry::tombstone(1000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip a tombstoned row" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn missing_ec_consent_skips_every_provider() { + let coordinator = coordinator(""); + let cases = [ + ( + "GPC opt-out in a US state", + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..Default::default() + }, + ), + ( + "unknown jurisdiction", + ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "GDPR without a TC string", + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ]; + + for (label, consent) in cases { + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + + let results = run(&coordinator, &transport, &seeded_snapshot(&entry), &consent); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip for {label}" + ); + assert!( + transport.sent.borrow().is_empty(), + "should send nothing for {label}" + ); + } + } + + #[test] + fn failed_snapshot_skips_every_provider() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let snapshot = EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }; + + let results = run(&coordinator, &transport, &snapshot, &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip when the row could not be read" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn tcf_vendor_id_does_not_gate_traffic_without_tcf() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert!( + !matches!(only_status(&results), LookupStatus::Skipped(_)), + "should look up when no TCF string applies outside GDPR" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + fn tcf_consent(vendor_consents: Vec) -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents, + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + + #[test] + fn missing_tcf_vendor_consent_skips_before_building_a_request() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &tcf_consent(vec![1])); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn granted_tcf_vendor_consent_allows_the_lookup() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &missing(), + &tcf_consent(vec![9999]), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up with vendor consent" + ); + } + + #[test] + fn hashed_email_sources_are_skipped_for_input() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + echo_provider(SourceInput::HashedEmail), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Input), + "should skip sources needing hashed email" + ); + } + + #[test] + fn rate_limited_provider_is_skipped() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + echo_provider(SourceInput::Anonymous), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run_with( + &coordinator, + &transport, + &missing(), + &granting(), + &Limiter(true), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip a rate-limited provider" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn built_url_outside_allowlist_is_rejected() { + let coordinator = coordinator(""); + // The echo provider builds its URL on the configured host; narrow the + // policy afterwards so the built URL falls outside the allowlist. + let coordinator = ProviderCoordinator { + providers: coordinator + .providers + .into_vec() + .into_iter() + .map(|mut registered| { + registered.policy.allowed_domains = Box::new([Box::from("other.example.com")]); + registered + }) + .collect(), + ..coordinator + }; + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Destination), + "should reject the destination" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn transport_and_http_outcomes_map_to_statuses() { + let cases: [(Reply, LookupStatus); 4] = [ + (timeout_reply, LookupStatus::Timeout), + (not_found_reply, LookupStatus::NoMatch), + ( + server_error_reply, + LookupStatus::HttpError(http::StatusCode::BAD_GATEWAY), + ), + (invalid_reply, LookupStatus::InvalidResponse), + ]; + for (reply, expected) in cases { + let coordinator = coordinator(""); + let transport = CannedTransport::new(reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report"); + assert_eq!(lookup.status, expected, "should map the outcome"); + assert!(lookup.entries.is_empty(), "should carry no entries"); + } + } + + #[test] + fn disabled_provider_is_not_registered() { + let coordinator = coordinator("enabled = false"); + + assert!(coordinator.is_empty(), "should drop disabled providers"); + } + + #[test] + fn endpoint_outside_allowlist_is_rejected_at_build() { + let mut config = provider_config(""); + config.endpoint = "https://other.example.com/".to_owned(); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject endpoint"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn source_outside_capabilities_is_rejected_at_build() { + let provider = EchoProvider(ProviderCapabilities { + sources: Box::new([]), + ..echo_provider(SourceInput::Anonymous).0 + }); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config(""), provider)], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject source outside capabilities"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + fn refreshable_record(from: u64) -> KvPartnerId { + refreshable_record_owned_by("example-provider", from) + } + + fn refreshable_record_owned_by(owner: &str, from: u64) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new(owner)), + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(from), + }); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + fn snapshot_holding(record: KvPartnerId) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert("ids.example.com".to_owned(), record); + seeded_snapshot(&entry) + } + + #[test] + fn a_due_refresh_makes_a_live_source_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should refresh a live UID whose refresh time has passed" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); + } + + #[test] + fn a_future_refresh_leaves_the_source_held() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(2000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh before the refresh time" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn remembered_opt_out_skips_the_provider_until_it_lapses() { + // Arrange + let coordinator = coordinator(""); + let provider = ProviderId::new("example-provider"); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.opt_outs.insert(provider.clone(), 1001); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::OptedOut), + "should skip a provider whose opt-out is remembered" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not call a provider the visitor opted out of" + ); + + // Arrange: the opt-out lapsed. + entry.opt_outs.insert(provider, 1000); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should call the provider again once the opt-out lapses" + ); + } + + #[test] + fn another_providers_refresh_state_is_never_handed_over() { + // Arrange: a live UID with due refresh state written by a provider that + // no longer owns the source. + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record_owned_by("former-provider", 900)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh a UID another provider wrote" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not send another provider's refresh state" + ); + } + + fn optout_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"optout")) + } + + #[test] + fn an_opted_out_reply_clears_the_sources() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(optout_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should report the opt-out" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + + /// An echo provider whose request can never be built. + struct UnbuildableProvider(EchoProvider); + + impl IdentityProvider for UnbuildableProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + } + + #[test] + fn a_request_the_provider_cannot_build_is_reported_as_a_failure() { + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + UnbuildableProvider(echo_provider(SourceInput::Anonymous)), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::RequestFailed, + "should report a provider bug instead of a skip" + ); + } + + /// An echo provider that generates its ID for the given source instead of + /// looking it up. + struct GeneratingProvider(EchoProvider, &'static str); + + impl IdentityProvider for GeneratingProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + Some(vec![ProviderEid { + source: Cow::Borrowed(self.1), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed("generated-1"), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }]) + } + } + + fn generating_coordinator() -> ProviderCoordinator { + generating_coordinator_for("ids.example.com") + } + + fn generating_coordinator_for(source: &'static str) -> ProviderCoordinator { + ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + GeneratingProvider(echo_provider(SourceInput::Anonymous), source), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator") + } + + #[test] + fn a_generated_id_replaces_the_lookup_for_a_due_source() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "generated-1", + "should store the generated ID" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn a_generated_id_matches_its_due_source_after_normalizing() { + let coordinator = generating_coordinator_for("IDS.example.com"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID for a differently cased source" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn a_refresh_due_live_source_is_looked_up_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup + .entries + .iter() + .flat_map(|(_, eids)| eids) + .flat_map(KvEid::uids) + .all(|uid| uid.id().as_str() != "generated-1"), + "should not replace the live UID with a generated one" + ); + assert_eq!( + transport.sent.borrow().len(), + 1, + "should send the refresh lookup instead" + ); + } + + #[test] + fn a_held_source_is_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(provider_record("p-1")), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the held ID" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs new file mode 100644 index 0000000000..e7348cda02 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -0,0 +1,181 @@ +//! Per-provider outcomes of one navigation's lookups. + +use crate::ec::eid_types::{EidValidationError, ProviderId, SourceDomain}; +use crate::ec::kv_types::KvEid; + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + /// Blank or oversized UIDs, or negative agent types. + pub invalid_uid: u16, + /// Invalid `ext`, `inserter`, `matcher` or `mm`. + pub invalid_field: u16, + /// Entries for sources the provider does not claim. + pub unclaimed_source: u16, + /// UIDs whose expiry is not after the request time. + pub expired: u16, + /// Entries or UIDs beyond the provider's shape limits. + pub over_limit: u16, + /// Duplicate UIDs within one source. + pub duplicate: u16, +} + +impl DropCounts { + /// Counts one validation failure. + pub fn record(&mut self, reason: EidValidationError) { + let counter = match reason { + EidValidationError::EmptyUid + | EidValidationError::UidTooLong + | EidValidationError::NegativeAgentType => &mut self.invalid_uid, + EidValidationError::UnknownMatchMethod + | EidValidationError::InvalidExt + | EidValidationError::FieldTooLong => &mut self.invalid_field, + EidValidationError::UnclaimedSource => &mut self.unclaimed_source, + EidValidationError::Expired => &mut self.expired, + }; + *counter = counter.saturating_add(1); + } + + /// Total drops. + #[must_use] + pub fn total(&self) -> u16 { + self.invalid_uid + .saturating_add(self.invalid_field) + .saturating_add(self.unclaimed_source) + .saturating_add(self.expired) + .saturating_add(self.over_limit) + .saturating_add(self.duplicate) + } +} + +/// Why a provider was not called. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + /// Consent does not permit this provider. + Consent, + /// None of its sources is due. + NotDue, + /// Every due source needs input V1 never has (hashed email). + Input, + /// The built URL is not `https` or not allowlisted. + Destination, + /// The provider's hourly rate limit is reached. + RateLimited, + /// The vendor reports an opt-out; the provider's stored UIDs are + /// removed. + OptedOut, +} + +/// One outcome per provider per navigation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + /// Valid entries were returned. + Found { + /// UIDs kept after validation. + kept: u16, + /// UIDs or entries dropped by validation. + dropped: DropCounts, + }, + /// The provider knows no ID for this visitor. + NoMatch, + /// No response before the shared deadline. + Timeout, + /// A non-success HTTP status. + HttpError(http::StatusCode), + /// The provider could not build its request. + RequestFailed, + /// The request could not be sent or the body exceeded the limit. + TransportFailed, + /// The body could not be parsed at all. + InvalidResponse, + /// The provider was not called. + Skipped(SkipReason), +} + +impl LookupStatus { + /// Metric name for logs (spec §12.2). + #[must_use] + pub fn metric_name(&self) -> &'static str { + match self { + Self::Found { .. } => "found", + Self::NoMatch => "no_match", + Self::Timeout => "timeout", + Self::HttpError(_) => "http_error", + Self::RequestFailed => "request_error", + Self::TransportFailed => "transport_error", + Self::InvalidResponse => "invalid_response", + Self::Skipped(SkipReason::Consent) => "skipped_consent", + Self::Skipped(SkipReason::NotDue) => "skipped_not_due", + Self::Skipped(SkipReason::Input) => "skipped_input", + Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::Destination) => "destination_rejected", + Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", + } + } +} + +/// One provider's outcome and validated entries. +#[derive(Debug, Clone)] +pub struct ProviderLookup { + /// The provider. + pub provider: ProviderId, + /// What happened. + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `Found`. + pub entries: Vec<(SourceDomain, Vec)>, + /// Sources whose stored records are removed, because the user opted + /// out of the provider; empty unless `Skipped(OptedOut)`. + pub cleared: Vec, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Clone, Default)] +pub struct LookupResults { + /// EC ID the lookups were made for; `None` when nothing was looked up. + ec_id: Option, + lookups: Vec, +} + +impl LookupResults { + /// Wraps per-provider results, made for `ec_id`, in provider-config + /// order. + #[must_use] + pub fn new(ec_id: &str, lookups: Vec) -> Self { + Self { + ec_id: Some(ec_id.to_owned()), + lookups, + } + } + + /// Whether these results were looked up for `ec_id`. + /// + /// A navigation can switch EC IDs after dispatch (orphan recovery mints a + /// new one), so results for another ID must not be attributed to it. + #[must_use] + pub fn is_for(&self, ec_id: &str) -> bool { + self.ec_id.as_deref() == Some(ec_id) + } + + /// Retargets results made for `orphan` to `replacement`. + /// + /// Orphan recovery replaces a proven-absent EC ID with a new one for the + /// same browser. Lookups depend on the browser, not on the EC ID, and the + /// orphan has no row left, so its results belong to the replacement row. + /// Results made for any other ID are left untouched. + pub(crate) fn retarget_orphan(&mut self, orphan: &str, replacement: &str) { + if self.is_for(orphan) { + self.ec_id = Some(replacement.to_owned()); + } + } + + /// Iterates results in provider-config order. + pub fn iter(&self) -> impl Iterator { + self.lookups.iter() + } + + /// Whether no provider produced a result. + #[must_use] + pub fn is_empty(&self) -> bool { + self.lookups.is_empty() + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs new file mode 100644 index 0000000000..668fbb5b91 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -0,0 +1,463 @@ +//! Sends provider lookups and gathers responses before a shared deadline. + +use std::cell::Cell; +use std::time::Duration; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; + +use crate::ec::provider::ProviderRequest; +use crate::platform::{ + PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, + RuntimeServices, +}; + +/// Largest provider response body accepted. +pub const MAX_PROVIDER_RESPONSE_BYTES: usize = 64 * 1024; + +/// Smallest transport timeout registered for a lookup backend, in +/// milliseconds; the shared deadline still bounds collection. +const MIN_TRANSPORT_TIMEOUT_MS: u32 = 50; + +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + /// The provider's position in the coordinator. + pub provider: ProviderIndex, + /// The labelled value. + pub value: T, +} + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ProviderIndex(u16); + +impl ProviderIndex { + /// Wraps a position; the coordinator rejects more than `u16::MAX` providers. + /// + /// # Panics + /// + /// Panics when `index` does not fit in a `u16`. + pub(crate) fn new(index: usize) -> Self { + Self(u16::try_from(index).expect("should hold fewer than 65536 providers")) + } + + /// The position. + #[must_use] + pub fn get(self) -> usize { + usize::from(self.0) + } +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: http::HeaderMap, + /// Response body, at most [`MAX_PROVIDER_RESPONSE_BYTES`]. + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +impl Deadline { + /// A deadline `budget` from now. + #[must_use] + pub fn after(budget: Duration) -> Self { + Self(web_time::Instant::now() + budget) + } + + /// The deadline instant. + #[must_use] + pub fn instant(self) -> web_time::Instant { + self.0 + } + + /// Time left before the deadline; zero once it has passed. + #[must_use] + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(web_time::Instant::now()) + } +} + +/// Why a lookup produced no response. +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + /// The request could not be sent. + #[display("lookup request could not be sent")] + Send, + /// No response before the deadline. + #[display("lookup did not complete before the deadline")] + Timeout, + /// The body exceeded [`MAX_PROVIDER_RESPONSE_BYTES`]. + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} + +/// [`LookupTransport`] over the platform HTTP client. +/// +/// Each request gets its own backend name (a per-transport sequence number is +/// the backend discriminator), so a ready response maps back to its provider +/// by backend name. Names repeat across requests and the transport timeout is +/// canonicalized, keeping the number of registered dynamic backends bounded. +pub struct PlatformLookupTransport<'s> { + services: &'s RuntimeServices, + next: Cell, +} + +impl<'s> PlatformLookupTransport<'s> { + /// Borrows the request's runtime services. + /// + /// Lookups always verify the provider's certificate: they carry the EC ID + /// and the provider's credential, so `proxy.certificate_check` (meant for + /// development origins) does not apply to them. + #[must_use] + pub fn new(services: &'s RuntimeServices) -> Self { + Self { + services, + next: Cell::new(0), + } + } + + fn transport_timeout(&self, deadline: Deadline) -> Duration { + let remaining_ms = u32::try_from(deadline.remaining().as_millis()).unwrap_or(u32::MAX); + let canonical = self + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, u32::MAX); + Duration::from_millis(u64::from(canonical.max(MIN_TRANSPORT_TIMEOUT_MS))) + } +} + +impl LookupTransport for PlatformLookupTransport<'_> { + type Pending = PlatformPendingRequest; + + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result> { + let ProviderRequest { + method, + url, + headers, + body, + context: _, + } = request; + let host = url + .host_str() + .ok_or_else(|| Report::new(TransportError::Send))? + .to_owned(); + let timeout = self.transport_timeout(deadline); + let sequence = self.next.get(); + self.next.set(sequence.wrapping_add(1)); + let backend = self + .services + .backend() + .ensure(&PlatformBackendSpec { + scheme: url.scheme().to_owned(), + host, + port: url.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(format!("ts-identity-{sequence}")), + }) + .change_context(TransportError::Send)?; + let mut builder = http::Request::builder().method(method).uri(url.as_str()); + if let Some(target) = builder.headers_mut() { + *target = headers; + } + let edge_request = builder + .body(EdgeBody::from(body.to_vec())) + .change_context(TransportError::Send)?; + self.services + .http_client() + .send_async(PlatformHttpRequest::new(edge_request, backend).with_cache_bypass()) + .await + .change_context(TransportError::Send) + } + + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>> { + // Names are captured before the first select: some adapters drop + // them from the requests that remain in flight. + let mut outstanding: Vec<(String, ProviderIndex)> = Vec::with_capacity(pending.len()); + let mut remaining = Vec::with_capacity(pending.len()); + let mut results = Vec::with_capacity(pending.len()); + for Tagged { provider, value } in pending { + match value.backend_name() { + Some(name) => { + outstanding.push((name.to_owned(), provider)); + remaining.push(value); + } + None => results.push(Tagged { + provider, + value: Err(Report::new(TransportError::Send)), + }), + } + } + while !remaining.is_empty() { + match self + .services + .http_client() + .select_until(remaining, deadline.instant()) + .await + { + Ok(Some(selected)) => { + remaining = selected.remaining; + let (name, value) = match selected.ready { + Ok(response) => { + (response.backend_name.clone(), read_response(response).await) + } + Err(report) => ( + selected.failed_backend_name, + Err(report.change_context(TransportError::Send)), + ), + }; + match name.and_then(|name| take_provider(&mut outstanding, &name)) { + Some(provider) => results.push(Tagged { provider, value }), + None => { + log::warn!("Identity lookup: dropping response with unknown backend"); + } + } + } + Ok(None) => break, + Err(report) => { + log::warn!("Identity lookup: select failed: {report:?}"); + break; + } + } + } + // Every request not yet answered timed out or was abandoned. + results.extend(outstanding.into_iter().map(|(_, provider)| Tagged { + provider, + value: Err(Report::new(TransportError::Timeout)), + })); + results + } +} + +/// Removes the first outstanding request with backend `name`. +fn take_provider( + outstanding: &mut Vec<(String, ProviderIndex)>, + name: &str, +) -> Option { + let index = outstanding + .iter() + .position(|(candidate, _)| candidate == name)?; + Some(outstanding.remove(index).1) +} + +async fn read_response( + response: PlatformResponse, +) -> Result> { + let (parts, body) = response.response.into_parts(); + let body = match body { + EdgeBody::Once(bytes) => bytes, + stream => stream + .into_bytes_bounded(MAX_PROVIDER_RESPONSE_BYTES) + .await + .change_context(TransportError::BodyTooLarge)?, + }; + if body.len() > MAX_PROVIDER_RESPONSE_BYTES { + return Err(Report::new(TransportError::BodyTooLarge)); + } + Ok(TransportResponse { + status: parts.status, + headers: parts.headers, + body, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use std::sync::Arc; + + fn request(path: &str) -> ProviderRequest { + ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!("https://ids.example.com/{path}")).expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + context: None, + } + } + + fn gather_one( + transport: &PlatformLookupTransport<'_>, + deadline: Deadline, + ) -> Vec>>> { + futures::executor::block_on(async { + let pending = transport + .start(request("a"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![Tagged { + provider: ProviderIndex::new(0), + value: pending, + }], + deadline, + ) + .await + }) + } + + #[test] + fn gather_returns_tagged_responses() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, b"{\"a\":1}".to_vec()); + client.push_response(204, Vec::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + let deadline = Deadline::after(Duration::from_millis(200)); + + let results = futures::executor::block_on(async { + let first = transport + .start(request("a"), deadline) + .await + .expect("should start"); + let second = transport + .start(request("b"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![ + Tagged { + provider: ProviderIndex::new(0), + value: first, + }, + Tagged { + provider: ProviderIndex::new(1), + value: second, + }, + ], + deadline, + ) + .await + }); + + assert_eq!(results.len(), 2, "should return one result per request"); + let first = results + .iter() + .find(|result| result.provider == ProviderIndex::new(0)) + .expect("should tag provider 0"); + let first = first.value.as_ref().expect("should succeed"); + assert_eq!(first.status, http::StatusCode::OK, "should map provider 0"); + assert_eq!(first.body.as_ref(), b"{\"a\":1}", "should carry the body"); + let second = results + .iter() + .find(|result| result.provider == ProviderIndex::new(1)) + .expect("should tag provider 1"); + assert_eq!( + second.value.as_ref().expect("should succeed").status, + http::StatusCode::NO_CONTENT, + "should map provider 1" + ); + assert_eq!( + client.recorded_request_uris(), + [ + "https://ids.example.com/a".to_owned(), + "https://ids.example.com/b".to_owned() + ], + "should send both requests" + ); + } + + #[test] + fn gather_reports_timeout_for_requests_past_the_deadline() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, Vec::new()); + client.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(1))); + + let error = results[0].value.as_ref().expect_err("should time out"); + assert!( + matches!(error.current_context(), TransportError::Timeout), + "should report timeout" + ); + } + + #[test] + fn gather_rejects_oversized_bodies() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, vec![b'x'; MAX_PROVIDER_RESPONSE_BYTES + 1]); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(200))); + + let error = results[0] + .value + .as_ref() + .expect_err("should reject the body"); + assert!( + matches!(error.current_context(), TransportError::BodyTooLarge), + "should report an oversized body" + ); + } + + #[test] + fn gather_with_no_requests_returns_nothing() { + let client = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + + let results = futures::executor::block_on( + transport.gather(Vec::new(), Deadline::after(Duration::from_millis(1))), + ); + + assert!(results.is_empty(), "should return nothing"); + assert_eq!( + client.recorded_select_count(), + 0, + "should not select an empty list" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs new file mode 100644 index 0000000000..9d9a63b175 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -0,0 +1,437 @@ +//! Turns a provider's parsed response into stored entries (spec §8.2). + +use std::borrow::Cow; + +use crate::ec::coordinator::results::DropCounts; +use crate::ec::eid_types::{ + AgentType, EidFields, EidValidationError, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, + RawJson, SourceDomain, UidValue, UnixSeconds, +}; +use crate::ec::kv_types::{KvEid, KvRefresh, KvUid}; +use crate::ec::provider::{EidLevelFields, EidShape, ProviderEid, ProviderExpiry, ProviderUid}; +use crate::ec::registry::PartnerRegistry; + +/// Largest private refresh state a provider may keep with one UID. +const MAX_REFRESH_STATE_BYTES: usize = 2048; + +/// Policy one response is validated against. +pub(crate) struct ValidationPolicy<'a> { + /// Owner recorded on every kept UID. + pub provider: &'a ProviderId, + /// Sources the provider claims in config. + pub claimed: &'a [SourceDomain], + /// The provider's declared shape limits. + pub shape: EidShape, + /// Partner config, for the default `atype`. + pub partners: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Validated entries grouped by source, in response order. +pub(crate) type ValidatedEntries = Vec<(SourceDomain, Vec)>; + +/// Validates parsed entries, keeping valid ones grouped by source in +/// response order. +pub(crate) fn validate( + eids: Vec>, + policy: &ValidationPolicy<'_>, +) -> (ValidatedEntries, DropCounts) { + let mut dropped = DropCounts::default(); + let mut grouped: ValidatedEntries = Vec::new(); + let mut kept_eids = 0_usize; + + for eid in eids { + let Some(source) = claimed_source(&eid.source, policy.claimed) else { + dropped.record(EidValidationError::UnclaimedSource); + continue; + }; + if kept_eids == policy.shape.max_eids.get() { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + let fields = match eid_fields(&eid, policy.shape.eid_level_fields) { + Ok(fields) => fields, + Err(reason) => { + dropped.record(reason); + continue; + } + }; + let default_atype = policy + .partners + .find_by_source_domain(source.as_str()) + .map_or(1, |partner| partner.openrtb_atype); + let slot = match grouped.iter().position(|(existing, _)| existing == source) { + Some(index) => index, + None => { + grouped.push((source.clone(), Vec::new())); + grouped.len() - 1 + } + }; + let max_uids = policy.shape.max_uids_per_eid.get(); + let mut uids = Vec::with_capacity(eid.uids.len().min(max_uids)); + for uid in eid.uids { + if uids.len() == max_uids { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + match stored_uid(uid, default_atype, policy) { + Ok(uid) if is_duplicate(&grouped[slot].1, &uids, &uid) => { + dropped.duplicate = dropped.duplicate.saturating_add(1); + } + Ok(uid) => uids.push(uid), + Err(reason) => dropped.record(reason), + } + } + if !uids.is_empty() { + grouped[slot].1.push(KvEid::new(fields, uids)); + kept_eids += 1; + } + } + + grouped.retain(|(_, eids)| !eids.is_empty()); + (grouped, dropped) +} + +fn claimed_source<'p>(raw: &str, claimed: &'p [SourceDomain]) -> Option<&'p SourceDomain> { + let source = SourceDomain::try_from(raw).ok()?; + claimed.iter().find(|candidate| **candidate == source) +} + +fn bounded_field(value: Option<&Cow<'_, str>>) -> Result>, EidValidationError> { + match value { + Some(text) if text.len() > MAX_EID_FIELD_BYTES => Err(EidValidationError::FieldTooLong), + Some(text) => Ok(Some(Box::from(text.as_ref()))), + None => Ok(None), + } +} + +fn eid_fields( + eid: &ProviderEid<'_>, + level: EidLevelFields, +) -> Result { + if level == EidLevelFields::Ignored { + return Ok(EidFields::default()); + } + Ok(EidFields { + inserter: bounded_field(eid.inserter.as_ref())?, + matcher: bounded_field(eid.matcher.as_ref())?, + mm: eid.mm.map(MatchMethod::try_from).transpose()?, + ext: eid.ext.map(RawJson::try_from).transpose()?, + }) +} + +fn stored_uid( + uid: ProviderUid<'_>, + default_atype: i32, + policy: &ValidationPolicy<'_>, +) -> Result { + let id = UidValue::try_from(uid.id.into_owned())?; + let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; + let ext = uid.ext.map(RawJson::try_from).transpose()?; + let expiry = match resolve_expiry(uid.expiry, policy.now) { + Some(at) => Expiry::At(at), + None => Expiry::Never, + }; + if !expiry.is_live(policy.now) { + return Err(EidValidationError::Expired); + } + let stored = KvUid::new(id, atype, ext, Some(policy.provider.clone()), expiry); + let Some(refresh) = uid.refresh else { + return Ok(stored); + }; + // Unusable refresh state is dropped, never the UID: a refresh with no + // stated time, with empty or oversized state, or due already (which + // would make the source due on every navigation) is simply not stored. + let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; + match resolve_expiry(refresh.from, policy.now) { + Some(from) if usable && from > policy.now => Ok(stored.with_refresh(KvRefresh { + state: refresh.state.into_owned(), + from, + })), + _ => { + log::debug!( + "Identity lookup: provider={} drops unusable refresh state", + policy.provider + ); + Ok(stored) + } + } +} + +/// Resolves a provider-stated time against `now`. `None` when no time was +/// stated. +fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { + match expiry { + ProviderExpiry::Unspecified => None, + ProviderExpiry::At(at) => Some(at), + ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), + } +} + +fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { + earlier + .iter() + .flat_map(KvEid::uids) + .chain(current) + .any(|existing| existing.id() == uid.id()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::provider::ProviderRefresh; + use crate::settings::EcPartner; + use serde_json::value::RawValue; + use std::num::NonZeroUsize; + use std::time::Duration; + + fn registry() -> PartnerRegistry { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "openrtb_atype": 3, "bidstream_enabled": true + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn uid(id: &str) -> ProviderUid<'_> { + ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + } + } + + fn eid<'a>(source: &'a str, uids: Vec>) -> ProviderEid<'a> { + ProviderEid { + source: Cow::Borrowed(source), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } + + fn policy<'a>( + provider: &'a ProviderId, + claimed: &'a [SourceDomain], + partners: &'a PartnerRegistry, + ) -> ValidationPolicy<'a> { + ValidationPolicy { + provider, + claimed, + shape: EidShape { + max_eids: NonZeroUsize::new(2).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(2).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + partners, + now: UnixSeconds::new(1000), + } + } + + fn claimed() -> [SourceDomain; 1] { + [SourceDomain::try_from("ids.example.com").expect("should parse")] + } + + #[test] + fn keeps_valid_uids_and_fills_owner_atype_and_expiry() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut timed = uid("u-2"); + timed.expiry = ProviderExpiry::After(Duration::from_secs(60)); + + let (entries, dropped) = validate( + vec![eid("IDS.example.com", vec![uid("u-1"), timed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + let uids = entries[0].1[0].uids(); + assert_eq!(uids[0].atype().get(), 3, "should fill atype from partner"); + assert_eq!(uids[0].owner(), Some(&provider), "should record owner"); + assert_eq!( + uids[1].expiry(), + Expiry::At(UnixSeconds::new(1060)), + "should resolve relative expiry" + ); + } + + #[test] + fn keeps_the_uid_when_its_refresh_state_is_unusable() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let in_a_minute = ProviderExpiry::After(Duration::from_secs(60)); + let cases = [ + ( + "state".to_owned(), + in_a_minute, + Some(UnixSeconds::new(1060)), + ), + (String::new(), in_a_minute, None), + ("x".repeat(MAX_REFRESH_STATE_BYTES + 1), in_a_minute, None), + ("state".to_owned(), ProviderExpiry::Unspecified, None), + ( + "state".to_owned(), + ProviderExpiry::After(Duration::ZERO), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(1000)), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(999)), + None, + ), + ]; + + for (state, from, expected) in cases { + let mut refreshed = uid("u-1"); + refreshed.refresh = Some(ProviderRefresh { + state: Cow::Owned(state), + from, + }); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![refreshed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop no UID"); + assert_eq!( + entries[0].1[0].uids()[0] + .refresh() + .map(|refresh| refresh.from), + expected, + "should keep only usable refresh state" + ); + } + } + + #[test] + fn drops_invalid_values_by_reason() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let long = "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1); + let mut expired = uid("u-old"); + expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); + let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); + let mut bad_ext = uid("u-ext"); + bad_ext.ext = Some(&array_ext); + + let (entries, dropped) = validate( + vec![ + eid( + "ids.example.com", + vec![uid(" "), uid(&long), expired, bad_ext, uid("ok"), uid("ok")], + ), + eid("unclaimed.example.com", vec![uid("u-3")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped.invalid_uid, 2, "should drop blank and long UIDs"); + assert_eq!(dropped.expired, 1, "should drop expired UID"); + assert_eq!( + dropped.invalid_field, 1, + "should drop UID with non-object ext" + ); + assert_eq!(dropped.duplicate, 1, "should drop duplicate UID"); + assert_eq!(dropped.unclaimed_source, 1, "should drop unclaimed source"); + assert_eq!(entries.len(), 1, "should keep the claimed source only"); + assert_eq!(entries[0].1[0].uids().len(), 1, "should keep one valid UID"); + } + + #[test] + fn enforces_shape_limits_from_the_end() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + + let (entries, dropped) = validate( + vec![ + eid("ids.example.com", vec![uid("a"), uid("b"), uid("c")]), + eid("ids.example.com", vec![uid("d")]), + eid("ids.example.com", vec![uid("e")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!( + dropped.over_limit, 2, + "should drop third entry and third UID" + ); + let ids: Vec<&str> = entries[0] + .1 + .iter() + .flat_map(KvEid::uids) + .map(|uid| uid.id().as_str()) + .collect(); + assert_eq!( + ids, + ["a", "b", "d"], + "should keep the first entries and UIDs" + ); + } + + #[test] + fn ignored_eid_level_fields_are_stripped_not_rejected() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut with_fields = eid("ids.example.com", vec![uid("a")]); + with_fields.inserter = Some(Cow::Borrowed("inserter.example.com")); + with_fields.mm = Some(99); + let mut policy = policy(&provider, &claimed, &partners); + policy.shape.eid_level_fields = EidLevelFields::Ignored; + + let (entries, dropped) = validate(vec![with_fields], &policy); + + assert_eq!( + dropped, + DropCounts::default(), + "should not count ignored fields" + ); + assert_eq!( + entries[0].1[0].fields(), + &EidFields::default(), + "should strip EID-level fields" + ); + } + + #[test] + fn preserved_eid_level_fields_are_validated() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut bad_mm = eid("ids.example.com", vec![uid("a")]); + bad_mm.mm = Some(99); + let mut kept = eid("ids.example.com", vec![uid("b")]); + kept.inserter = Some(Cow::Borrowed("inserter.example.com")); + kept.mm = Some(3); + + let (entries, dropped) = + validate(vec![bad_mm, kept], &policy(&provider, &claimed, &partners)); + + assert_eq!(dropped.invalid_field, 1, "should drop unknown mm entry"); + assert_eq!( + entries[0].1[0].fields().inserter.as_deref(), + Some("inserter.example.com"), + "should keep valid inserter" + ); + assert_eq!( + entries[0].1[0].fields().mm, + Some(MatchMethod::Authenticated), + "should keep valid mm" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eid_types.rs b/crates/trusted-server-core/src/ec/eid_types.rs new file mode 100644 index 0000000000..9c9826e5d3 --- /dev/null +++ b/crates/trusted-server-core/src/ec/eid_types.rs @@ -0,0 +1,616 @@ +//! Value types for full-fidelity EIDs stored in the EC identity graph. +//! +//! Provider output is converted into these types at the trust boundary +//! (coordinator validation); past that point an invalid UID, agent type or +//! match method cannot be represented. + +use core::fmt; +use std::sync::Arc; + +use serde::de::{self, Deserializer, Visitor}; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::value::RawValue; + +use super::kv_types::MAX_STORED_UID_LENGTH; +use super::partner::normalize_partner_source_domain; + +/// Maximum serialized size of an `ext` object, in bytes. +pub const MAX_EXT_BYTES: usize = 1024; + +/// Maximum size of `inserter` and `matcher`, in bytes. +pub const MAX_EID_FIELD_BYTES: usize = 255; + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + /// The UID is blank. + #[display("UID is empty")] + EmptyUid, + /// The UID exceeds [`MAX_STORED_UID_LENGTH`]. + #[display("UID is too long")] + UidTooLong, + /// The agent type is negative. + #[display("agent type is negative")] + NegativeAgentType, + /// The match method is not an `OpenRTB` 2.6 value. + #[display("match method is unknown")] + UnknownMatchMethod, + /// An `ext` is not a JSON object within [`MAX_EXT_BYTES`]. + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + /// An `inserter` or `matcher` exceeds [`MAX_EID_FIELD_BYTES`]. + #[display("EID field is too long")] + FieldTooLong, + /// The source is not a valid domain or not claimed by the provider. + #[display("source is not claimed by this provider")] + UnclaimedSource, + /// The expiry is not after the request time. + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// Unix time in whole seconds. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + derive_more::Display, +)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +impl UnixSeconds { + /// Wraps a Unix timestamp in seconds. + #[must_use] + pub const fn new(seconds: u64) -> Self { + Self(seconds) + } + + /// Returns the timestamp in seconds. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } + + /// Adds `seconds`, saturating at `u64::MAX`. + #[must_use] + pub const fn saturating_add_secs(self, seconds: u64) -> Self { + Self(self.0.saturating_add(seconds)) + } +} + +struct ArcStrVisitor; + +impl<'de> Visitor<'de> for ArcStrVisitor { + type Value = Arc; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, value: &str) -> Result { + Ok(Arc::from(value)) + } +} + +/// Operator-chosen identifier of a configured provider instance. +/// +/// Clones share one allocation. Serialized as a plain string. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +impl ProviderId { + /// Creates a provider ID from configuration. + #[must_use] + pub fn new(id: &str) -> Self { + Self(Arc::from(id)) + } + + /// Returns the ID as a string slice. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Serialize for ProviderId { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ProviderId { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_str(ArcStrVisitor).map(Self) + } +} + +/// Normalized EID source domain. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +impl SourceDomain { + /// Returns the normalized domain. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom<&str> for SourceDomain { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + normalize_partner_source_domain(value) + .map(|normalized| Self(Arc::from(normalized))) + .map_err(|_| EidValidationError::UnclaimedSource) + } +} + +impl Serialize for SourceDomain { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SourceDomain { + fn deserialize>(deserializer: D) -> Result { + let raw = deserializer.deserialize_str(ArcStrVisitor)?; + Self::try_from(raw.as_ref()).map_err(de::Error::custom) + } +} + +/// A non-blank UID of at most [`MAX_STORED_UID_LENGTH`] bytes, kept verbatim. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl UidValue { + /// Returns the UID. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + fn check(value: &str) -> Result<(), EidValidationError> { + if value.trim().is_empty() { + return Err(EidValidationError::EmptyUid); + } + if value.len() > MAX_STORED_UID_LENGTH { + return Err(EidValidationError::UidTooLong); + } + Ok(()) + } +} + +impl TryFrom<&str> for UidValue { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + Self::check(value)?; + Ok(Self(Box::from(value))) + } +} + +impl TryFrom for UidValue { + type Error = EidValidationError; + + fn try_from(value: String) -> Result { + Self::check(&value)?; + Ok(Self(value.into_boxed_str())) + } +} + +/// Non-negative `OpenRTB` agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub struct AgentType(i32); + +impl AgentType { + /// Returns the raw agent type. + #[must_use] + pub const fn get(self) -> i32 { + self.0 + } +} + +impl TryFrom for AgentType { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + if value < 0 { + return Err(EidValidationError::NegativeAgentType); + } + Ok(Self(value)) + } +} + +impl From for i32 { + fn from(value: AgentType) -> Self { + value.0 + } +} + +/// `OpenRTB` 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub enum MatchMethod { + /// `0`: unknown. + Unknown, + /// `1`: no matching. + NoMatching, + /// `2`: cookie sync. + CookieSync, + /// `3`: authenticated. + Authenticated, + /// `4`: observed. + Observed, + /// `5`: inference. + Inference, +} + +impl TryFrom for MatchMethod { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + match value { + 0 => Ok(Self::Unknown), + 1 => Ok(Self::NoMatching), + 2 => Ok(Self::CookieSync), + 3 => Ok(Self::Authenticated), + 4 => Ok(Self::Observed), + 5 => Ok(Self::Inference), + _ => Err(EidValidationError::UnknownMatchMethod), + } + } +} + +impl From for i32 { + fn from(value: MatchMethod) -> Self { + match value { + MatchMethod::Unknown => 0, + MatchMethod::NoMatching => 1, + MatchMethod::CookieSync => 2, + MatchMethod::Authenticated => 3, + MatchMethod::Observed => 4, + MatchMethod::Inference => 5, + } + } +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(expires_at) => now < expires_at, + } + } + + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} + +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +pub mod optional_exp { + use serde::{Deserialize as _, Deserializer, Serialize as _, Serializer}; + + use super::{Expiry, UnixSeconds}; + + /// Serializes `At` as its seconds; `Never` is skipped by the caller. + /// + /// # Errors + /// + /// Returns the serializer's error. + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(expires_at) => expires_at.serialize(serializer), + Expiry::Never => serializer.serialize_none(), + } + } + + /// Reads an optional number; absent or `null` means [`Expiry::Never`]. + /// + /// # Errors + /// + /// Returns the deserializer's error for a non-numeric value. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} + +/// An opaque JSON object of at most [`MAX_EXT_BYTES`], compared by bytes. +/// +/// Deserializing applies the same checks as [`RawJson::try_from`], so a +/// stored row cannot carry an oversized or non-object `ext`. +#[derive(Debug, Clone)] +pub struct RawJson(Box); + +impl RawJson { + /// Returns the raw JSON text. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.get() + } + + /// Parses the raw JSON into a [`serde_json::Value`] for `OpenRTB` output. + /// + /// # Panics + /// + /// Never in practice: the bytes were validated as JSON when stored. + #[must_use] + pub fn to_value(&self) -> serde_json::Value { + serde_json::from_str(self.0.get()).expect("should parse stored raw JSON") + } +} + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +impl RawJson { + fn check(value: &RawValue) -> Result<(), EidValidationError> { + let text = value.get(); + if text.len() > MAX_EXT_BYTES || !text.trim_start().starts_with('{') { + return Err(EidValidationError::InvalidExt); + } + // `RawValue` skips nested values without serde_json's recursion limit, + // so parse once here: an `ext` that cannot become a `Value` would make + // `to_value` panic on every serve of the row. + serde_json::from_str::(text) + .map(|_| ()) + .map_err(|_| EidValidationError::InvalidExt) + } +} + +impl TryFrom<&RawValue> for RawJson { + type Error = EidValidationError; + + fn try_from(value: &RawValue) -> Result { + Self::check(value)?; + Ok(Self(value.to_owned())) + } +} + +impl TryFrom> for RawJson { + type Error = EidValidationError; + + fn try_from(value: Box) -> Result { + Self::check(&value)?; + Ok(Self(value)) + } +} + +impl Serialize for RawJson { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for RawJson { + fn deserialize>(deserializer: D) -> Result { + let raw = Box::::deserialize(deserializer)?; + Self::try_from(raw).map_err(de::Error::custom) + } +} + +/// EID-level fields shared by all UIDs of one stored entry. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct EidFields { + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option>, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option>, + /// `OpenRTB` `mm`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// `OpenRTB` EID `ext`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn uid_value_rejects_blank_and_oversized() { + assert!(UidValue::try_from(" ").is_err(), "should reject blank UID"); + let long = "a".repeat(MAX_STORED_UID_LENGTH + 1); + assert!( + UidValue::try_from(long.as_str()).is_err(), + "should reject oversized UID" + ); + let uid = UidValue::try_from("ID-123").expect("should accept valid UID"); + assert_eq!(uid.as_str(), "ID-123", "should keep UID verbatim"); + } + + #[test] + fn agent_type_rejects_negative() { + assert!( + AgentType::try_from(-1).is_err(), + "should reject negative atype" + ); + assert_eq!( + AgentType::try_from(571_187) + .expect("should accept vendor atype") + .get(), + 571_187, + "should keep vendor atype" + ); + } + + #[test] + fn match_method_round_trips_openrtb_values() { + for value in 0..=5 { + let method = MatchMethod::try_from(value).expect("should accept OpenRTB mm"); + assert_eq!(i32::from(method), value, "should round-trip mm {value}"); + } + assert!( + MatchMethod::try_from(6).is_err(), + "should reject unknown mm" + ); + } + + #[test] + fn source_domain_normalizes() { + let source = SourceDomain::try_from("IDS.Example.COM").expect("should accept domain"); + assert_eq!( + source.as_str(), + "ids.example.com", + "should lowercase source domain" + ); + } + + #[test] + fn provider_id_round_trips_as_string() { + let id = ProviderId::new("example-provider"); + let encoded = serde_json::to_string(&id).expect("should serialize provider id"); + assert_eq!( + encoded, "\"example-provider\"", + "should serialize as plain string" + ); + let decoded: ProviderId = + serde_json::from_str(&encoded).expect("should deserialize provider id"); + assert_eq!(decoded, id, "should round-trip provider id"); + } + + #[test] + fn expiry_liveness() { + let now = UnixSeconds::new(100); + assert!( + Expiry::Never.is_live(now), + "should serve UIDs without expiry" + ); + assert!( + Expiry::At(UnixSeconds::new(101)).is_live(now), + "should serve before expiry" + ); + assert!( + !Expiry::At(UnixSeconds::new(100)).is_live(now), + "should not serve at expiry" + ); + } + + #[test] + fn raw_json_accepts_small_objects_only() { + let object = + serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode object"); + let raw = RawJson::try_from(object.as_ref()).expect("should accept object"); + assert_eq!(raw.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + let array = serde_json::value::to_raw_value(&json!([1])).expect("should encode array"); + assert!( + RawJson::try_from(array.as_ref()).is_err(), + "should reject non-object" + ); + let big = serde_json::value::to_raw_value(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + RawJson::try_from(big.as_ref()).is_err(), + "should reject oversized ext" + ); + } + + #[test] + fn raw_json_rejects_ext_nested_past_the_parse_limit() { + let depth = 150; + let text = format!("{}1{}", r#"{"a":"#.repeat(depth), "}".repeat(depth)); + let raw: Box = + serde_json::from_str(&text).expect("should read deep JSON as a raw value"); + assert!( + text.len() <= MAX_EXT_BYTES, + "should fit the size limit so only the depth is under test" + ); + + assert!( + RawJson::try_from(raw).is_err(), + "should reject an ext that cannot be parsed into a value" + ); + } + + #[test] + fn raw_json_compares_by_bytes() { + let a = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + let b = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + assert_eq!( + RawJson::try_from(a.as_ref()).expect("should accept"), + RawJson::try_from(b.as_ref()).expect("should accept"), + "should compare equal bytes as equal" + ); + } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct WithExpiry { + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, + } + + #[test] + fn optional_exp_omits_never_and_reads_missing() { + let never = serde_json::to_value(WithExpiry { + expiry: Expiry::Never, + }) + .expect("should serialize"); + assert_eq!(never, json!({}), "should omit exp for Never"); + let at: WithExpiry = + serde_json::from_value(json!({ "exp": 42 })).expect("should deserialize exp"); + assert_eq!( + at.expiry, + Expiry::At(UnixSeconds::new(42)), + "should read exp" + ); + let missing: WithExpiry = + serde_json::from_value(json!({})).expect("should deserialize empty"); + assert_eq!(missing.expiry, Expiry::Never, "should default to Never"); + } + + #[test] + fn raw_json_deserialize_applies_same_checks() { + let ok: RawJson = serde_json::from_str(r#"{"k":1}"#).expect("should accept small object"); + assert_eq!(ok.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + assert!( + serde_json::from_str::("[1]").is_err(), + "should reject non-object on deserialize" + ); + let big = serde_json::to_string(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + serde_json::from_str::(&big).is_err(), + "should reject oversized ext on deserialize" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 1359f75091..a19f51e008 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -1,102 +1,136 @@ //! Shared EID resolution and formatting helpers. //! -//! Used by both `/_ts/api/v1/identify` and `/auction` to resolve source-domain -//! keyed IDs from KV entries, convert them to `OpenRTB` EID structures, and -//! build base64-encoded response headers. +//! Used by both `/_ts/api/v1/identify` and `/auction` to view source-domain +//! keyed IDs from KV entries as `OpenRTB` EID structures, and to build +//! base64-encoded response headers. use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use error_stack::{Report, ResultExt as _}; +use crate::consent::{ConsentContext, tcf_vendor_consent_denied}; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; -use super::kv_types::KvEntry; +use super::eid_types::{RawJson, UnixSeconds}; +use super::kv_types::{KvEid, KvEntry, KvPartnerId}; use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. pub const MAX_EIDS_HEADER_BYTES: usize = 4096; -/// A source-domain keyed ID resolved from a KV entry against the partner registry. -/// -/// Only includes partners with `bidstream_enabled = true` and a non-empty UID. -pub struct ResolvedPartnerId { - /// The partner's identity source domain and EC KV `ids` key. - pub source_domain: String, - /// The synced user ID value. - pub uid: String, - /// `OpenRTB` agent type for this partner's identifiers. - pub openrtb_atype: i32, +/// A borrowed EID entry: a single legacy UID or a full-fidelity provider entry. +#[derive(Debug, Clone, Copy)] +pub enum EidView<'row> { + /// A record holding one UID and no EID-level fields. + Single { + /// Source domain. + source: &'row str, + /// The UID. + uid: &'row str, + /// The partner's configured agent type. + atype: i32, + }, + /// A provider-written entry; only live UIDs are emitted. + Full { + /// Source domain. + source: &'row str, + /// The stored entry. + eid: &'row KvEid, + /// Request time used to filter UIDs. + now: UnixSeconds, + }, } -/// Resolves source-domain keyed IDs from a KV entry against the partner registry. -/// -/// Filters to partners with `bidstream_enabled = true` and non-empty UIDs, -/// sorted deterministically by source domain. -#[must_use] -pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec { - let mut resolved = Vec::new(); - - for (source_domain, partner_uid) in &entry.ids { - if partner_uid.uid.is_empty() { - continue; - } - - let Some(partner) = registry.get(source_domain) else { - continue; - }; - if !partner.bidstream_enabled { - continue; +impl From> for Eid { + fn from(view: EidView<'_>) -> Self { + match view { + EidView::Single { source, uid, atype } => Eid::new( + source, + vec![Uid { + id: uid.to_owned(), + atype: Some(atype), + ext: None, + }], + ), + EidView::Full { source, eid, now } => { + let fields = eid.fields(); + let uids = eid + .uids() + .iter() + .filter(|uid| uid.is_live(now)) + .map(|uid| Uid { + id: uid.id().as_str().to_owned(), + atype: Some(uid.atype().get()), + ext: uid.ext().map(RawJson::to_value), + }) + .collect(); + Eid { + inserter: fields.inserter.as_deref().map(str::to_owned), + matcher: fields.matcher.as_deref().map(str::to_owned), + mm: fields.mm.map(i32::from), + ext: fields.ext.as_ref().map(RawJson::to_value), + ..Eid::new(source, uids) + } + } } - - resolved.push(ResolvedPartnerId { - source_domain: partner.source_domain.clone(), - uid: partner_uid.uid.clone(), - openrtb_atype: partner.openrtb_atype, - }); } - - resolved.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - resolved } -/// Converts resolved partner IDs to `OpenRTB` `Eid` entries. -#[must_use] -pub fn to_eids(resolved: &[ResolvedPartnerId]) -> Vec { - resolved +/// Yields the live views of one stored record. +/// +/// Legacy records yield one [`EidView::Single`]; provider records yield one +/// [`EidView::Full`] per entry that still has a live UID. +pub fn source_views<'row>( + source: &'row str, + record: &'row KvPartnerId, + default_atype: i32, + now: UnixSeconds, +) -> impl Iterator> + 'row { + let single = + (!record.is_provider_written() && !record.uid.is_empty()).then_some(EidView::Single { + source, + uid: record.uid.as_str(), + atype: default_atype, + }); + let full = record + .eids() .iter() - .map(|item| Eid { - source: item.source_domain.clone(), - uids: vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], - }) - .collect() + .filter(move |eid| eid.uids().iter().any(|uid| uid.is_live(now))) + .map(move |eid| EidView::Full { source, eid, now }); + single.into_iter().chain(full) } -/// Builds a base64-encoded EID header value, truncating if needed. -/// -/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds -/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, partners are removed -/// from the end of the deterministic partner ordering until it fits. -/// -/// # Errors -/// -/// Returns an error if JSON serialization fails. -pub fn build_eids_header( - resolved: &[ResolvedPartnerId], -) -> Result<(String, bool), Report> { - let eids = to_eids(resolved); - encode_eids_header(&eids) +impl KvEntry { + /// Iterates live entries for registered, `bidstream_enabled` sources in + /// source order, borrowing from the row. + /// + /// A source whose owning provider has a TCF vendor ID is skipped while + /// `consent` withholds that vendor's consent. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + consent: &'row ConsentContext, + now: UnixSeconds, + ) -> impl Iterator> + 'row { + self.ids.iter().flat_map(move |(source, record)| { + let partner = registry.get(source).filter(|partner| { + partner.bidstream_enabled + && !partner + .owner_tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(consent, vendor)) + }); + partner + .into_iter() + .flat_map(move |partner| source_views(source, record, partner.openrtb_atype, now)) + }) + } } /// Encodes a pre-built EID slice into a base64 header value with truncation. /// -/// Like [`build_eids_header`] but operates on already-constructed `Eid` values -/// (e.g., from `UserInfo.eids` in the auction response path). -/// -/// Returns `(encoded_value, was_truncated)`. +/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds +/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, EIDs are removed from the +/// end until it fits. /// /// # Errors /// @@ -145,8 +179,12 @@ pub fn encode_eids_header(eids: &[Eid]) -> Result<(String, bool), Report EcPartner { EcPartner { @@ -167,113 +205,162 @@ mod tests { } } + fn header_eid(source_domain: String, uid: String) -> Eid { + Eid::new( + source_domain, + vec![Uid { + id: uid, + atype: Some(3), + ext: None, + }], + ) + } + #[test] - fn resolve_partner_ids_sorts_by_source_domain() { - let partners = vec![ - make_test_partner("zeta.example.com"), - make_test_partner("alpha.example.com"), - ]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + fn encode_eids_header_truncates_when_too_large() { + let eids: Vec = (0..64) + .map(|idx| { + header_eid( + format!("partner-{idx}.example.com"), + format!("uid_{}", "x".repeat(100)), + ) + }) + .collect(); - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "zeta.example.com".to_owned(), - super::super::kv_types::KvPartnerId { - uid: "uid-z".to_owned(), - }, + let (encoded, truncated) = + encode_eids_header(&eids).expect("should build truncated header"); + + assert!(truncated, "should report truncation for large payload"); + assert!( + encoded.len() <= MAX_EIDS_HEADER_BYTES, + "should cap encoded header bytes" ); + } + + #[test] + fn encode_eids_header_fits_without_truncation() { + let eids = vec![header_eid("ssp.example.com".to_owned(), "u1".to_owned())]; + + let (encoded, truncated) = + encode_eids_header(&eids).expect("should build header without truncation"); + + assert!(!truncated, "should not truncate small payload"); + assert!(!encoded.is_empty(), "should produce non-empty value"); + } + + fn registry_with(source: &str, bidstream: bool) -> PartnerRegistry { + let mut partner = make_test_partner(source); + partner.bidstream_enabled = bidstream; + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn provider_record(ids: &[(&str, Expiry)]) -> KvPartnerId { + let uids = ids + .iter() + .map(|(id, expiry)| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + *expiry, + ) + }) + .collect(); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, uids)], UnixSeconds::new(0)) + .expect("should build record") + } + + #[test] + fn live_eids_skip_expired_uids_and_keep_eid_fields() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); entry.ids.insert( - "alpha.example.com".to_owned(), - super::super::kv_types::KvPartnerId { - uid: "uid-a".to_owned(), - }, + "ids.example.com".to_owned(), + provider_record(&[ + ("old", Expiry::At(UnixSeconds::new(10))), + ("new", Expiry::Never), + ]), ); - let resolved = resolve_partner_ids(®istry, &entry); - let source_domains: Vec<&str> = resolved - .iter() - .map(|item| item.source_domain.as_str()) + let eids: Vec = entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(20)) + .map(Eid::from) .collect(); + assert_eq!(eids.len(), 1, "should emit one entry"); assert_eq!( - source_domains, - vec!["alpha.example.com", "zeta.example.com"], - "should sort deterministically by source domain" + eids[0].inserter.as_deref(), + Some("inserter.example.com"), + "should keep inserter" ); + let ids: Vec<&str> = eids[0].uids.iter().map(|uid| uid.id.as_str()).collect(); + assert_eq!(ids, vec!["new"], "should drop expired uid"); } #[test] - fn to_eids_maps_resolved_ids_correctly() { - let resolved = vec![ - ResolvedPartnerId { - uid: "LR_xyz".to_owned(), - source_domain: "liveramp.com".to_owned(), - openrtb_atype: 3, - }, - ResolvedPartnerId { - uid: "ID5_abc".to_owned(), - source_domain: "id5-sync.com".to_owned(), - openrtb_atype: 1, - }, - ResolvedPartnerId { - uid: "pair-id".to_owned(), - source_domain: "google.com".to_owned(), - openrtb_atype: 571187, - }, - ]; - - let eids = to_eids(&resolved); + fn live_eids_withhold_a_source_whose_owner_lacks_vendor_consent() { + let mut registry = registry_with("ids.example.com", true); + assert!( + registry.mark_provider_owned("ids.example.com", NonZeroU16::new(9999)), + "should mark the source" + ); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[("p-1", Expiry::Never)]), + ); + let denied = consent_granting_vendors(vec![1]); + let granted = consent_granting_vendors(vec![9999]); - assert_eq!(eids.len(), 3, "should produce one EID per resolved partner"); - assert_eq!(eids[0].source, "liveramp.com"); - assert_eq!(eids[0].uids[0].id, "LR_xyz"); - assert_eq!(eids[0].uids[0].atype, Some(3)); - assert_eq!(eids[1].source, "id5-sync.com"); - assert_eq!(eids[1].uids[0].id, "ID5_abc"); - assert_eq!(eids[1].uids[0].atype, Some(1)); - assert_eq!(eids[2].source, "google.com", "should preserve PAIR source"); - assert_eq!(eids[2].uids[0].id, "pair-id", "should preserve PAIR ID"); assert_eq!( - eids[2].uids[0].atype, - Some(571187), - "should preserve PAIR vendor-specific atype" + entry + .live_eids(®istry, &denied, UnixSeconds::new(0)) + .count(), + 0, + "should withhold the provider's UIDs without its vendor consent" + ); + assert_eq!( + entry + .live_eids(®istry, &granted, UnixSeconds::new(0)) + .count(), + 1, + "should serve the provider's UIDs with its vendor consent" ); } #[test] - fn build_eids_header_truncates_when_too_large() { - let mut resolved = Vec::new(); - for idx in 0..64 { - resolved.push(ResolvedPartnerId { - uid: format!("uid_{}", "x".repeat(100)), - source_domain: format!("partner-{idx}.example.com"), - openrtb_atype: 3, - }); - } - - let (encoded, truncated) = - build_eids_header(&resolved).expect("should build truncated header"); - - assert!(truncated, "should report truncation for large payload"); - assert!( - encoded.len() <= MAX_EIDS_HEADER_BYTES, - "should cap encoded header bytes" + fn live_eids_respect_bidstream_flag() { + let registry = registry_with("ids.example.com", false); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + assert_eq!( + entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .count(), + 0, + "should hide non-bidstream source" ); } #[test] - fn build_eids_header_fits_without_truncation() { - let resolved = vec![ResolvedPartnerId { - uid: "u1".to_owned(), - source_domain: "ssp.com".to_owned(), - openrtb_atype: 3, - }]; - - let (encoded, truncated) = - build_eids_header(&resolved).expect("should build header without truncation"); - - assert!(!truncated, "should not truncate small payload"); - assert!(!encoded.is_empty(), "should produce non-empty value"); + fn live_eids_emit_legacy_single_uid() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + let eids: Vec = entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .map(Eid::from) + .collect(); + assert_eq!(eids[0].uids[0].id, "v", "should emit legacy uid"); + assert_eq!(eids[0].uids[0].atype, Some(3), "should use partner atype"); } } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 019ef2aa40..d563c5b182 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1,9 +1,12 @@ //! EC response finalization. //! //! Centralizes post-routing EC behavior so all handlers get consistent cookie -//! and KV semantics. +//! and KV semantics. Every browser-driven KV write goes through the +//! navigation's single page-load write ([`KvIdentityGraph::commit_page_load`]), +//! so a request without a navigation permit never writes. The one exception is +//! a consent withdrawal, whose tombstone is written on any request. -use std::collections::HashSet; +use std::collections::BTreeSet; use edgezero_core::body::Body as EdgeBody; use http::Response; @@ -14,16 +17,16 @@ use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; +use super::eid_types::UnixSeconds; use super::generation::{generate_ec_id, is_valid_ec_id}; -use super::kv::{ - CreateIfAbsentOutcome, EidCookieSyncOutcome, KvIdentityGraph, PartnerIdUpdate, - apply_partner_id_updates, -}; +use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::page_load::{PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, EidSyncSource, current_timestamp, log_id}; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -35,12 +38,18 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// Finalizes EC response behavior for all routes. /// -/// Applies withdrawal handling, last-seen updates, cookie reconciliation, -/// Prebid EID ingestion, and cookie writes for new EC generation. +/// Applies withdrawal handling, browser EID ingestion, provider lookup +/// results, orphan recovery and cookie writes for new EC generation. /// -/// On consent withdrawal, the browser response clears the EC cookie -/// immediately and the EC identity-graph KV tombstone is the authoritative -/// revocation marker. There is no separate consent KV store to clean up. +/// The identity row is written at most once, and only when the request +/// carries a navigation permit (see [`super::page_load`]) and the final +/// response is not a redirect. Without a permit a newly generated EC is not +/// issued. +/// +/// Consent withdrawal is the one exception: on any request carrying the EC +/// cookie, the KV tombstone is written as the authoritative revocation marker +/// and the EC cookie is expired once every tombstone is in place (or the row +/// is proven absent). /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted /// from the request *before* routing consumes it. @@ -54,6 +63,10 @@ pub fn ec_finalize_response( response: &mut Response, ) { ec_context.validate_pull_sync_marker(settings, registry); + let (permit, after_response) = ec_context + .take_navigation_permit() + .and_then(|permit| permit.confirm_with_after_response(response.status())) + .unzip(); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -75,153 +88,191 @@ pub fn ec_finalize_response( return; } - // Returning user: consent is granted and EC came from request. - if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { - if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { - let source = ec_context.eid_sync_source(); - let collected = source - .map(|_| { - collect_eid_updates( - eids_cookie, - sharedid_cookie, - ec_context.client_eids(), - registry, - ) - }) - .unwrap_or_default(); - let had_eid_updates = !collected.is_empty(); - let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); - // `sync_eid_cookie_updates` early-returns without refreshing the - // snapshot when `updates` is empty, which is correct when there - // was never anything to write (an unconfigured registry, no sync - // source, or no EID cookies/body this request — the case - // `finalize_not_read_snapshot_does_not_rotate` covers). But when - // consent gating (e.g. TCF Purpose 4 denial) is what emptied - // `updates`, the request *did* have EID data, and orphan recovery - // below still needs an actual `Missing` read to detect an - // orphaned cookie. An unread `NotRead` snapshot proves nothing, so - // consent gating must not also skip that one read on a - // recovery-eligible request. - if updates.is_empty() - && had_eid_updates - && ec_context.recovery_eligible() - && matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) - { - let snapshot = graph.load_snapshot(&ec_id); - ec_context.set_kv_snapshot(snapshot); - } else if let Some(source) = source { - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, source); - } - if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) - && ec_context.recovery_eligible() - { - confirm_then_recover_orphaned_ec( - settings, ec_context, graph, &ec_id, &updates, response, - ); - } - } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); - - // Ordinary returning-user page views no longer refresh the browser - // cookie, emit the EC header, or update KV TTL. - return; - } - - // Newly generated EC in this request. Do not emit a generated EC when - // there is no KV graph: that would mint a browser cookie with no backing - // identity-graph row, producing a phantom ID on later requests. - if ec_context.ec_generated() { - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { - log::info!("Skipping generated EC response write because KV graph is unavailable"); - reconcile_pull_sync_marker(settings, registry, ec_context, response); - return; - }; - - let updates = gate_eid_updates_by_consent( + if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { + let collected = if ec_context.eid_sync_source().is_some() || ec_context.ec_generated() { collect_eid_updates( eids_cookie, sharedid_cookie, ec_context.client_eids(), registry, - ), - ec_context.consent(), - ); - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); - if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { - set_ec_cookie_on_response(settings, ec_context, response); + ) } else { - log::warn!("Skipping generated EC cookie because backing row is not authoritative"); + Vec::new() + }; + let had_eid_updates = !collected.is_empty(); + let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); + let commit = PageLoadCommit { + settings, + graph, + registry, + ec_id: &ec_id, + updates: &updates, + had_eid_updates, + }; + let page_write_ok = if ec_context.ec_generated() { + commit_generated(&commit, ec_context, permit, response) + } else if ec_context.ec_was_present() { + commit_returning(&commit, ec_context, permit, response) + } else { + false + }; + // The follow-up write is allowed only once consent allows the EC and + // the page-load write succeeded or had nothing to change. + if page_write_ok && let Some(after_response) = after_response { + ec_context.set_after_response_permit(after_response); } + } else if ec_context.ec_generated() { + // Do not emit a generated EC when there is no KV graph: that would + // mint a browser cookie with no backing identity-graph row. + log::info!("Skipping generated EC response write because KV graph is unavailable"); } reconcile_pull_sync_marker(settings, registry, ec_context, response); } -fn sync_eid_cookie_updates( - graph: &KvIdentityGraph, - ec_context: &mut EcContext, - ec_id: &str, - updates: &[PartnerIdUpdate], - source: EidSyncSource, -) { - if updates.is_empty() { - return; - } +/// Shared inputs for the navigation's single write. +struct PageLoadCommit<'a> { + settings: &'a Settings, + graph: &'a KvIdentityGraph, + registry: &'a PartnerRegistry, + ec_id: &'a str, + /// Browser ingest, already consent-gated. + updates: &'a [PartnerIdUpdate], + /// Whether the request carried EID data before consent gating. + had_eid_updates: bool, +} - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - ec_id, - updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); - record_eid_sync_terminal(source, outcome); +impl PageLoadCommit<'_> { + /// Plans the row change for `ec_id` against `snapshot`. + fn plan( + &self, + ec_context: &EcContext, + ec_id: &str, + snapshot: &EcKvSnapshot, + new_entry: Option<&KvEntry>, + ) -> Option { + plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry, + browser_ingest: self.updates, + lookups: ec_context.lookup_results(), + registry: self.registry, + now: UnixSeconds::new(current_timestamp()), + }) + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct EidSyncMeasurement { - source: EidSyncSource, - outcome: EidCookieSyncOutcome, - already_matched: u8, - written: u8, - conflict_duplicate: u8, - deferred: u8, +fn present_snapshot(ec_id: &str, entry: KvEntry) -> EcKvSnapshot { + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: None, + } } -impl EidSyncMeasurement { - fn new(source: EidSyncSource, outcome: EidCookieSyncOutcome) -> Self { - Self { - source, - outcome, - already_matched: u8::from(matches!(outcome, EidCookieSyncOutcome::AlreadyMatched)), - written: u8::from(matches!( - outcome, - EidCookieSyncOutcome::Written | EidCookieSyncOutcome::WrittenWithDeferredFreshness - )), - conflict_duplicate: u8::from(matches!(outcome, EidCookieSyncOutcome::ConflictMatched)), - deferred: u8::from(matches!( - outcome, - EidCookieSyncOutcome::WrittenWithDeferredFreshness - | EidCookieSyncOutcome::DeferredConflict - | EidCookieSyncOutcome::DeferredFreshness - | EidCookieSyncOutcome::DeferredStaleRead - )), +/// Creates the row of a newly generated EC and issues its cookie on success. +/// +/// Returns whether the row was created. +fn commit_generated( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) -> bool { + let Some(new_entry) = ec_context.take_pending_entry() else { + log::warn!("Skipping generated EC cookie because no pending row was built"); + return false; + }; + let Some(permit) = permit else { + log::info!("Skipping generated EC: not a page-load navigation"); + return false; + }; + let Some(change) = commit.plan( + ec_context, + commit.ec_id, + ec_context.kv_snapshot(), + Some(&new_entry), + ) else { + log::warn!("Skipping generated EC: no row change was planned"); + return false; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + set_ec_cookie_on_response(commit.settings, ec_context, response); + true + } + Err(report) => { + log::warn!( + "Page-load write failed for new EC '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + false } } } -fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome) { - let measurement = EidSyncMeasurement::new(source, outcome); - log::info!( - "EID sync measurement: source={} outcome={} attempted=1 already_matched={} written={} \ - conflict_duplicate={} deferred={}", - measurement.source, - measurement.outcome, - measurement.already_matched, - measurement.written, - measurement.conflict_duplicate, - measurement.deferred, - ); +/// Applies a returning EC's page-load write, or recovers an orphaned EC. +/// +/// Returns whether the row is live afterwards: written, recovered, or left +/// unchanged because there was nothing to write. +fn commit_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) -> bool { + let Some(permit) = permit else { + return false; + }; + // An unread snapshot is read only when this navigation has something to + // write, or when consent gating emptied real EID data on a request that + // may still need orphan recovery. A navigation with nothing to do costs + // no KV operation. + let has_changes = !commit.updates.is_empty() || !ec_context.lookup_results().is_empty(); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) + && (has_changes || (commit.had_eid_updates && ec_context.recovery_eligible())) + { + let snapshot = commit.graph.load_snapshot(commit.ec_id); + ec_context.set_kv_snapshot(snapshot); + } + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) { + return ec_context.recovery_eligible() + && confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); + } + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Failed { .. }) { + return false; + } + update_returning(commit, ec_context, permit) +} + +/// Writes the planned update for a returning EC whose row snapshot is held. +/// +/// Returns `false` only when a planned write failed. +fn update_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: PageLoadWrite, +) -> bool { + let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { + return true; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + true + } + Err(report) => { + log::info!( + "Page-load write skipped for '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + false + } + } } fn reconcile_pull_sync_marker( @@ -252,12 +303,11 @@ fn reconcile_pull_sync_marker( /// the outbound `/auction` bid request, evaluated here through /// [`allows_eid_persistence`]. Without this, a user who denies Purpose 4 /// would have their EIDs correctly stripped from the bid request but still -/// written to the identity graph from the `ts-eids`/`sharedId` cookies or the -/// `/auction` request body. +/// written to the identity graph from the `ts-eids`/`sharedId` cookies. fn gate_eid_updates_by_consent( - updates: Vec, + updates: Vec, consent: &ConsentContext, -) -> Vec { +) -> Vec { if updates.is_empty() || allows_eid_persistence(consent) { return updates; } @@ -268,75 +318,63 @@ fn gate_eid_updates_by_consent( Vec::new() } +/// Replaces a proven-orphaned EC with a new ID whose row is created by this +/// navigation's single write. One attempt: a collision leaves the orphan in +/// place for a later navigation. fn recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, ) { - // Snapshot the orphaned ID once so every fail-closed exit binds the failed - // snapshot to the same key. - let Some(orphan_id) = ec_context.ec_value().map(str::to_owned) else { - return; - }; + let orphan_id = commit.ec_id.to_owned(); let Some(client_ip) = ec_context.client_ip().map(str::to_owned) else { log::warn!("Orphan EC recovery skipped because client IP is unavailable"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); return; }; - - const MAX_RECOVERY_ATTEMPTS: usize = 5; - for _attempt in 0..MAX_RECOVERY_ATTEMPTS { - let ec_id = match generate_ec_id(settings, &client_ip) { - Ok(ec_id) => ec_id, - Err(err) => { - log::warn!("Orphan EC recovery ID generation failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } - }; - let mut entry = KvEntry::new( - ec_context.consent(), - ec_context.geo_info(), - current_timestamp(), - &settings.publisher.domain, - ); - entry.device = ec_context - .device_signals() - .map(super::device::DeviceSignals::to_kv_device); - apply_partner_id_updates(&mut entry, updates); - - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - ec_context.replace_with_generated(ec_id, snapshot); - set_ec_cookie_on_response(settings, ec_context, response); - return; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => continue, - Err(err) => { - log::warn!("Orphan EC recovery failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } + let new_id = match generate_ec_id(commit.settings, &client_ip) { + Ok(ec_id) => ec_id, + Err(err) => { + log::warn!("Orphan EC recovery ID generation failed: {err:?}"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); + return; + } + }; + let mut new_entry = KvEntry::new( + ec_context.consent(), + ec_context.geo_info(), + current_timestamp(), + &commit.settings.publisher.domain, + ); + new_entry.device = ec_context + .device_signals() + .map(super::device::DeviceSignals::to_kv_device); + + // Spec: the recovered row is created with this navigation's provider + // results, which were looked up under the orphaned ID. + ec_context.retarget_orphan_lookups(&orphan_id, &new_id); + let missing = EcKvSnapshot::Missing { + ec_id: new_id.clone(), + }; + let Some(change) = commit.plan(ec_context, &new_id, &missing, Some(&new_entry)) else { + log::warn!("Orphan EC recovery skipped: no row change was planned"); + return; + }; + match commit.graph.commit_page_load(permit, &new_id, change) { + Ok(committed) => { + let snapshot = present_snapshot(&new_id, committed); + ec_context.replace_with_generated(new_id, snapshot); + set_ec_cookie_on_response(commit.settings, ec_context, response); + } + Err(report) => { + log::warn!( + "Orphan EC recovery write failed: {}", + report.current_context() + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); } } - - log::warn!("Orphan EC recovery exhausted collision retries"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); } /// Proves an orphaned cookie is genuinely absent before rotating it. @@ -348,8 +386,8 @@ fn recover_orphaned_ec( /// POP that has not converged. Two point reads do not fix that — both can be /// stale — so absence has to be *proved*, not observed twice: /// -/// - a row that became visible after the origin round trip is adopted, with any -/// pending updates merged, and is never rotated; +/// - a row that became visible after the origin round trip is adopted, and the +/// navigation's single write is planned against it; it is never rotated; /// - a second miss is escalated to /// [`key_exists_confirmed`](KvIdentityGraph::key_exists_confirmed), which /// reads the primary data source. Only a proven-absent key rotates; @@ -357,37 +395,44 @@ fn recover_orphaned_ec( /// the identity stays intact and recovery is retried on a later navigation; /// - neither a read failure nor a failed existence check is a miss, and neither /// rotates. +/// +/// Returns whether the navigation's write succeeded or had nothing to change, +/// which is what allows the after-response follow-up write. fn confirm_then_recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - ec_id: &str, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, -) { - let confirmed = graph.load_snapshot(ec_id); +) -> bool { + let confirmed = commit.graph.load_snapshot(commit.ec_id); match confirmed { EcKvSnapshot::Present { .. } => { // The row became visible after the origin round trip: adopt it and - // merge any pending updates rather than rotating a valid identity. + // plan the navigation's write against it rather than rotating a + // valid identity. ec_context.set_kv_snapshot(confirmed); - sync_eid_cookie_updates(graph, ec_context, ec_id, updates, EidSyncSource::Navigation); + update_returning(commit, ec_context, permit) } - EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { - Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), + EcKvSnapshot::Missing { .. } => match commit.graph.key_exists_confirmed(commit.ec_id) { + Ok(false) => { + recover_orphaned_ec(commit, ec_context, permit, response); + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }) + } Ok(true) => { log::warn!( "Orphan EC recovery skipped: both point reads missed a row the store still \ lists; leaving the identity intact for a later navigation" ); + false } Err(err) => { log::warn!("Orphan EC recovery skipped: existence check failed: {err:?}"); + false } }, // A failed or not-read confirmation is not an authoritative miss: leave // the existing snapshot in place and do not rotate an unconfirmed miss. - EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => {} + EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => false, } } @@ -439,10 +484,17 @@ pub fn clear_ec_on_response(settings: &Settings, response: &mut Response HashSet { - let mut hashes = HashSet::new(); - - if let Some(cookie_ec_id) = ec_context.existing_cookie_ec_id() - && is_valid_ec_id(cookie_ec_id) - { - hashes.insert(cookie_ec_id.to_owned()); + let mut complete = true; + for ec_id in withdrawal_ec_ids(ec_context) { + // Only the active ID has a snapshot in the context to start from and + // to correct, so post-send work sees the tombstone just written. + let initial = if ec_context.kv_snapshot().belongs_to(&ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead + }; + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, initial); + if matches!(outcome, EcKvSnapshot::Failed { .. }) { + log::warn!( + "EC withdrawal tombstone for '{}' failed; keeping the cookie so the next \ + request retries", + log_id(&ec_id) + ); + complete = false; + } + if ec_context.ec_value() == Some(ec_id.as_str()) { + ec_context.set_kv_snapshot(outcome); + } } - - if let Some(active_ec_id) = ec_context.ec_value() - && is_valid_ec_id(active_ec_id) - { - hashes.insert(active_ec_id.to_owned()); + if complete { + expire_ec_cookie(settings, response); } - - hashes } -fn apply_withdrawal_tombstones(ec_ids: &HashSet, mut write_tombstone: F) -where - F: FnMut(&str), -{ - for ec_id in ec_ids { - write_tombstone(ec_id); - } +/// The EC IDs a withdrawal applies to: the cookie's and the active one. +fn withdrawal_ec_ids(ec_context: &EcContext) -> BTreeSet { + [ec_context.existing_cookie_ec_id(), ec_context.ec_value()] + .into_iter() + .flatten() + .filter(|ec_id| is_valid_ec_id(ec_id)) + .map(str::to_owned) + .collect() } #[cfg(test)] @@ -531,6 +561,13 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::EidSyncSource; + use crate::ec::coordinator::results::{ + DropCounts, LookupResults, LookupStatus, ProviderLookup, + }; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv_types::{KvEid, KvUid}; + use crate::ec::page_load::NavigationPermit; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -620,90 +657,6 @@ mod tests { } } - #[test] - fn withdrawal_ec_ids_returns_cookie_ec_only_when_active_missing() { - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context(None, Some(&cookie_ec), true, false, Jurisdiction::Unknown); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should include exactly one EC ID"); - assert!( - ids.contains(&cookie_ec), - "should include the cookie EC value" - ); - } - - #[test] - fn withdrawal_ec_ids_deduplicates_matching_cookie_and_active_ec() { - let ec_id = sample_ec_id("same01"); - let ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should deduplicate identical EC IDs"); - assert!(ids.contains(&ec_id), "should retain the shared EC ID"); - } - - #[test] - fn withdrawal_ec_ids_includes_both_cookie_and_active_when_different() { - let active_ec = sample_ec_id("activ1"); - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context( - Some(&active_ec), - Some(&cookie_ec), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 2, "should include both distinct EC IDs"); - assert!(ids.contains(&active_ec), "should include active EC ID"); - assert!(ids.contains(&cookie_ec), "should include cookie EC ID"); - } - - #[test] - fn withdrawal_ec_ids_filters_invalid_values() { - let valid_ec = sample_ec_id("valid1"); - let ec_context = make_context( - Some(&valid_ec), - Some("not-an-ec-id"), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should ignore malformed EC values"); - assert!(ids.contains(&valid_ec), "should keep the valid EC ID"); - } - - #[test] - fn apply_withdrawal_tombstones_invokes_writer_for_each_ec_id() { - let first = sample_ec_id("first1"); - let second = sample_ec_id("second"); - let mut ids = HashSet::new(); - ids.insert(first.clone()); - ids.insert(second.clone()); - - let mut written = Vec::new(); - apply_withdrawal_tombstones(&ids, |ec_id| written.push(ec_id.to_owned())); - written.sort(); - - let mut expected = vec![first, second]; - expected.sort(); - assert_eq!(written, expected, "should write a tombstone for each EC ID"); - } - #[test] fn clear_ec_on_response_removes_headers_and_expires_cookie() { let settings = create_test_settings(); @@ -755,6 +708,7 @@ mod tests { let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); let kv = KvIdentityGraph::in_memory("test-store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -807,6 +761,7 @@ mod tests { ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. } )); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -834,13 +789,17 @@ mod tests { .expect("should replace the stale miss with a tombstone snapshot"); assert!(!snapshot_entry.consent.ok && snapshot_entry.ids.is_empty()); assert_eq!(ec_context.kv_snapshot().generation_for(&ec_id), None); + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups after a withdrawal" + ); } #[test] - fn withdrawal_still_expires_the_cookie_when_the_store_is_unavailable() { - // Cookie expiry is the primary enforcement, so it has to survive a - // store that cannot answer at all — the case where the identity-graph - // marker is exactly what goes missing. + fn withdrawal_keeps_the_cookie_when_the_tombstone_write_fails() { + // A failed tombstone write is not retried. The EC cookie is kept so the + // next navigation can still write the tombstone; clearing it would + // orphan a live row (spec §9.2). let settings = create_test_settings(); let ec_id = sample_ec_id("dead01"); let consent = ConsentContext { @@ -856,6 +815,7 @@ mod tests { entry: Box::new(live_entry()), generation: Some(1), }); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let kv = KvIdentityGraph::failing("test-store"); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); @@ -871,10 +831,9 @@ mod tests { &mut response, ); - let set_cookie = get_header_str(&response, "set-cookie").unwrap_or_default(); assert!( - set_cookie.contains("Max-Age=0"), - "should expire the EC cookie even when the store is unavailable: {set_cookie}" + get_header(&response, "set-cookie").is_none(), + "should keep the EC cookie when the tombstone write fails" ); assert!( get_header(&response, "x-ts-ec").is_none(), @@ -898,6 +857,7 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); set_header(&mut response, "x-ts-eids", "[]"); @@ -1046,6 +1006,72 @@ mod tests { ); } + #[test] + fn finalize_recovery_writes_lookups_made_for_the_orphan() { + let settings = create_test_settings(); + let orphaned_ec = sample_ec_id("orphn2"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip( + Some(orphaned_ec.clone()), + consent, + Some("192.0.2.10".to_owned()), + ); + ec_context.set_recovery_eligible(true); + ec_context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: orphaned_ec.clone(), + }); + let uid = KvUid::new( + UidValue::try_from("provider-uid-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + ec_context.set_lookup_results(LookupResults::new( + &orphaned_ec, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("provider.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + )); + let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let replacement = ec_context.ec_value().expect("should rotate orphan"); + assert_ne!(replacement, orphaned_ec, "should mint a replacement EC ID"); + let (entry, _) = graph + .get(replacement) + .expect("should read replacement") + .expect("should create replacement row"); + assert!( + entry.ids.contains_key("provider.example.com"), + "should write the orphan's provider results to the replacement row" + ); + } + #[test] fn finalize_rotates_orphaned_cookie_to_new_backed_ec() { let settings = create_test_settings(); @@ -1065,6 +1091,7 @@ mod tests { ec_id: orphaned_ec.clone(), }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1115,17 +1142,18 @@ mod tests { ); ec_context.set_recovery_eligible(true); ec_context.set_eid_sync_source(EidSyncSource::Navigation); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1181,6 +1209,7 @@ mod tests { partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1240,7 +1269,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("sharedid.org"), "a subresource response must not persist request EID cookies" @@ -1248,10 +1277,22 @@ mod tests { } #[test] - fn finalize_navigation_routes_persist_returning_user_eid_updates() { - for (source, suffix, cookie_id) in [ - (EidSyncSource::Navigation, "naveid", "navigation-cookie-id"), - (EidSyncSource::PageBids, "spaeid", "page-bids-cookie-id"), + fn finalize_persists_returning_user_eid_updates_only_on_navigation() { + // `/_ts/page-bids` belongs to a page load whose navigation already + // made the write, so it never receives a permit (spec D1). + for (source, suffix, cookie_id, permit) in [ + ( + EidSyncSource::Navigation, + "naveid", + "navigation-cookie-id", + true, + ), + ( + EidSyncSource::PageBids, + "spaeid", + "page-bids-cookie-id", + false, + ), ] { let settings = create_test_settings(); let ec_id = sample_ec_id(suffix); @@ -1265,7 +1306,9 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + // A recovery-eligible context is a navigation and carries a permit. + let mut ec_context = + returning_user_context(&ec_id, graph.load_snapshot(&ec_id), permit); ec_context.set_eid_sync_source(source); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1284,11 +1327,11 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some(cookie_id), - "{source} should persist the returning-user EID cookie" + permit.then_some(cookie_id), + "{source} should persist the returning-user EID cookie only with a permit" ); } } @@ -1296,20 +1339,17 @@ mod tests { #[test] fn finalize_generated_ec_persists_eid_updates() { let settings = create_test_settings(); - let ec_id = sample_ec_id("geneid"); let graph = KvIdentityGraph::in_memory("test_store"); - let live = KvEntry::new( - &granting_consent(), + let mut ec_context = EcContext::new_for_test_with_ip( None, - current_timestamp(), - &settings.publisher.domain, + granting_consent(), + Some("192.0.2.10".to_owned()), ); - graph - .create(&ec_id, &live) - .expect("should seed generated row"); - let mut ec_context = - make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); - ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1327,114 +1367,18 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should create the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("generated-cookie-id") - ); - } - - #[test] - fn eid_sync_measurement_dimensions_are_bounded_and_identity_free() { - let sources = [ - EidSyncSource::Navigation, - EidSyncSource::Auction, - EidSyncSource::PageBids, - EidSyncSource::NewEc, - ]; - let outcomes = [ - EidCookieSyncOutcome::AlreadyMatched, - EidCookieSyncOutcome::Written, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - EidCookieSyncOutcome::ConflictMatched, - EidCookieSyncOutcome::DeferredConflict, - EidCookieSyncOutcome::DeferredFreshness, - EidCookieSyncOutcome::DeferredStaleRead, - EidCookieSyncOutcome::Missing, - EidCookieSyncOutcome::ConsentWithdrawn, - EidCookieSyncOutcome::Failed, - ]; - - assert_eq!( - sources.map(|source| source.to_string()), - ["navigation", "auction", "page_bids", "new_ec"] - ); - assert_eq!( - outcomes.map(|outcome| outcome.to_string()), - [ - "already_matched", - "written", - "written_with_deferred_freshness", - "conflict_matched", - "deferred_conflict", - "deferred_freshness", - "deferred_stale_read", - "missing", - "consent_withdrawn", - "failed", - ] - ); - - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Navigation, - EidCookieSyncOutcome::AlreadyMatched, - ), - EidSyncMeasurement { - source: EidSyncSource::Navigation, - outcome: EidCookieSyncOutcome::AlreadyMatched, - already_matched: 1, - written: 0, - conflict_duplicate: 0, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Auction, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - ), - EidSyncMeasurement { - source: EidSyncSource::Auction, - outcome: EidCookieSyncOutcome::WrittenWithDeferredFreshness, - already_matched: 0, - written: 1, - conflict_duplicate: 0, - deferred: 1, - } - ); - assert_eq!( - EidSyncMeasurement::new(EidSyncSource::NewEc, EidCookieSyncOutcome::ConflictMatched,), - EidSyncMeasurement { - source: EidSyncSource::NewEc, - outcome: EidCookieSyncOutcome::ConflictMatched, - already_matched: 0, - written: 0, - conflict_duplicate: 1, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::PageBids, - EidCookieSyncOutcome::DeferredStaleRead, - ), - EidSyncMeasurement { - source: EidSyncSource::PageBids, - outcome: EidCookieSyncOutcome::DeferredStaleRead, - already_matched: 0, - written: 0, - conflict_duplicate: 0, - deferred: 1, - } + Some("generated-cookie-id"), + "should ingest the browser EID in the create" ); } #[test] - fn finalize_auction_transient_miss_still_persists_eid_updates() { - // `/auction` saves its first lookup into the context and is never - // recovery eligible, so a stale miss there has no later chance to - // retry. Finalization must revalidate before dropping collected IDs. + fn finalize_auction_transient_miss_does_not_persist_eid_updates() { + // `/auction` never writes KV (spec D1): browser EIDs are persisted only + // by the page load's navigation write, whatever its snapshot says. let settings = create_test_settings(); let ec_id = sample_ec_id("named1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1473,21 +1417,18 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "a stale endpoint miss must not suppress EID persistence" + .expect("should keep the row"); + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[test] - fn finalize_persists_every_configured_partner_from_client_eids_over_a_trimmed_cookie() { - // Regression test for #1184: the `ts-eids` cookie only carries what - // the browser could fit under its size cap, but `/auction` also - // hands finalization the full EID set from the request body via - // `EcContext::set_client_eids`. That full set must land in KV even - // when the cookie alone would have dropped a configured partner. + fn finalize_never_persists_auction_body_eids() { + // `/auction` body EIDs go to bidders on that request only and are + // never persisted (spec §5.1); neither is the `ts-eids` cookie on a + // non-navigation request. let settings = create_test_settings(); let ec_id = sample_ec_id("cleids1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1517,22 +1458,22 @@ mod tests { .expect("should serialize test cookie payload"), ); ec_context.set_client_eids(vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_from_body".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_from_body".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]); let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1551,16 +1492,10 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should exist after ingestion"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the request body's EID should win over the cookie's stale value" - ); - assert_eq!( - stored.ids.get("liveramp.com").map(|id| id.uid.as_str()), - Some("LR_from_body"), - "a partner the cookie trimmed must still be ingested from the body" + .expect("should keep the row"); + assert!( + stored.ids.is_empty(), + "should persist neither the auction body nor the cookie EIDs" ); } @@ -1591,18 +1526,17 @@ mod tests { false, purpose1_only_consent, ); - ec_context.set_kv_snapshot(EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }); - ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1620,7 +1554,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("id5-sync.com"), "denying TCF Purpose 4 must keep the EID out of KV even though Purpose 1 \ @@ -1628,6 +1562,115 @@ mod tests { ); } + #[test] + fn finalize_withholds_provider_owned_fallback_when_purpose_four_is_denied() { + // Arrange + let settings = create_test_settings(); + let ec_id = sample_ec_id("purp4o"); + let graph = KvIdentityGraph::in_memory("test_store"); + let purpose1_only_consent = purpose_one_only_consent(); + let live = KvEntry::new( + &purpose1_only_consent, + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row this request updates"); + let mut ec_context = EcContext::new_for_test_with_cookie( + Some(ec_id.clone()), + Some(ec_id.clone()), + true, + false, + purpose1_only_consent, + ); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { + id: "ID5_should_not_persist".to_owned(), + atype: Some(1), + ext: None, + }], + )]); + let mut registry = PartnerRegistry::from_config(&[make_partner("id5-sync.com")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("id5-sync.com", None), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should keep the row"); + assert!( + !stored.ids.contains_key("id5-sync.com"), + "should keep the provider-owned fallback out of KV without Purpose 4" + ); + } + + #[test] + fn finalize_skips_browser_uid_for_provider_owned_source() { + // Arrange + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut registry = PartnerRegistry::from_config(&[make_partner("sharedid.org")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("sharedid.org", None), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("fallback-cookie-id"), + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should create the row"); + assert!( + !stored.ids.contains_key("sharedid.org"), + "should not store a browser UID for a provider-owned source" + ); + } + #[test] fn finalize_auction_confirmed_miss_does_not_create_a_row() { // The same path with a genuinely absent row must stay a no-op: a route @@ -1870,6 +1913,7 @@ mod tests { ec_id: generated_ec, }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1978,6 +2022,11 @@ mod tests { Some("192.0.2.10".to_owned()), ); ec.set_recovery_eligible(recovery_eligible); + if recovery_eligible { + // Only a top-level navigation is recovery eligible, and only a + // navigation carries the page-load write permit. + ec.set_navigation_permit(NavigationPermit::for_test()); + } ec.set_kv_snapshot(snapshot); ec } @@ -2121,6 +2170,7 @@ mod tests { .create(&active_ec, &live_entry()) .expect("should seed active row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2148,7 +2198,7 @@ mod tests { } #[test] - fn finalize_withdrawal_tombstones_both_present_ids_once() { + fn finalize_withdrawal_tombstones_the_active_and_cookie_ids_once() { let settings = create_test_settings(); let active_ec = sample_ec_id("activ3"); let cookie_ec = sample_ec_id("cook3e"); @@ -2174,6 +2224,7 @@ mod tests { ) .expect("should seed cookie row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2204,13 +2255,10 @@ mod tests { ); assert!( !cookie_tombstone.consent.ok, - "cookie row should be withdrawn" - ); - assert!( - cookie_tombstone.ids.is_empty(), - "cookie IDs should be cleared" + "the cookie's row should be withdrawn too" ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut repeated_response = empty_response(); ec_finalize_response( &settings, @@ -2238,12 +2286,12 @@ mod tests { .expect("should retain cookie tombstone") .1, cookie_generation, - "repeated finalization should not rewrite cookie tombstone" + "repeated finalization should not rewrite the cookie tombstone" ); } #[test] - fn finalize_withdrawal_keeps_cookie_deletion_on_kv_failure() { + fn finalize_withdrawal_keeps_ec_cookie_but_expires_marker_on_kv_failure() { let settings = create_test_settings(); let ec_id = sample_ec_id("failw1"); let consent = ConsentContext { @@ -2257,6 +2305,7 @@ mod tests { ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2282,10 +2331,8 @@ mod tests { "KV failure should not change response status" ); assert!( - cookies - .iter() - .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), - "KV failure should not prevent EC cookie deletion" + cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), + "KV failure should keep the EC cookie so a later navigation can tombstone the row" ); assert!( cookies.iter().any(|cookie| { @@ -2315,9 +2362,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "partner-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("partner-uid".to_owned()), ); ec_context.set_kv_snapshot(EcKvSnapshot::Present { ec_id, @@ -2421,9 +2466,436 @@ mod tests { ); } + fn registry_for(source: &str) -> PartnerRegistry { + PartnerRegistry::from_config(&[make_partner(source)]).expect("should build registry") + } + + fn eids_cookie(source: &str, uid: &str) -> String { + base64::engine::general_purpose::STANDARD.encode( + serde_json::json!([{ "source": source, "uids": [{ "id": uid, "atype": 1 }] }]) + .to_string(), + ) + } + + #[test] + fn auction_request_never_writes_browser_eids() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("auc001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Auction); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!( + entry.ids.is_empty(), + "should not write from a non-navigation request" + ); + } + + #[test] + fn navigation_writes_browser_eids_once() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("nav001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert_eq!( + entry.ids["partner.example.com"].uid, "u-1", + "should persist browser uid" + ); + assert!( + ec_context.take_navigation_permit().is_none(), + "should spend the permit" + ); + } + + #[test] + fn redirect_navigation_does_not_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("red001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!(entry.ids.is_empty(), "should not write on a redirect"); + } + + #[test] + fn generated_ec_without_permit_gets_no_cookie_or_row() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_none(), + "should not create without a permit" + ); + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue the EC" + ); + } + + #[test] + fn generated_ec_with_permit_creates_row_and_sets_cookie() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_some(), + "should create the row" + ); + assert!( + get_header(&response, "set-cookie").is_some(), + "should issue the EC" + ); + } + + #[test] + fn generated_ec_collision_is_not_retried_or_issued() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed a colliding row"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue an EC whose row was not created" + ); + } + + fn withdrawing_context(ec_id: &str) -> EcContext { + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + make_context_with_consent(Some(ec_id), Some(ec_id), true, false, consent) + } + + fn assert_withdrawn(graph: &KvIdentityGraph, ec_id: &str, response: &Response) { + let (entry, _) = graph + .get(ec_id) + .expect("should read") + .expect("should keep the tombstone"); + assert!(!entry.consent.ok, "should tombstone the row"); + assert!( + get_header_str(response, "set-cookie") + .is_some_and(|cookie| cookie.contains("Max-Age=0")), + "should expire the EC cookie" + ); + } + + #[test] + fn withdrawal_without_permit_tombstones_row_and_expires_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + let mut response = empty_response(); + set_header(&mut response, "x-ts-ec", "stale"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + assert!( + get_header(&response, "x-ts-ec").is_none(), + "should still strip EC response headers" + ); + } + + #[test] + fn withdrawal_on_a_redirect_tombstones_row_and_expires_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0002"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + } + + #[test] + fn withdrawal_retries_after_a_concurrent_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0003"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + graph + .upsert_partner_id(&ec_id, "partner.example.com", "late-uid") + .expect("should write concurrently after the snapshot"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; entry } + + #[test] + fn finalize_withholds_after_response_lookups_when_the_confirmed_write_fails() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after3"); + let graph = KvIdentityGraph::write_failing_with( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ); + // The preload missed a row the confirmation read then finds, and the + // navigation's write against it fails. + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + true, + ); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow a follow-up write after the page-load write failed" + ); + } + + #[test] + fn finalize_allows_after_response_lookups_on_a_live_row() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after1"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ) + .expect("should seed the identity"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_some(), + "should allow after-response lookups when the page write had nothing to change" + ); + } + + #[test] + fn finalize_withholds_after_response_lookups_when_the_row_read_failed() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after2"); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups when the page phase failed" + ); + } } diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 94464d1a57..1c217e09e2 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -11,14 +11,17 @@ use url::Url; use super::auth::authenticate_bearer; use super::consent::ec_consent_granted; +use crate::consent::tcf_vendor_consent_denied; use crate::error::TrustedServerError; -use crate::openrtb::{Eid, Uid}; +use crate::openrtb::Eid; use crate::settings::Settings; -use super::EcContext; +use super::eid_types::UnixSeconds; +use super::eids::source_views; use super::kv::KvIdentityGraph; use super::log_id; use super::registry::PartnerRegistry; +use super::{EcContext, current_timestamp}; /// Handles `GET /_ts/api/v1/identify`. /// @@ -84,7 +87,7 @@ pub fn handle_identify( }; let mut degraded = false; - let mut uid: Option = None; + let mut eid: Option = None; let mut cluster_size: Option = None; match kv.get(ec_id) { @@ -95,12 +98,22 @@ pub fn handle_identify( // write back with the live-entry TTL. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { - // Extract only this partner's UID. - if let Some(partner_uid) = entry.ids.get(&partner.source_domain) - && !partner_uid.uid.is_empty() - { - uid = Some(partner_uid.uid.clone()); - } + // Extract only this partner's first live entry. A + // provider-owned source is withheld while its owner lacks TCF + // vendor consent, as in the bidstream. + let now = UnixSeconds::new(current_timestamp()); + let vendor_denied = partner + .owner_tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(ec_context.consent(), vendor)); + eid = entry + .ids + .get(&partner.source_domain) + .filter(|_| !vendor_denied) + .and_then(|record| { + source_views(&partner.source_domain, record, partner.openrtb_atype, now) + .next() + }) + .map(Eid::from); // Evaluate cluster size lazily for identify responses. Existing // stored cluster_size values are reused without a prefix-list call. @@ -124,14 +137,10 @@ pub fn handle_identify( } } - let eid = uid.as_ref().map(|u| Eid { - source: partner.source_domain.clone(), - uids: vec![Uid { - id: u.clone(), - atype: Some(partner.openrtb_atype), - ext: None, - }], - }); + let uid = eid + .as_ref() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id.clone()); let body = IdentifyResponse { ec: ec_id.to_owned(), @@ -334,10 +343,13 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{consent_granting_vendors, create_test_settings}; + use std::num::NonZeroU16; const VALID_API_TOKEN: &str = "identify-test-token-32-bytes-min"; @@ -756,4 +768,115 @@ mod tests { "should vary on identity request inputs for preflight" ); } + + fn provider_record(id: &str, expiry: Expiry) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, vec![uid])], UnixSeconds::new(0)) + .expect("should build record") + } + + fn identify_body_for_record(record: KvPartnerId) -> serde_json::Value { + let registry = + PartnerRegistry::from_config(&[make_test_partner("ssp.example.com", VALID_API_TOKEN)]) + .expect("should build registry"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..ConsentContext::default() + }; + identify_body_with(record, ®istry, consent) + } + + fn identify_body_with( + record: KvPartnerId, + registry: &PartnerRegistry, + consent: ConsentContext, + ) -> serde_json::Value { + let settings = create_test_settings(); + let kv = KvIdentityGraph::in_memory("identify_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); + entry.ids.insert("ssp.example.com".to_owned(), record); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + let req = Request::builder() + .method("GET") + .uri("https://edge.example.com/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + let ec_context = EcContext::new_for_test(Some(ec_id), consent); + + let response = handle_identify(&settings, &kv, registry, &req, &ec_context) + .expect("should construct identify response"); + + assert_eq!(response.status(), StatusCode::OK, "should return 200"); + serde_json::from_slice::( + &response.into_body().into_bytes().unwrap_or_default(), + ) + .expect("should decode identify response JSON") + } + + #[test] + fn handle_identify_withholds_a_source_whose_owner_lacks_vendor_consent() { + let mut registry = + PartnerRegistry::from_config(&[make_test_partner("ssp.example.com", VALID_API_TOKEN)]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("ssp.example.com", NonZeroU16::new(9999)), + "should mark the source provider-owned" + ); + let record = || provider_record("live", Expiry::Never); + + let denied = identify_body_with(record(), ®istry, consent_granting_vendors(vec![1])); + let granted = identify_body_with(record(), ®istry, consent_granting_vendors(vec![9999])); + + assert!( + denied.get("uid").is_none(), + "should withhold the UID without the owner's vendor consent" + ); + assert_eq!( + granted["uid"], "live", + "should return the UID with the owner's vendor consent" + ); + } + + #[test] + fn handle_identify_omits_expired_provider_uid() { + let body = + identify_body_for_record(provider_record("old", Expiry::At(UnixSeconds::new(1)))); + + assert!( + body.get("uid").is_none(), + "should omit expired provider uid" + ); + assert!( + body.get("eid").is_none(), + "should omit eid without a live uid" + ); + } + + #[test] + fn handle_identify_returns_live_provider_eid_with_eid_fields() { + let body = identify_body_for_record(provider_record("live", Expiry::Never)); + + assert_eq!(body["uid"], "live", "should return live provider uid"); + assert_eq!( + body["eid"]["inserter"], "inserter.example.com", + "should keep provider EID-level fields" + ); + assert_eq!( + body["eid"]["uids"][0]["atype"], 1, + "should use the provider atype" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0eb83d3fa..3d2c40f10c 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,8 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; -use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; +use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; +use super::page_load::{AfterResponseWrite, Generation, KvWriteError, PageLoadWrite, RowChange}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -59,29 +60,31 @@ pub enum UpsertResult { ConsentWithdrawn, /// The partner ID already had the requested UID, so no write was needed. Unchanged, + /// The row with this partner ID would exceed [`MAX_ENTRY_BYTES`], so + /// nothing was written. Retrying gives the same result. + RowFull, } -/// Outcome of atomically creating an identity-graph root when absent. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CreateIfAbsentOutcome { - /// The candidate entry was persisted. - Written, - /// A row already exists for the candidate key. - AlreadyExists, +/// A serialized entry, or the size that kept it from being written. +enum SerializedEntry { + /// Body and metadata ready to write. + Fits { body: String, meta: String }, + /// The body is this many bytes, over [`MAX_ENTRY_BYTES`]. + TooLarge(usize), } /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PartnerIdUpdate { - /// Partner namespace key in [`KvEntry::ids`]. - pub(crate) partner_id: String, - /// Partner-scoped user ID value. - pub(crate) uid: String, +pub struct PartnerIdUpdate { + /// Source domain key in [`KvEntry::ids`]. + pub partner_id: String, + /// UID to fill in. + pub uid: String, } impl PartnerIdUpdate { /// Creates a partner UID update. - pub(crate) fn new(partner_id: impl Into, uid: impl Into) -> Self { + pub fn new(partner_id: impl Into, uid: impl Into) -> Self { Self { partner_id: partner_id.into(), uid: uid.into(), @@ -89,104 +92,6 @@ impl PartnerIdUpdate { } } -/// Terminal result of one browser EID-cookie persistence attempt. -#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] -pub(crate) enum EidCookieSyncOutcome { - /// The stored values already matched without a write. - #[display("already_matched")] - AlreadyMatched, - /// The one conditional write succeeded. - #[display("written")] - Written, - /// The write added missing IDs while deferring different values of unknown freshness. - #[display("written_with_deferred_freshness")] - WrittenWithDeferredFreshness, - /// A conflicting writer persisted every desired value. - #[display("conflict_matched")] - ConflictMatched, - /// A conflict left at least one desired value absent or different. - #[display("deferred_conflict")] - DeferredConflict, - /// A different stored value had unknown freshness. - #[display("deferred_freshness")] - DeferredFreshness, - /// An eventually consistent read missed a row already proven to exist. - #[display("deferred_stale_read")] - DeferredStaleRead, - /// The identity graph row was missing. - #[display("missing")] - Missing, - /// Consent had been withdrawn in the identity graph. - #[display("consent_withdrawn")] - ConsentWithdrawn, - /// KV or serialization failed. - #[display("failed")] - Failed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CookieUpdateApplication { - AlreadyMatched, - Changed { deferred: bool }, - DeferredFreshness, -} - -fn apply_cookie_partner_id_updates( - entry: &mut KvEntry, - updates: &[PartnerIdUpdate], -) -> CookieUpdateApplication { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - let mut changed = false; - let mut deferred = false; - for (partner_id, uid) in latest_updates { - match entry.ids.get(partner_id) { - Some(existing) if existing.uid == uid => continue, - // Browser EID cookies carry no value-owned sequence or timestamp. - // A different value therefore has unknown freshness and must not - // replace the stored value. - Some(_) => { - deferred = true; - continue; - } - None => {} - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, - ); - changed = true; - } - - if changed { - CookieUpdateApplication::Changed { deferred } - } else if deferred { - CookieUpdateApplication::DeferredFreshness - } else { - CookieUpdateApplication::AlreadyMatched - } -} - -fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - latest_updates.into_iter().all(|(partner_id, uid)| { - entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.uid == uid) - }) -} - pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -205,9 +110,7 @@ pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerId entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); changed = true; } @@ -269,10 +172,30 @@ impl KvIdentityGraph { } /// Serializes an entry body and metadata for insertion. + /// + /// A body over [`MAX_ENTRY_BYTES`] is an error; callers that report a + /// full row separately use [`Self::serialize_entry_unless_full`]. fn serialize_entry( entry: &KvEntry, store_name: &str, ) -> Result<(String, String), Report> { + match Self::serialize_entry_unless_full(entry, store_name)? { + SerializedEntry::Fits { body, meta } => Ok((body, meta)), + SerializedEntry::TooLarge(bytes) => Err(Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Refusing to write KV entry of {bytes} bytes (limit {MAX_ENTRY_BYTES})" + ), + })), + } + } + + /// Serializes an entry body and metadata, reporting an oversized body as + /// [`SerializedEntry::TooLarge`] rather than an error. + fn serialize_entry_unless_full( + entry: &KvEntry, + store_name: &str, + ) -> Result> { entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -284,13 +207,19 @@ impl KvIdentityGraph { store_name: store_name.to_owned(), message: "Failed to serialize KV entry body".to_owned(), })?; + if body.len() > MAX_ENTRY_BYTES { + return Ok(SerializedEntry::TooLarge(body.len())); + } let meta = KvMetadata::from_entry(entry); let meta_str = serde_json::to_string(&meta).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: "Failed to serialize KV entry metadata".to_owned(), })?; - Ok((body, meta_str)) + Ok(SerializedEntry::Fits { + body, + meta: meta_str, + }) } /// Reads the raw stored body, metadata, and generation for an EC ID key. @@ -357,12 +286,20 @@ impl KvIdentityGraph { ec_id: &str, body_bytes: &[u8], ) -> Result> { - let entry: KvEntry = + let mut entry: KvEntry = serde_json::from_slice(body_bytes).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: format!("Failed to deserialize entry for key '{}'", log_id(ec_id)), })?; + let dropped = entry.drop_unreadable_ids(); + if dropped > 0 { + log::warn!( + "Dropped {dropped} unreadable partner IDs loading key '{}'", + log_id(ec_id) + ); + } + entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -423,23 +360,6 @@ impl KvIdentityGraph { } } - /// Atomically creates an entry while preserving collision as normal control flow. - /// - /// # Errors - /// - /// Returns [`TrustedServerError::KvStore`] when serialization or store I/O fails. - pub fn create_if_absent( - &self, - ec_id: &str, - entry: &KvEntry, - ) -> Result> { - let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; - match self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? { - EcKvWriteOutcome::Written => Ok(CreateIfAbsentOutcome::Written), - EcKvWriteOutcome::PreconditionFailed => Ok(CreateIfAbsentOutcome::AlreadyExists), - } - } - /// Low-level write with shared error context. fn write_entry( &self, @@ -460,6 +380,85 @@ impl KvIdentityGraph { ) } + /// Applies the navigation's single row change in one attempt. + /// + /// The `permit` is consumed, so a navigation cannot write twice. Consent + /// withdrawals do not go through here: their tombstone is a privacy write + /// on any request ([`Self::tombstone_existing_from_snapshot`]). + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, [`KvWriteError::AlreadyExists`] when a create finds a row, or + /// [`KvWriteError::Store`] on a serialization or store failure, including + /// a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result> { + permit.spend(); + let (entry, mode) = match change { + RowChange::Create(entry) => (entry, EcKvWriteMode::Add), + RowChange::Update { entry, expected } => { + (entry, EcKvWriteMode::IfGenerationMatch(expected.get())) + } + }; + self.write_row(ec_id, &entry, ENTRY_TTL, mode)?; + Ok(entry) + } + + /// Applies a navigation's follow-up update after its response was sent. + /// + /// The row is replaced only while it is still at `expected`, so a + /// concurrent write is never overwritten. The `permit` is consumed. + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, or [`KvWriteError::Store`] on a serialization or store failure, + /// including a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_after_response( + &self, + permit: AfterResponseWrite, + ec_id: &str, + entry: &KvEntry, + expected: Generation, + ) -> Result<(), Report> { + permit.spend(); + self.write_row( + ec_id, + entry, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ) + } + + /// Serializes `entry` and writes it with `mode`, mapping a failed + /// precondition to [`KvWriteError::AlreadyExists`] for a create and + /// [`KvWriteError::Conflict`] otherwise. + fn write_row( + &self, + ec_id: &str, + entry: &KvEntry, + ttl: Duration, + mode: EcKvWriteMode, + ) -> Result<(), Report> { + let (body, metadata) = + Self::serialize_entry(entry, self.store_name()).change_context(KvWriteError::Store)?; + let outcome = self + .write_entry(ec_id, &body, &metadata, ttl, mode) + .change_context(KvWriteError::Store)?; + match outcome { + EcKvWriteOutcome::Written => Ok(()), + EcKvWriteOutcome::PreconditionFailed => Err(Report::new(match mode { + EcKvWriteMode::Add => KvWriteError::AlreadyExists, + _ => KvWriteError::Conflict, + })), + } + } + /// Creates a new entry, or overwrites an existing tombstone on re-consent. /// /// Three-way behavior: @@ -468,8 +467,9 @@ impl KvIdentityGraph { /// - **Existing tombstone** (`consent.ok = false`) — CAS overwrite with /// the new entry. Retries up to [`MAX_CAS_RETRIES`] on conflict. /// - /// This method is reserved for explicit same-key revival. Production EC - /// generation uses [`Self::create_if_absent`] with a freshly generated ID. + /// This method is reserved for explicit same-key revival. A freshly + /// generated EC's row is created by the navigation's page-load write + /// ([`Self::commit_page_load`]). /// /// # Errors /// @@ -555,154 +555,6 @@ impl KvIdentityGraph { ))) } - /// Persists browser EID cookies with one conditional write and one conflict read. - /// - /// Browser cookies do not carry a value-owned version, so a different - /// existing value has unknown freshness and is always deferred. After a CAS - /// conflict this method never writes again. A live follow-up becomes the - /// authoritative snapshot; a missing or failed follow-up retains the live - /// pre-write snapshot as proof and defers the update. - pub(crate) fn sync_eid_cookie_updates_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> (EcKvSnapshot, EidCookieSyncOutcome) { - if updates.is_empty() { - return (snapshot, EidCookieSyncOutcome::AlreadyMatched); - } - - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - let current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return (snapshot, EidCookieSyncOutcome::Failed), - _ => self.load_snapshot(ec_id), - }; - - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - EcKvSnapshot::Missing { .. } => { - let outcome = if proven.is_some() { - EidCookieSyncOutcome::DeferredStaleRead - } else { - EidCookieSyncOutcome::Missing - }; - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, outcome); - } - EcKvSnapshot::Failed { .. } => { - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, EidCookieSyncOutcome::Failed); - } - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - } - }; - - if !entry.consent.ok { - return (current, EidCookieSyncOutcome::ConsentWithdrawn); - } - let deferred_freshness = match apply_cookie_partner_id_updates(&mut entry, updates) { - CookieUpdateApplication::AlreadyMatched => { - return (current, EidCookieSyncOutcome::AlreadyMatched); - } - CookieUpdateApplication::DeferredFreshness => { - return (current, EidCookieSyncOutcome::DeferredFreshness); - } - CookieUpdateApplication::Changed { deferred } => deferred, - }; - - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => ( - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }, - if deferred_freshness { - EidCookieSyncOutcome::WrittenWithDeferredFreshness - } else { - EidCookieSyncOutcome::Written - }, - ), - Ok(EcKvWriteOutcome::PreconditionFailed) => { - let refreshed = self.load_snapshot(ec_id); - let Some(refreshed_entry) = refreshed.entry_for(ec_id) else { - // The failed CAS proved `current`'s generation stale. Keep - // only its existence proof so later writes must reread. - let proof = match current { - EcKvSnapshot::Present { - ec_id: proven_id, - entry, - .. - } => EcKvSnapshot::Present { - ec_id: proven_id, - entry, - generation: None, - }, - other => other, - }; - let kept = Self::keep_proven(ec_id, refreshed, Some(&proof)); - return (kept, EidCookieSyncOutcome::DeferredConflict); - }; - if !refreshed_entry.consent.ok { - return (refreshed, EidCookieSyncOutcome::ConsentWithdrawn); - } - let outcome = if partner_id_updates_match(refreshed_entry, updates) { - EidCookieSyncOutcome::ConflictMatched - } else { - EidCookieSyncOutcome::DeferredConflict - }; - (refreshed, outcome) - } - Err(err) => { - log::warn!("EID cookie sync write failed: {err:?}"); - ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ) - } - } - } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. /// /// A caller-supplied `Present` snapshot is *proof* that the row exists — @@ -792,11 +644,29 @@ impl KvIdentityGraph { if !apply_partner_id_updates(&mut entry, updates) { return current; } - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), + let (body, meta_str) = + match Self::serialize_entry_unless_full(&entry, self.store_name()) { + Ok(SerializedEntry::Fits { body, meta }) => (body, meta), + Ok(SerializedEntry::TooLarge(bytes)) => { + log::warn!( + "snapshot partner upsert for '{}' skipped: row would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES})", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + Err(err) => { + log::warn!( + "snapshot partner upsert for '{}' could not serialize the row: {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } }; - }; match self.write_entry( ec_id, &body, @@ -920,9 +790,7 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -992,12 +860,21 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); - let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; + let (body, meta_str) = + match Self::serialize_entry_unless_full(&entry, self.store_name())? { + SerializedEntry::Fits { body, meta } => (body, meta), + SerializedEntry::TooLarge(bytes) => { + log::warn!( + "upsert_partner_id_if_exists: row for '{}' would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES}); partner '{partner_id}' not written", + log_id(ec_id), + ); + return Ok(UpsertResult::RowFull); + } + }; match self.write_entry( ec_id, @@ -1051,7 +928,12 @@ impl KvIdentityGraph { ) } - fn withdrawal_marker_exists( + /// Whether a still-valid withdrawal-completion marker exists for `ec_id`. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] when the marker listing fails. + pub(crate) fn withdrawal_marker_exists( &self, ec_id: &str, now: Option, @@ -1253,117 +1135,45 @@ impl KvIdentityGraph { }) } - /// Resolves a tombstone attempt whose point read reported the row absent. + /// Writes a withdrawal tombstone only when an existing row can be confirmed. /// - /// A completion marker proves an earlier withdrawal finished and avoids a - /// second root existence check. Otherwise, a proven-absent key is a no-op: - /// there is nothing to withdraw, and a forged cookie must not mint a row. A - /// key that provably exists is tombstoned unconditionally because no CAS - /// generation is available after a missed read. Marker-check failure falls - /// back to that privacy write, as does an unusable clock; root-existence - /// failure leaves withdrawal unresolved rather than silently dropped. - fn tombstone_unproven_missing( + /// This is a privacy write outside the one-write-per-page-load rule: it + /// runs on any finalized request that carries an explicit withdrawal and + /// an EC cookie, whatever the route or response status. + /// + /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` + /// cookie must not mint a row. But a *point read* cannot prove absence on + /// an eventually-consistent store, and dropping a withdrawal is worse than + /// a redundant read, so absence is established in two stages: + /// + /// 1. Any snapshot that is not a usable `Present` for this EC ID — a + /// publisher preload that read `Missing`, a read that `Failed`, or one + /// lacking a CAS generation — is re-read. + /// 2. A re-read that still reports the row absent is checked against + /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads + /// the primary data source. + /// + /// A tombstone is idempotent, so a CAS conflict re-reads and retries up to + /// [`MAX_CAS_RETRIES`] times. Resolving the initial snapshot happens + /// outside the retry counter. + pub(crate) fn tombstone_existing_from_snapshot( &self, ec_id: &str, - missing: EcKvSnapshot, - now: Option, + snapshot: EcKvSnapshot, ) -> EcKvSnapshot { - match self.withdrawal_marker_exists(ec_id, now) { - Ok(true) => { - log::debug!( - "withdrawal tombstone for '{}': completion marker already exists", - log_id(ec_id) - ); - return missing; - } - Ok(false) => {} - Err(err) => { - // Marker failure must not weaken withdrawal. Fall back to the - // existing root existence check and unconditional privacy write. - log::warn!( - "withdrawal completion marker lookup failed for '{}': {err:?}", - log_id(ec_id) - ); - } - } - - match self.key_exists_confirmed(ec_id) { - Ok(false) => missing, - Ok(true) => { - log::warn!( - "withdrawal tombstone for '{}': point read missed a row the store still \ - lists; writing an unconditional tombstone", - log_id(ec_id) - ); - match self.overwrite_withdrawal_tombstone(ec_id) { - Ok(tombstone) => { - self.record_withdrawal_completion(ec_id, tombstone.consent.updated); - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - } - } - Err(err) => { - log::warn!( - "unconditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - Err(err) => { - log::warn!( - "withdrawal tombstone for '{}': existence check failed, cannot confirm \ - absence: {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - - /// Writes a tombstone only when an existing row can be confirmed. - /// - /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` - /// cookie must not mint a row. But a *point read* cannot prove absence on - /// an eventually-consistent store, and dropping a withdrawal is worse than - /// a redundant read, so absence is established in two stages: - /// - /// 1. Any snapshot that is not a usable `Present` for this EC ID — a - /// publisher preload that read `Missing`, a read that `Failed`, or one - /// lacking a CAS generation — is re-read. On the publisher path that - /// re-read is separated from the preload by the full origin round trip, - /// which gives replication time to converge. - /// 2. A re-read that still reports the row absent is checked against - /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads - /// the primary data source. - /// - /// Resolving the initial snapshot happens outside the retry counter, so all - /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. - pub(crate) fn tombstone_existing_from_snapshot( - &self, - ec_id: &str, - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - .. - } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - _ => self.load_snapshot(ec_id), - }; + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + _ => self.load_snapshot(ec_id), + }; for _attempt in 0..MAX_CAS_RETRIES { let generation = match current { @@ -1434,9 +1244,6 @@ impl KvIdentityGraph { } } - // Withdrawal enforcement lost every CAS race, so the row can still be - // live with consent granted while the browser cookie is cleared. That - // divergence is only visible to operators if it is logged here. log::warn!( "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ identity-graph row may still be live with consent granted", @@ -1447,6 +1254,80 @@ impl KvIdentityGraph { } } + /// Resolves a withdrawal whose row read as missing. + /// + /// A still-valid completion marker or a confirmed absent root means + /// there is nothing to withdraw, and a forged cookie must not mint a row. A + /// key that provably exists is tombstoned unconditionally because no CAS + /// generation is available after a missed read. Marker-check failure falls + /// back to that privacy write; root-existence failure leaves withdrawal + /// unresolved rather than silently dropped. + fn tombstone_unproven_missing( + &self, + ec_id: &str, + missing: EcKvSnapshot, + now: Option, + ) -> EcKvSnapshot { + match self.withdrawal_marker_exists(ec_id, now) { + Ok(true) => { + log::debug!( + "withdrawal tombstone for '{}': completion marker already exists", + log_id(ec_id) + ); + return missing; + } + Ok(false) => {} + Err(err) => { + // Marker failure must not weaken withdrawal. Fall back to the + // existing root existence check and unconditional privacy write. + log::warn!( + "withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } + + match self.key_exists_confirmed(ec_id) { + Ok(false) => missing, + Ok(true) => { + log::warn!( + "withdrawal tombstone for '{}': point read missed a row the store still \ + lists; writing an unconditional tombstone", + log_id(ec_id) + ); + match self.overwrite_withdrawal_tombstone(ec_id) { + Ok(tombstone) => { + self.record_withdrawal_completion(ec_id, tombstone.consent.updated); + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + } + } + Err(err) => { + log::warn!( + "unconditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + Err(err) => { + log::warn!( + "withdrawal tombstone for '{}': existence check failed, cannot confirm \ + absence: {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -1519,7 +1400,20 @@ impl KvIdentityGraph { network.cluster_size = Some(cluster_size); updated_entry.network = Some(network); - let (body, meta_str) = Self::serialize_entry(&updated_entry, self.store_name())?; + let (body, meta_str) = + match Self::serialize_entry_unless_full(&updated_entry, self.store_name())? { + SerializedEntry::Fits { body, meta } => (body, meta), + SerializedEntry::TooLarge(bytes) => { + // The computed value is still valid; only the cache write + // is skipped. + log::warn!( + "evaluate_cluster: row for '{}' would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES}); cluster_size not stored", + log_id(ec_id) + ); + return Ok(Some(cluster_size)); + } + }; match self.write_entry( ec_id, @@ -1551,7 +1445,7 @@ impl KvIdentityGraph { /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). Consent - /// withdrawal uses [`Self::tombstone_existing_from_snapshot`] instead. + /// withdrawal uses a page-load tombstone via [`Self::commit_page_load`] instead. /// /// # Errors /// @@ -1574,6 +1468,16 @@ impl KvIdentityGraph { )) } + /// A graph holding `entry` at `ec_id` whose reads succeed and whose + /// writes all fail. + pub(crate) fn write_failing_with(ec_id: &str, entry: &KvEntry) -> Self { + let store = super::kv_backend::test_support::WriteFailingEcKv::new(); + let (body, meta) = Self::serialize_entry(entry, store.store_name()) + .expect("should serialize the seed row"); + store.seed(ec_id, &body, &meta); + Self::new(store) + } + /// Test helper: a graph backed by an in-memory store with generation /// tracking. pub(crate) fn in_memory(store_name: impl Into) -> Self { @@ -1618,86 +1522,10 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - - /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition - /// and deletes the key, simulating a concurrent withdrawal that removes the - /// row between this writer's read and its write. - struct DisappearOnConflictEcKv { - inner: InMemoryEcKv, - conflicts_remaining: std::sync::Mutex, - } - - impl DisappearOnConflictEcKv { - fn new(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("disappear-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - } - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize seeded entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed live entry"); - } - } - - impl EcKvStore for DisappearOnConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - let mut remaining = self - .conflicts_remaining - .lock() - .expect("should lock conflict counter"); - if *remaining > 0 { - *remaining -= 1; - self.inner.delete(key).expect("should delete on conflict"); - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } + use crate::ec::checked_current_timestamp; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; + use crate::ec::kv_backend::test_support::{InMemoryEcKv, WriteFailingEcKv}; + use crate::ec::kv_types::{KvEid, KvUid}; fn snapshot_ec_id() -> String { format!("{}.ABC123", "a".repeat(64)) @@ -1733,22 +1561,27 @@ mod tests { } #[test] - fn deserialize_entry_rejects_invalid_legacy_values() { + fn deserialize_entry_drops_oversized_ids() { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), + ), + ); + entry.ids.insert( + "ssp_y".to_owned(), + crate::ec::kv_types::KvPartnerId::single("kept"), ); - let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); + let body = serde_json::to_vec(&entry).expect("should serialize oversized entry payload"); - let err = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) - .expect_err("should reject invalid legacy entry values"); - let err_text = format!("{err}"); - assert!( - err_text.contains("Loaded invalid entry"), - "should report validation failure for loaded entries" + let loaded = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) + .expect("should load an entry holding an oversized ID"); + + assert_eq!( + loaded.ids.keys().collect::>(), + vec!["ssp_y"], + "should drop only the oversized ID" ); } @@ -1772,9 +1605,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), + ), ); let err = KvIdentityGraph::serialize_entry(&entry, "test-store") @@ -1786,6 +1619,48 @@ mod tests { ); } + #[test] + fn create_refuses_oversized_provider_entry_before_write() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let uids = (0..40) + .map(|index| { + let id = format!("{index:03}{}", "x".repeat(480)); + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + let record = crate::ec::kv_types::KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), uids)], + UnixSeconds::new(0), + ) + .expect("should build record"); + let mut entry = live_entry(); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!( + entry.validate().is_ok(), + "should be a valid entry apart from its size" + ); + + let err = graph + .create(&ec_id, &entry) + .expect_err("should refuse oversized entry"); + + assert!( + format!("{err}").contains("Refusing to write KV entry"), + "should report the size limit" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "should not write the oversized entry" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; @@ -1796,9 +1671,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "concurrent.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), ); entry } @@ -1828,15 +1701,6 @@ mod tests { } } - fn with_partner_update_on_conflict(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("partner-conflict-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - revive_on_conflict: true, - partner_update_on_conflict: true, - } - } - fn seed_tombstone(&self, ec_id: &str) { let (body, meta) = KvIdentityGraph::serialize_entry( &KvEntry::tombstone(1000), @@ -1943,131 +1807,6 @@ mod tests { } } - /// Store that replaces the row during the first EID CAS write and records - /// the request's reads and conditional writes. - struct EidConflictEcKv { - inner: InMemoryEcKv, - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - follow_up_miss: bool, - miss_next_lookup: std::sync::atomic::AtomicBool, - } - - impl EidConflictEcKv { - fn new( - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - ) -> Self { - Self { - inner: InMemoryEcKv::new("eid-conflict-store"), - concurrent_entry, - lookups, - conditional_writes, - follow_up_miss: false, - miss_next_lookup: std::sync::atomic::AtomicBool::new(false), - } - } - - fn with_follow_up_miss(mut self) -> Self { - self.follow_up_miss = true; - self - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize initial entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed initial entry"); - } - } - - impl EcKvStore for EidConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.lookups - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if self - .miss_next_lookup - .swap(false, std::sync::atomic::Ordering::Relaxed) - { - return Ok(None); - } - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - self.conditional_writes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let (body, meta) = KvIdentityGraph::serialize_entry( - &self.concurrent_entry, - self.inner.store_name(), - ) - .expect("should serialize concurrent entry"); - self.inner - .insert( - key, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Overwrite, - }, - ) - .expect("should write concurrent entry"); - if self.follow_up_miss { - self.miss_next_lookup - .store(true, std::sync::atomic::Ordering::Relaxed); - } - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn count_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result> { - self.inner.count_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn create_or_revive_retries_cas_conflict_and_succeeds() { let store = ConflictInjectingEcKv::new(2, false); @@ -2131,9 +1870,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-1".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-1".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; @@ -2159,9 +1896,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "old-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("old-uid".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; @@ -2191,9 +1926,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "original".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("original".to_owned()), ); let updates = vec![ PartnerIdUpdate::new("ssp_x", "intermediate"), @@ -2209,287 +1942,6 @@ mod tests { assert_eq!(entry.ids["ssp_x"].uid, "original"); } - #[test] - fn eid_cookie_sync_conflict_matching_value_stops_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "desired-uid".to_owned(), - }, - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::ConflictMatched); - assert_eq!( - snapshot - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("desired-uid"), - "the authoritative follow-up snapshot should replace stale request state" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - } - - #[test] - fn eid_cookie_sync_keeps_live_proof_when_conflict_follow_up_misses() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - live_entry(), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ) - .with_follow_up_miss(); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the live pre-write row should prevent conflict deferral from entering orphan recovery" - ); - assert_eq!( - snapshot.generation_for(&ec_id), - None, - "a failed CAS must not return its rejected generation" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "the initial refresh and conflict follow-up should be the only reads" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "the conflict must remain the request's only conditional write" - ); - } - - #[test] - fn eid_cookie_sync_reports_proven_refresh_miss_as_deferred() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("empty-store"); - let proven = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - proven, - ); - - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the add-confirmed row should survive an eventually consistent miss" - ); - assert_eq!( - outcome, - EidCookieSyncOutcome::DeferredStaleRead, - "a row already proven to exist should report a deferred stale read" - ); - } - - #[test] - fn partner_id_conflict_match_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "latest-uid".to_owned(), - }, - ); - let updates = [ - PartnerIdUpdate::new("ssp_x", "older-uid"), - PartnerIdUpdate::new("ssp_x", "latest-uid"), - ]; - - assert!( - partner_id_updates_match(&entry, &updates), - "conflict matching should use the same last-value-wins rule as writes" - ); - } - - #[test] - fn eid_cookie_sync_conflicting_value_defers_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "stale-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read concurrent row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "concurrent-uid"); - } - - #[test] - fn eid_cookie_sync_preserves_concurrent_withdrawal_after_conflict() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - KvEntry::tombstone(2_000), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::ConsentWithdrawn); - assert!( - snapshot - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the refreshed withdrawal tombstone must remain authoritative" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - } - - #[test] - fn eid_cookie_sync_defers_different_values_without_value_owned_freshness() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "incoming-uid")], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read protected row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - } - - #[test] - fn eid_cookie_sync_adds_missing_ids_while_deferring_different_values() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("mixed-freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[ - PartnerIdUpdate::new("ssp_x", "different-uid"), - PartnerIdUpdate::new("ssp_y", "new-uid"), - ], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::WrittenWithDeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read updated row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - assert_eq!(stored.ids["ssp_y"].uid, "new-uid"); - } - #[test] fn evaluate_cluster_returns_stored_value_without_store_io() { let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); @@ -2558,34 +2010,6 @@ mod tests { ); } - #[test] - fn create_if_absent_reports_written_and_collision() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should create absent entry"), - CreateIfAbsentOutcome::Written - ); - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should report collision"), - CreateIfAbsentOutcome::AlreadyExists - ); - } - - #[test] - fn create_if_absent_propagates_store_error() { - let kv = KvIdentityGraph::failing("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert!( - kv.create_if_absent(&ec_id, &live_entry()).is_err(), - "should preserve store failures instead of reporting a collision" - ); - } - #[test] fn create_or_revive_revives_tombstone() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -2633,185 +2057,6 @@ mod tests { ); } - #[test] - fn create_or_revive_clears_withdrawal_marker() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" - ); - - kv.create_or_revive(&ec_id, &live_entry()) - .expect("should revive tombstone"); - - let (loaded, _) = kv - .get(&ec_id) - .expect("should read revived entry") - .expect("should find revived entry"); - assert!(loaded.consent.ok, "should be live after revive"); - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "revival should clear stale withdrawal completion" - ); - } - - #[test] - fn stale_completion_marker_does_not_suppress_withdrawal_after_root_recreation() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - let expired_updated = current_timestamp().saturating_sub(TOMBSTONE_TTL.as_secs() + 1); - let expired_tombstone = KvEntry::tombstone(expired_updated); - graph - .create(&ec_id, &expired_tombstone) - .expect("should seed old tombstone"); - graph - .write_withdrawal_marker(&ec_id, expired_updated) - .expect("should seed old completion marker"); - - // Model the root expiring just before its later-written completion - // marker, followed by recreation of the same key. - graph.store.delete(&ec_id).expect("should expire root row"); - assert_eq!( - graph - .create_if_absent(&ec_id, &live_entry()) - .expect("should recreate expired root"), - CreateIfAbsentOutcome::Written, - "should make the same key live again after root expiry" - ); - - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "an expired completion marker must not suppress withdrawal of a recreated live row" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read recreated row") - .expect("should retain recreated row"); - assert!(!stored.consent.ok, "recreated row should be tombstoned"); - } - - #[test] - fn withdrawal_failed_clock_stale_miss_tombstones_live_root() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &concurrent_live_entry()) - .expect("should seed a live row with partner IDs"); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a completion marker left after root recreation"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - assert!( - matches!(missing, EcKvSnapshot::Missing { .. }), - "should reproduce a stale point-read miss for the live row" - ); - - // None is the checked clock's failure result, not Unix epoch zero. - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); - - let (stored, generation) = graph - .get(&ec_id) - .expect("should read back persisted withdrawal") - .expect("should retain the root"); - assert!( - !stored.consent.ok, - "an unusable clock must not let a marker suppress withdrawal" - ); - assert!( - stored.ids.is_empty(), - "withdrawal should clear stored partner IDs" - ); - assert_eq!( - generation, 2, - "withdrawal should write the existing root once" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "clock failure should not fail a strongly confirmed withdrawal" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should strongly check the root" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL - }, - ], - "should write only the root tombstone and its completion marker" - ); - } - - #[test] - fn withdrawal_failed_clock_does_not_create_absent_root() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a marker without a root"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "absent root should remain missing" - ); - assert!( - graph - .get(&ec_id) - .expect("should read absent root") - .is_none(), - "clock failure must not mint an unknown root" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should prove root absence despite the marker" - ); - assert!( - operations.inserts().is_empty(), - "absent identity should cause no writes" - ); - } - #[test] fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { let graph = KvIdentityGraph::in_memory("test_store"); @@ -2976,7 +2221,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read root after rejected hard delete") - .expect("rejected hard delete should retain root"); + .expect("should retain the root after a rejected hard delete"); assert!( !stored.consent.ok, "rejected hard delete should leave the tombstoned root in place" @@ -2989,110 +2234,57 @@ mod tests { } #[test] - fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept a marker removed by another request"); - - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker removal"), - "completion marker should remain absent" - ); - } - - #[test] - fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.clear_withdrawal_marker(&ec_id).is_err(), - "a marker that remains after delete failure should block revival" - ); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker remains"), - "completion marker should remain present" - ); - } - - #[test] - fn withdrawal_marker_is_excluded_from_hash_prefix_count() { - let hash = "a".repeat(64); - let ec_id = format!("{hash}.ABC123"); - let kv = KvIdentityGraph::in_memory("test_store"); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count live root"), - 1, - "the root should be the only hash-prefix key" - ); - - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" - ); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count tombstoned root"), - 1, - "the marker namespace must not affect cluster counts" - ); - } - - #[test] - fn delete_removes_withdrawal_marker() { + fn upsert_partner_id_if_exists_reports_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - kv.delete(&ec_id).expect("should delete entry and marker"); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "hard delete should remove the root" - ); - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "hard delete should remove withdrawal completion" + let result = kv + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should not error on missing key"); + assert_eq!( + result, + UpsertResult::NotFound, + "should reject a partner upsert for an EC ID the store does not hold" ); } #[test] - fn upsert_partner_id_if_exists_reports_missing_key() { + fn upsert_partner_id_if_exists_reports_a_full_row_without_writing() { let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = live_entry(); + let uid = "u".repeat(500); + let mut partner = 0; + while serde_json::to_string(&entry) + .expect("should serialize entry") + .len() + < MAX_ENTRY_BYTES - 600 + { + entry.ids.insert( + format!("ssp{partner}.example.com"), + crate::ec::kv_types::KvPartnerId::single(uid.clone()), + ); + partner += 1; + } + kv.create(&ec_id, &entry) + .expect("should create a near-full row"); let result = kv - .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") - .expect("should not error on missing key"); + .upsert_partner_id_if_exists(&ec_id, "full.example.com", &uid) + .expect("should report a full row as a result, not an error"); + assert_eq!( result, - UpsertResult::NotFound, - "should reject a partner upsert for an EC ID the store does not hold" + UpsertResult::RowFull, + "should refuse a partner ID that would overflow the row" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read the row") + .expect("should keep the row"); + assert!( + !stored.ids.contains_key("full.example.com"), + "should not write the refused partner ID" ); } @@ -3207,48 +2399,6 @@ mod tests { assert!(kv.get(&ec_id).expect("should read store").is_none()); } - #[test] - fn tombstone_existing_from_snapshot_never_creates_missing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let snapshot = EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }; - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "withdrawal should preserve the missing snapshot for an absent identity" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "withdrawal must not create a tombstone for an absent key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_uses_existing_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should return the persisted tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "should persist withdrawal state"); - } - #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -3317,14 +2467,6 @@ mod tests { self.deletes.load(std::sync::atomic::Ordering::Relaxed) } - fn operation_count(&self) -> usize { - self.lookup_count() - + self.exact_check_count() - + self.inserts().len() - + self.list_count() - + self.delete_count() - } - fn inserts(&self) -> Vec { self.inserts .lock() @@ -3378,49 +2520,6 @@ mod tests { } } - fn completed_with_root_check_failure( - operations: Arc, - ec_id: &str, - ) -> Self { - let store = Self { - root_check_fails: true, - stale_lookups_remaining: std::sync::Mutex::new(u32::MAX), - ..Self::new(operations) - }; - let tombstone = KvEntry::tombstone(current_timestamp()); - let (body, metadata) = - KvIdentityGraph::serialize_entry(&tombstone, store.inner.store_name()) - .expect("should serialize tombstone"); - store - .inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &metadata, - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed tombstone"); - store - .inner - .insert( - &KvIdentityGraph::withdrawal_marker_key( - ec_id, - tombstone.consent.updated + TOMBSTONE_TTL.as_secs(), - ), - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed completion marker"); - store - } - fn with_marker_delete_failure( operations: Arc, remove_before_error: bool, @@ -3536,53 +2635,6 @@ mod tests { } } - /// [`EcKvStore`] whose reads succeed but every write fails, simulating a - /// store that becomes unwritable mid-request. - struct WriteFailingEcKv { - inner: InMemoryEcKv, - } - - impl WriteFailingEcKv { - fn new() -> Self { - Self { - inner: InMemoryEcKv::new("write-failing-store"), - } - } - } - - impl EcKvStore for WriteFailingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - _key: &str, - _write: EcKvWrite<'_>, - ) -> Result> { - Err(Report::new(TrustedServerError::KvStore { - store_name: self.inner.store_name().to_owned(), - message: "write failing test store".to_owned(), - })) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn snapshot_upsert_with_generation_writes_without_reading() { let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -3751,578 +2803,103 @@ mod tests { let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("ssp_x").map(|id| id.uid.as_str()), Some("uid-1"), - "the revalidated update must reach the store" - ); - } - - #[test] - fn snapshot_upsert_confirmed_missing_still_never_creates() { - // Revalidation only changes what a *stale* miss does. A row that is - // genuinely absent on the refresh must stay absent. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a confirmed miss must stay missing" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "must not create a root entry for a missing key" - ); - } - - #[test] - fn snapshot_upsert_failed_snapshot_is_not_revalidated() { - // A lookup that already errored is not retried on the hot path, even - // though the row exists. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed lookup must not be retried by partner enrichment" - ); - } - - #[test] - fn snapshot_upsert_rejects_tombstone() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should seed tombstone"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| entry.ids.is_empty()), - "a tombstone must reject partner enrichment" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("tombstone should remain"); - assert!(stored.ids.is_empty(), "no update should reach the store"); - } - - #[test] - fn snapshot_upsert_store_failure_returns_failed_not_request_local() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a store write failure must not claim request-local IDs were persisted" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { - for generation in [Some(7), None] { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation, - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - - assert_eq!( - outcome, snapshot, - "should preserve authoritative tombstone state" - ); - assert_eq!( - operations.operation_count(), - 0, - "an authoritative tombstone should not access the backend" - ); - } - } - - #[test] - fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let live_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); - - assert_eq!( - operations.lookup_count(), - 0, - "usable generation should avoid a read" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "first withdrawal should write the root and its completion marker" - ); - let first_snapshot = graph.load_snapshot(&ec_id); - let (first_entry, first_generation) = match &first_snapshot { - EcKvSnapshot::Present { - entry, generation, .. - } => (entry.as_ref().clone(), *generation), - other => panic!("should load first tombstone, got {other:?}"), - }; - operations.reset(); - - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); - - assert_eq!( - operations.operation_count(), - 0, - "repeated withdrawal should not access the backend" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - first_generation, - "repeated withdrawal should preserve the stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - Some(first_entry.consent.updated), - "repeated withdrawal should preserve the first tombstone timestamp" - ); - } - - #[test] - fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 2, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - let first_updated = first_outcome - .entry_for(&ec_id) - .expect("should return first tombstone") - .consent - .updated; - assert_eq!( - operations.lookup_count(), - 1, - "first stale miss should retry its point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "first stale miss should list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "first stale miss should confirm root existence once" - ); - assert_eq!( - operations.inserts().len(), - 2, - "first stale miss should write the root and completion marker" - ); - operations.reset(); - - graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!( - operations.lookup_count(), - 1, - "a repeated stale miss should retry its point read once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "a valid completion marker should avoid a root existence check" - ); - assert_eq!( - operations.list_count(), - 1, - "a repeated stale miss should list completion markers once" - ); - assert_eq!( - operations.delete_count(), - 0, - "a repeated withdrawal should not delete marker state" - ); - assert!( - operations.inserts().is_empty(), - "a repeated stale miss should not rewrite the completed tombstone" - ); - let (stored, generation) = graph - .get(&ec_id) - .expect("should read stored tombstone") - .expect("should preserve tombstone"); - assert_eq!( - generation, 2, - "only the first withdrawal should advance the root generation" - ); - assert_eq!( - stored.consent.updated, first_updated, - "repeated withdrawal should preserve the first tombstone timestamp" - ); - } - - #[test] - fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let stale_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); - - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - ], - "parallel loser should attempt stale CAS once and never replace the winner" - ); - assert_eq!( - operations.lookup_count(), - 1, - "parallel loser should reread exactly once after its conflict" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - Some(2), - "parallel loser should return the winner's stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - first_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - "parallel loser should preserve the winner's tombstone" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { - let graph = KvIdentityGraph::failing("unavailable-store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(3), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - - assert_eq!( - outcome, snapshot, - "authoritative tombstone should not touch unavailable backend" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { - let ec_id = snapshot_ec_id(); - let states = [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }, - EcKvSnapshot::Present { - ec_id: "different-ec-id".to_owned(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(9), - }, - ]; - - for state in states { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); - - assert_eq!( - operations.lookup_count(), - 1, - "state should force one reread" - ); - assert_eq!( - operations.inserts().len(), - 2, - "live reread should write the root and completion marker" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "reread live row should be tombstoned" - ); - } - } - - #[test] - fn tombstone_stale_miss_valid_marker_skips_root_check_and_writes() { - let ec_id = snapshot_ec_id(); - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::completed_with_root_check_failure( - Arc::clone(&operations), - &ec_id, - )); - - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a completion marker should resolve a repeated stale miss" - ); - assert_eq!( - operations.lookup_count(), - 1, - "should retry the stale point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "should strongly list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "valid marker should bypass the failing root check" - ); - assert!( - operations.inserts().is_empty(), - "valid marker should prevent redundant writes" - ); - } - - #[test] - fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_marker_failures( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "marker failures must not suppress the withdrawal write" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "failed completion recording should still be attempted after the root write" + "the revalidated update must reach the store" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read stored row") - .expect("should preserve the root"); - assert!(!stored.consent.ok, "root should remain tombstoned"); } #[test] - fn tombstone_existing_from_snapshot_retries_cas_conflict() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + fn snapshot_upsert_confirmed_missing_still_never_creates() { + // Revalidation only changes what a *stale* miss does. A row that is + // genuinely absent on the refresh must stay absent. + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should retry the conflict and persist the tombstone" + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a confirmed miss must stay missing" + ); + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "must not create a root entry for a missing key" ); } #[test] - fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot refreshes once before its CAS. That - // refresh must not spend a CAS attempt, so a withdrawal tombstone still - // persists after four conflicts and a successful fifth write. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + fn snapshot_upsert_failed_snapshot_is_not_revalidated() { + // A lookup that already errored is not retried on the hot path, even + // though the row exists. + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; + kv.create(&ec_id, &live_entry()).expect("should seed live"); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a failed lookup must not be retried by partner enrichment" ); } #[test] - fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { - // Every CAS attempt loses its race, so the row stays live with consent - // granted while the browser cookie is already cleared. The caller must - // see a failure it can report rather than a silent no-op. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); + fn snapshot_upsert_rejects_tombstone() { + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let snapshot = kv.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "CAS exhaustion must report a failed withdrawal" + outcome + .entry_for(&ec_id) + .is_some_and(|entry| entry.ids.is_empty()), + "a tombstone must reject partner enrichment" ); - let (stored, _) = graph + let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("row should remain"); - assert!( - stored.consent.ok, - "the row is still live, which is exactly why the failure must be reported" - ); + .expect("should keep the tombstone"); + assert!(stored.ids.is_empty(), "no update should reach the store"); } #[test] - fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::with_partner_update_on_conflict(1)); + fn snapshot_upsert_store_failure_returns_failed_not_request_local() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let snapshot = graph.load_snapshot(&ec_id); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - let entry = outcome - .entry_for(&ec_id) - .expect("should return persisted tombstone"); - assert!(!entry.consent.ok, "withdrawal should win after retry"); assert!( - entry.ids.is_empty(), - "withdrawal should clear concurrent partner IDs" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a store write failure must not claim request-local IDs were persisted" ); } @@ -4348,73 +2925,6 @@ mod tests { ); } - #[test] - fn tombstone_existing_from_snapshot_store_failure_returns_failed() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed tombstone write should return failed snapshot state" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { - let store = DisappearOnConflictEcKv::new(1); - store.seed_live(&snapshot_ec_id()); - let graph = KvIdentityGraph::new(store); - let ec_id = snapshot_ec_id(); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a row that disappears during retry becomes a no-op" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "must not recreate the disappeared key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { - // A prior request-scoped read failed, so the snapshot is `Failed`. A - // withdrawal must not silently drop consent removal: re-read the store - // and tombstone the row if it is authoritatively present. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "a failed snapshot must re-read and persist the tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "withdrawal must reach the store"); - } - #[test] fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { let graph = KvIdentityGraph::stale_lookup("stale-store", 1); @@ -4856,4 +3366,131 @@ mod tests { "a different suffix under the same hash is a different identity" ); } + + fn tombstone_via_page_load(kv: &KvIdentityGraph, ec_id: &str) { + let outcome = kv.tombstone_existing_from_snapshot(ec_id, EcKvSnapshot::NotRead); + assert!( + outcome + .entry_for(ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should write tombstone" + ); + } + + #[test] + fn create_or_revive_clears_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); + + let (loaded, _) = kv + .get(&ec_id) + .expect("should read revived entry") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "revival should clear stale withdrawal completion" + ); + } + + #[test] + fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept a marker removed by another request"); + + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker removal"), + "completion marker should remain absent" + ); + } + + #[test] + fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.clear_withdrawal_marker(&ec_id).is_err(), + "a marker that remains after delete failure should block revival" + ); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker remains"), + "completion marker should remain present" + ); + } + + #[test] + fn withdrawal_marker_is_excluded_from_hash_prefix_count() { + let hash = "a".repeat(64); + let ec_id = format!("{hash}.ABC123"); + let kv = KvIdentityGraph::in_memory("test_store"); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count live root"), + 1, + "the root should be the only hash-prefix key" + ); + + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count tombstoned root"), + 1, + "the marker namespace must not affect cluster counts" + ); + } + + #[test] + fn delete_removes_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + kv.delete(&ec_id).expect("should delete entry and marker"); + + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "hard delete should remove the root" + ); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "hard delete should remove withdrawal completion" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index bfe74ca6f3..70a999ffa0 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -178,6 +178,68 @@ pub(crate) mod test_support { use super::*; + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a + /// store that becomes unwritable mid-request. + pub(crate) struct WriteFailingEcKv { + inner: InMemoryEcKv, + } + + impl WriteFailingEcKv { + pub(crate) fn new() -> Self { + Self { + inner: InMemoryEcKv::new("write-failing-store"), + } + } + + /// Stores a row directly, bypassing the failing write path. + pub(crate) fn seed(&self, key: &str, body: &str, metadata: &str) { + self.inner + .insert( + key, + EcKvWrite { + body, + metadata, + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::Overwrite, + }, + ) + .expect("should seed the in-memory store"); + } + } + + impl EcKvStore for WriteFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + _key: &str, + _write: EcKvWrite<'_>, + ) -> Result> { + Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "write failing test store".to_owned(), + })) + } + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// [`EcKvStore`] wrapper that counts `lookup` calls through a shared counter /// so tests can prove exactly how many reads a flow performs. pub(crate) struct CountingEcKv { diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 589f280c11..f13b258dfc 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -6,10 +6,16 @@ //! //! The schema is versioned (`v: 1`) to allow future migrations. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, de}; +use serde_json::value::RawValue; +use super::eid_types::{ + AgentType, EidFields, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, UidValue, + UnixSeconds, optional_exp, +}; use crate::consent::ConsentContext; use crate::geo::GeoInfo; @@ -30,14 +36,27 @@ pub const MAX_SEEN_DOMAINS: usize = 50; pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// Maximum allowed length (in bytes) for a partner UID across all sync -/// mechanisms (pixel, batch, pull). Defined centrally to ensure consistent -/// validation. +/// mechanisms (pixel, batch, pull) and browser ingest. Defined centrally to +/// ensure consistent validation. +/// +/// Builds before identity providers reject a row holding a longer UID, so +/// these writers stay at this limit until rollback past them is no longer +/// supported; only provider writes use [`MAX_STORED_UID_LENGTH`]. pub const MAX_UID_LENGTH: usize = 512; +/// Maximum length (in bytes) of a UID a row may hold: the limit for identity +/// provider writes and for reading stored rows. Sized for `LiveRamp` envelopes +/// (about 700 bytes). +pub const MAX_STORED_UID_LENGTH: usize = 1024; + +/// How long a provider opt-out keeps that provider from being called again, +/// in seconds. Matches the longest lifetime a provider UID may have. +pub const OPT_OUT_MEMORY_SECONDS: u64 = 30 * 24 * 60 * 60; + /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). -/// **KV value:** JSON-serialized `KvEntry` (max ~5KB). +/// **KV value:** JSON-serialized `KvEntry` (at most [`MAX_ENTRY_BYTES`]). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvEntry { /// Schema version — always [`SCHEMA_VERSION`]. @@ -62,6 +81,10 @@ pub struct KvEntry { /// Populated by pixel sync, batch sync, and pull sync operations. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub ids: BTreeMap, + /// Provider ID → Unix time until which that provider's opt-out is + /// honoured without calling the provider again. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub opt_outs: BTreeMap, } /// Consent state within a KV entry. @@ -97,11 +120,364 @@ pub struct KvGeo { pub dma: Option, } -/// A partner user ID within a KV entry. +/// Maximum serialized size of a KV entry body. Every writer refuses a larger +/// row; only the page-load planner trims entries to fit. +pub const MAX_ENTRY_BYTES: usize = 16 * 1024; + +/// A partner ID record within a KV entry, keyed by source domain. +/// +/// `uid` is always present so older readers keep working. Records written by +/// an identity provider also carry full-fidelity `eids`; new code reads +/// `eids` whenever it is non-empty and treats `uid` as derived. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvPartnerId { - /// The partner's user identifier. + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. pub uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +impl KvPartnerId { + /// Builds a single-UID record (browser ingest, batch sync, pull sync). + #[must_use] + pub fn single(uid: impl Into) -> Self { + Self { + uid: uid.into(), + eids: Vec::new(), + } + } + + /// Builds a provider record, keeping only entries with live UIDs. + /// + /// Returns `None` when no live UID remains. + #[must_use] + pub fn from_eids(eids: Vec, now: UnixSeconds) -> Option { + let mut record = Self { + uid: String::new(), + eids, + }; + record.prune_expired(now); + (!record.eids.is_empty()).then_some(record) + } + + /// Full-fidelity provider entries; empty for single-UID records. + #[must_use] + pub fn eids(&self) -> &[KvEid] { + &self.eids + } + + /// Whether this record was written by an identity provider. + #[must_use] + pub fn is_provider_written(&self) -> bool { + !self.eids.is_empty() + } + + /// Whether a UID `owner` wrote asks it for a refresh by `now`. + /// + /// UIDs another provider wrote never make the record refresh-due for + /// `owner`, so their refresh state is never handed to it. + #[must_use] + pub fn refresh_due_for(&self, owner: &ProviderId, now: UnixSeconds) -> bool { + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.owner() == Some(owner) && uid.refresh_due(now)) + } + + /// Whether the record holds a UID that may be served at `now`. + #[must_use] + pub fn has_live_uid(&self, now: UnixSeconds) -> bool { + if self.eids.is_empty() { + return !self.uid.is_empty(); + } + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.is_live(now)) + } + + /// Drops expired UIDs and empty entries, then resyncs `uid`. + /// + /// Returns whether anything changed. Single-UID records never change. + pub fn prune_expired(&mut self, now: UnixSeconds) -> bool { + if self.eids.is_empty() && !self.uid.is_empty() { + return false; + } + let mut changed = false; + for eid in &mut self.eids { + let held = eid.uids.len(); + eid.uids.retain(|uid| uid.is_live(now)); + changed |= eid.uids.len() != held; + } + self.eids.retain(|eid| !eid.uids.is_empty()); + let first = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map_or("", |uid| uid.id().as_str()); + if self.uid != first { + self.uid = first.to_owned(); + changed = true; + } + changed + } + + /// Drops entries left without UIDs and resyncs `uid`. + /// + /// Returns whether the record still holds a UID within + /// [`MAX_STORED_UID_LENGTH`]. + fn retain_readable(&mut self) -> bool { + if self.eids.is_empty() { + return self.uid.len() <= MAX_STORED_UID_LENGTH; + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + !self.eids.is_empty() + } + + /// Removes the last UID of the last entry and resyncs `uid`; used by the + /// write planner's size cap. + pub(crate) fn drop_last_uid(&mut self) { + if let Some(eid) = self.eids.last_mut() { + eid.uids.pop(); + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + } +} + +/// One stored EID entry with its EID-level fields. +/// +/// Stored flat (`{"inserter": .., "uids": [..]}`). Serialization flattens +/// [`EidFields`]; deserialization goes through a flat private representation +/// because serde's `flatten` cannot deserialize the `RawValue` behind `ext`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(from = "KvEidRepr")] +pub struct KvEid { + #[serde(flatten)] + fields: EidFields, + uids: Vec, +} + +/// Flat stored shape of [`KvEid`], used only to deserialize it. +#[derive(Deserialize)] +struct KvEidRepr { + #[serde(default)] + inserter: Option>, + #[serde(default)] + matcher: Option>, + #[serde(default)] + mm: Option, + #[serde(default)] + ext: Option, + #[serde(deserialize_with = "deserialize_readable_uids")] + uids: Vec, +} + +/// The `id` of a stored UID, read without validating the rest of it. +#[derive(Deserialize)] +struct StoredId<'a> { + #[serde(borrow)] + id: Cow<'a, str>, +} + +/// Deserializes stored UIDs, skipping any whose `id` exceeds +/// [`MAX_STORED_UID_LENGTH`]. +/// +/// A row written under a higher limit then still loads; an entry left +/// without UIDs is removed by [`KvEntry::drop_unreadable_ids`]. Any other +/// invalid UID still fails the row. +fn deserialize_readable_uids<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let raw = Vec::>::deserialize(deserializer)?; + let mut uids = Vec::with_capacity(raw.len()); + for value in &raw { + let oversized = serde_json::from_str::>(value.get()) + .is_ok_and(|stored| stored.id.len() > MAX_STORED_UID_LENGTH); + if oversized { + continue; + } + uids.push(serde_json::from_str(value.get()).map_err(de::Error::custom)?); + } + Ok(uids) +} + +impl From for KvEid { + fn from(repr: KvEidRepr) -> Self { + let KvEidRepr { + inserter, + matcher, + mm, + ext, + uids, + } = repr; + Self { + fields: EidFields { + inserter, + matcher, + mm, + ext, + }, + uids, + } + } +} + +impl KvEid { + /// Builds an entry from validated parts. + #[must_use] + pub fn new(fields: EidFields, uids: Vec) -> Self { + Self { fields, uids } + } + + /// EID-level fields. + #[must_use] + pub fn fields(&self) -> &EidFields { + &self.fields + } + + /// Stored UIDs, live or not. + #[must_use] + pub fn uids(&self) -> &[KvUid] { + &self.uids + } +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, + /// Private refresh state; never served, mirrored or forwarded. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. +/// +/// It stays in the row only: outbound EIDs are built from the UID's `id`, +/// `atype` and `ext`, so this state never reaches bidders or cookies. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework. + pub state: String, + /// When the owning provider should be asked to refresh the UID. + pub from: UnixSeconds, +} + +impl core::fmt::Debug for KvRefresh { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("KvRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } +} + +impl KvUid { + /// Builds a UID from validated parts. + #[must_use] + pub fn new( + id: UidValue, + atype: AgentType, + ext: Option, + owner: Option, + expiry: Expiry, + ) -> Self { + Self { + id, + atype, + ext, + owner, + expiry, + refresh: None, + } + } + + /// Attaches the owning provider's private refresh state. + #[must_use] + pub fn with_refresh(mut self, refresh: KvRefresh) -> Self { + self.refresh = Some(refresh); + self + } + + /// The owning provider's private refresh state. + #[must_use] + pub fn refresh(&self) -> Option<&KvRefresh> { + self.refresh.as_ref() + } + + /// Whether the owning provider asked to refresh this live UID by `now`. + /// + /// An expired UID is never refresh-due: its source is due anyway, and a + /// refresh that keeps failing must not outlive the UID. + #[must_use] + pub fn refresh_due(&self, now: UnixSeconds) -> bool { + self.is_live(now) + && self + .refresh + .as_ref() + .is_some_and(|refresh| refresh.from <= now) + } + + /// The UID value. + #[must_use] + pub fn id(&self) -> &UidValue { + &self.id + } + + /// The agent type. + #[must_use] + pub fn atype(&self) -> AgentType { + self.atype + } + + /// The UID-level `ext`. + #[must_use] + pub fn ext(&self) -> Option<&RawJson> { + self.ext.as_ref() + } + + /// The provider that wrote this UID. + #[must_use] + pub fn owner(&self) -> Option<&ProviderId> { + self.owner.as_ref() + } + + /// When the UID stops being served. + #[must_use] + pub fn expiry(&self) -> Expiry { + self.expiry + } + + /// Whether the UID may be served at `now`. + #[must_use] + pub fn is_live(&self, now: UnixSeconds) -> bool { + self.expiry.is_live(now) + } } /// Publisher property metadata captured when an EC entry is created. @@ -263,6 +639,17 @@ pub(crate) fn validated_stored_domain(domain: &str) -> Option { } impl KvEntry { + /// Removes partner records holding no UID this build can serve. + /// + /// Lets this build load rows written by one with a higher + /// [`MAX_STORED_UID_LENGTH`] instead of rejecting the whole row. Returns the + /// number of records removed. + pub fn drop_unreadable_ids(&mut self) -> usize { + let before = self.ids.len(); + self.ids.retain(|_, record| record.retain_readable()); + before - self.ids.len() + } + /// Creates a new live entry from the current request context. /// /// `domain` is the publisher's apex domain (e.g. `"example.com"`), @@ -293,7 +680,35 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), + } + } + + /// Whether `provider` opted out and must not be called again by `now`. + #[must_use] + pub fn opted_out(&self, provider: &ProviderId, now: UnixSeconds) -> bool { + self.opt_outs + .get(provider) + .is_some_and(|until| *until > now.get()) + } + + /// Remembers an opt-out by `provider` for [`OPT_OUT_MEMORY_SECONDS`]. + /// + /// An opt-out already remembered keeps its end time, so a provider skipped + /// because of it is asked again once the time passes. + pub fn remember_opt_out(&mut self, provider: &ProviderId, now: UnixSeconds) { + if self.opted_out(provider, now) { + return; } + self.opt_outs.insert( + provider.clone(), + now.saturating_add_secs(OPT_OUT_MEMORY_SECONDS).get(), + ); + } + + /// Forgets opt-outs whose time has passed. + pub fn prune_opt_outs(&mut self, now: UnixSeconds) { + self.opt_outs.retain(|_, until| *until > now.get()); } /// Creates a minimal live entry. @@ -303,12 +718,7 @@ impl KvEntry { #[must_use] pub fn minimal(partner_id: &str, uid: &str, now: u64) -> Self { let mut ids = BTreeMap::new(); - ids.insert( - partner_id.to_owned(), - KvPartnerId { - uid: uid.to_owned(), - }, - ); + ids.insert(partner_id.to_owned(), KvPartnerId::single(uid.to_owned())); Self { v: SCHEMA_VERSION, created: now, @@ -328,6 +738,7 @@ impl KvEntry { device: None, network: None, ids, + opt_outs: BTreeMap::new(), } } @@ -360,6 +771,7 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), } } @@ -381,12 +793,29 @@ impl KvEntry { } for (partner_id, partner_uid) in &self.ids { - if partner_uid.uid.len() > MAX_UID_LENGTH { + if partner_uid.uid.len() > MAX_STORED_UID_LENGTH { return Err(format!( - "partner ID '{partner_id}' exceeds MAX_UID_LENGTH ({})", + "partner ID '{partner_id}' exceeds MAX_STORED_UID_LENGTH ({})", partner_uid.uid.len() )); } + let oversized_field = partner_uid.eids().iter().any(|eid| { + let fields = eid.fields(); + [fields.inserter.as_deref(), fields.matcher.as_deref()] + .into_iter() + .flatten() + .any(|field| field.len() > MAX_EID_FIELD_BYTES) + }); + if oversized_field { + return Err(format!( + "partner ID '{partner_id}' has an EID inserter or matcher over MAX_EID_FIELD_BYTES" + )); + } + if partner_uid.eids().iter().any(|eid| eid.uids().is_empty()) { + return Err(format!( + "partner ID '{partner_id}' has an EID entry with no UIDs" + )); + } } if let Some(pub_properties) = &self.pub_properties { @@ -463,6 +892,9 @@ impl KvGeo { #[cfg(test)] mod tests { use super::*; + use crate::ec::eid_types::{ + AgentType, EidFields, Expiry, MatchMethod, ProviderId, RawJson, UidValue, UnixSeconds, + }; fn sample_consent_context() -> ConsentContext { ConsentContext { @@ -492,9 +924,7 @@ mod tests { let mut entry = KvEntry::new(&consent, Some(&geo), 1_741_824_000, "example.com"); entry.ids.insert( "liveramp".to_owned(), - KvPartnerId { - uid: "LR_xyz".to_owned(), - }, + KvPartnerId::single("LR_xyz".to_owned()), ); let json = serde_json::to_string(&entry).expect("should serialize KvEntry"); @@ -737,16 +1167,14 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - KvPartnerId { - uid: "x".repeat(MAX_UID_LENGTH + 1), - }, + KvPartnerId::single("x".repeat(MAX_STORED_UID_LENGTH + 1)), ); let err = entry .validate() .expect_err("should reject oversized partner UIDs"); assert!( - err.contains("MAX_UID_LENGTH"), + err.contains("MAX_STORED_UID_LENGTH"), "should describe the UID length validation failure" ); } @@ -1206,4 +1634,244 @@ mod tests { "known_browser should default to None" ); } + + fn provider_uid(id: &str, expiry: Expiry) -> KvUid { + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ) + } + + #[test] + fn legacy_partner_id_round_trips_without_eids() { + let legacy: KvPartnerId = serde_json::from_value(serde_json::json!({ "uid": "abc" })) + .expect("should read legacy record"); + assert_eq!(legacy, KvPartnerId::single("abc"), "should read legacy uid"); + assert_eq!( + serde_json::to_value(&legacy).expect("should serialize"), + serde_json::json!({ "uid": "abc" }), + "should not add eids to legacy records" + ); + } + + #[test] + fn from_eids_sets_uid_to_first_live_uid() { + let now = UnixSeconds::new(100); + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("expired", Expiry::At(UnixSeconds::new(50))), + provider_uid("live", Expiry::Never), + ], + ); + let record = KvPartnerId::from_eids(vec![eid], now).expect("should keep live uid"); + assert_eq!(record.uid, "live", "should mirror first live uid into uid"); + assert!(record.is_provider_written(), "should mark provider record"); + } + + #[test] + fn prune_expired_drops_dead_uids_and_resyncs_uid() { + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("first", Expiry::At(UnixSeconds::new(150))), + provider_uid("second", Expiry::Never), + ], + ); + let mut record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(100)).expect("should build"); + assert_eq!(record.uid, "first", "should start on first uid"); + + let changed = record.prune_expired(UnixSeconds::new(200)); + + assert!(changed, "should report pruning"); + assert_eq!(record.eids()[0].uids().len(), 1, "should drop expired uid"); + assert_eq!(record.uid, "second", "should resync uid"); + } + + #[test] + fn provider_record_round_trips_with_owner_and_exp() { + let eid = KvEid::new( + EidFields::default(), + vec![provider_uid("v", Expiry::At(UnixSeconds::new(500)))], + ); + let record = KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build"); + let json = serde_json::to_value(&record).expect("should serialize"); + assert_eq!( + json, + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "owner": "example-provider", "exp": 500 }] }] }), + "should store owner and exp" + ); + let back: KvPartnerId = serde_json::from_value(json).expect("should deserialize"); + assert_eq!(back, record, "should round-trip"); + } + + #[test] + fn validate_rejects_empty_provider_eid() { + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [{ "uids": [] }] })) + .expect("should parse"); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!(entry.validate().is_err(), "should reject EID with no UIDs"); + } + + #[test] + fn validate_accepts_liveramp_sized_uid() { + let entry = KvEntry::minimal("liveramp.example.com", &"x".repeat(703), 1); + assert!( + entry.validate().is_ok(), + "should accept a 703-byte envelope-sized UID" + ); + } + + #[test] + fn drop_unreadable_ids_removes_oversized_single_uid() { + let mut entry = KvEntry::minimal("ids.example.com", "kept", 1); + entry.ids.insert( + "long.example.com".to_owned(), + KvPartnerId::single("x".repeat(MAX_STORED_UID_LENGTH + 1)), + ); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the oversized record"); + assert_eq!( + entry.ids.keys().collect::>(), + vec!["ids.example.com"], + "should keep the readable record" + ); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + + #[test] + fn deserialize_drops_oversized_provider_uids() { + let long = "x".repeat(MAX_STORED_UID_LENGTH + 1); + let json = serde_json::json!({ + "v": SCHEMA_VERSION, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "partial.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] }, + { "uids": [{ "id": "short", "atype": 1 }] } + ] }, + "gone.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] } + ] } + } + }) + .to_string(); + let mut entry: KvEntry = + serde_json::from_str(&json).expect("should parse a row with oversized UIDs"); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the record left without UIDs"); + let partial = entry + .ids + .get("partial.example.com") + .expect("should keep the record with a readable UID"); + assert_eq!(partial.uid, "short", "should resync uid to the kept UID"); + assert_eq!(partial.eids().len(), 1, "should drop the emptied entry"); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + + #[test] + fn provider_record_round_trips_eid_level_fields() { + let ext = serde_json::value::to_raw_value(&serde_json::json!({ "k": 1 })) + .expect("should encode ext"); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + matcher: Some("matcher.example.com".into()), + mm: Some(MatchMethod::Authenticated), + ext: Some(RawJson::try_from(ext.as_ref()).expect("should accept ext")), + }; + let eid = KvEid::new(fields, vec![provider_uid("v", Expiry::Never)]); + let record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build record"); + + let json = serde_json::to_string(&record).expect("should serialize"); + let back: KvPartnerId = serde_json::from_str(&json).expect("should deserialize"); + + assert_eq!( + serde_json::from_str::(&json).expect("should parse"), + serde_json::json!({ "uid": "v", "eids": [{ + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 3, + "ext": { "k": 1 }, + "uids": [{ "id": "v", "atype": 1, "owner": "example-provider" }] + }] }), + "should store EID-level fields flat beside uids" + ); + assert_eq!(back, record, "should round-trip EID-level fields"); + } + + #[test] + fn provider_record_rejects_invalid_stored_ext() { + let big = "x".repeat(crate::ec::eid_types::MAX_EXT_BYTES); + let cases = [ + serde_json::json!({ "uid": "v", "eids": [{ "ext": [1], "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "ext": { "k": big }, "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": { "k": big } }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": "text" }] }] }), + ]; + + for case in cases { + let text = serde_json::to_string(&case).expect("should encode case"); + assert!( + serde_json::from_str::(&text).is_err(), + "should reject invalid stored ext: {text}" + ); + } + } + + #[test] + fn validate_rejects_oversized_inserter_and_matcher() { + let long = "a".repeat(crate::ec::eid_types::MAX_EID_FIELD_BYTES + 1); + for field in ["inserter", "matcher"] { + let mut eid = serde_json::json!({ "uids": [{ "id": "v", "atype": 1 }] }); + eid[field] = serde_json::Value::String(long.clone()); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [eid] })) + .expect("should parse record"); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry.ids.insert("ids.example.com".to_owned(), record); + + assert!(entry.validate().is_err(), "should reject oversized {field}"); + } + } + + #[test] + fn refresh_is_due_only_while_the_uid_is_live() { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + None, + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(1000), + }); + + assert!( + !uid.refresh_due(UnixSeconds::new(999)), + "should wait for `from`" + ); + assert!( + uid.refresh_due(UnixSeconds::new(1000)), + "should be due at `from`" + ); + assert!( + !uid.refresh_due(UnixSeconds::new(5000)), + "should stop refreshing once the UID expires" + ); + } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f24c489edd..8b608df948 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -7,9 +7,10 @@ //! context. This is called pre-routing on every request. //! //! 2. **Generate** — [`EcContext::generate_if_needed`] creates a new EC ID -//! when none exists and consent allows it. This is called only in organic -//! handlers (publisher proxy, integration proxy) — never in read-only -//! endpoints like `/_ts/api/v1/identify`. +//! when none exists and consent allows it. This is called only for +//! publisher navigations — never in integration proxies or read-only +//! endpoints like `/_ts/api/v1/identify`. The new row is written by the +//! navigation's single page-load write in finalize ([`page_load`]). //! //! # Module structure //! @@ -26,16 +27,24 @@ //! - [`rate_limiter`] — Rate limiting abstraction (implemented by adapters) //! - [`identify`] — Identity read endpoint (`GET /_ts/api/v1/identify`) //! - [`eids`] — Shared EID resolution and formatting helpers +//! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes +//! - [`page_load`] — Single-use page-load write permit and row changes +//! - [`write_plan`] — Plans a navigation's single row change +//! - [`provider`] — Identity provider interface and configuration +//! - [`coordinator`] — Server-side identity lookup coordination mod auth; pub mod admin; +pub mod after_response; pub mod batch_sync; pub mod consent; pub mod cookies; +pub mod coordinator; pub mod device; +pub mod eid_types; pub mod eids; pub mod finalize; pub mod generation; @@ -43,12 +52,15 @@ pub mod identify; pub mod kv; pub mod kv_backend; pub mod kv_types; +pub mod page_load; pub mod partner; pub mod prebid_eids; +pub mod provider; pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; +pub mod write_plan; /// Characters of an identifier kept when redacting it for a log. const LOG_ID_PREFIX_CHARS: usize = 8; @@ -83,8 +95,10 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; -use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; +use self::coordinator::results::LookupResults; +use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; +use self::page_load::{AfterResponseWrite, NavigationPermit}; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. @@ -104,9 +118,6 @@ pub enum EidSyncSource { /// Admitted `GET /_ts/page-bids` SPA navigation. #[display("page_bids")] PageBids, - /// Request that generated a new EC identity during finalization. - #[display("new_ec")] - NewEc, } /// Request-scoped view of one EC identity-graph lookup. @@ -240,6 +251,28 @@ pub fn get_ec_id(req: &Request) -> Result, Report); + +impl Clone for PermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + +/// Holds the navigation's after-response permit. Like [`PermitSlot`], +/// cloning an [`EcContext`] never copies it. +#[derive(Debug, Default)] +struct AfterResponsePermitSlot(Option); + +impl Clone for AfterResponsePermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during @@ -282,6 +315,15 @@ pub struct EcContext { pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, + /// Navigation permit issued by the router for page-load navigations. + navigation_permit: PermitSlot, + /// Permit for the navigation's follow-up write after its response, set + /// by finalize when it confirms the navigation permit. + after_response_permit: AfterResponsePermitSlot, + /// Validated provider lookup results collected for this navigation. + lookup_results: LookupResults, + /// New row built by [`Self::generate_if_needed`], written by finalize. + pending_entry: Option>, } impl EcContext { @@ -364,14 +406,21 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, }) } /// Generates a new EC ID if none exists and consent allows it. /// - /// This is the second phase of the EC lifecycle. Call this only in - /// organic handlers (publisher proxy, integration proxy, auction) — - /// never in read-only endpoints. + /// This is the second phase of the EC lifecycle. Call this only for + /// publisher navigations — never in read-only endpoints. + /// + /// Never writes KV. When `kv` is available, the new row is held as the + /// pending entry and created by the navigation's single page-load write + /// in finalize; the EC is issued only if that write succeeds. /// /// If an EC ID already exists (from the request), this is a no-op. /// If consent does not permit EC creation, this is a no-op. @@ -403,62 +452,32 @@ impl EcContext { }) })?; - const MAX_CREATE_ATTEMPTS: usize = 5; - for attempt in 0..MAX_CREATE_ATTEMPTS { - let ec_id = generation::generate_ec_id(settings, client_ip)?; - log::info!("Generated new EC ID: {}", log_id(&ec_id)); - let now = current_timestamp(); - let mut entry = KvEntry::new( - &self.consent, - self.geo_info.as_ref(), - now, - &settings.publisher.domain, - ); - entry.device = self - .device_signals - .as_ref() - .map(DeviceSignals::to_kv_device); - - if let Some(graph) = kv { - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - self.kv_snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => { - log::warn!( - "Generated EC ID collision on attempt {}/{MAX_CREATE_ATTEMPTS}", - attempt + 1 - ); - continue; - } - Err(err) => { - log::error!( - "Failed to create EC entry for id '{}' after generation: {err:?}", - log_id(&ec_id), - ); - return Err(err.change_context(TrustedServerError::EdgeCookie { - message: "Failed to persist generated EC ID to KV identity graph" - .to_string(), - })); - } - } - } - - self.ec_value = Some(ec_id); - self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); - return Ok(()); + let ec_id = generation::generate_ec_id(settings, client_ip)?; + log::info!("Generated new EC ID: {}", log_id(&ec_id)); + let now = current_timestamp(); + let mut entry = KvEntry::new( + &self.consent, + self.geo_info.as_ref(), + now, + &settings.publisher.domain, + ); + entry.device = self + .device_signals + .as_ref() + .map(DeviceSignals::to_kv_device); + + if kv.is_some() { + // The row is created by the navigation's single page-load write in + // finalize; a collision there leaves this EC unissued. + self.pending_entry = Some(Box::new(entry)); + self.kv_snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; } - - Err(Report::new(TrustedServerError::EdgeCookie { - message: format!( - "Failed to allocate a unique EC ID after {MAX_CREATE_ATTEMPTS} attempts" - ), - })) + self.ec_value = Some(ec_id); + self.ec_generated = true; + self.pull_sync_marker.invalidate_for_replaced_ec(); + Ok(()) } /// Returns the EC ID value, if present (either from request or generated). @@ -607,6 +626,59 @@ impl EcContext { self.pull_sync_marker = state; } + /// Stores the navigation permit issued by the router. + pub fn set_navigation_permit(&mut self, permit: NavigationPermit) { + self.navigation_permit = PermitSlot(Some(permit)); + } + + /// Whether this request still holds its page-load write permit. + pub(crate) fn has_navigation_permit(&self) -> bool { + self.navigation_permit.0.is_some() + } + + /// Takes the navigation permit; returns `None` after the first call. + pub(crate) fn take_navigation_permit(&mut self) -> Option { + self.navigation_permit.0.take() + } + + /// Stores the after-response permit confirmed with the navigation permit. + pub(crate) fn set_after_response_permit(&mut self, permit: AfterResponseWrite) { + self.after_response_permit = AfterResponsePermitSlot(Some(permit)); + } + + /// Takes the after-response permit; returns `None` when finalize did not + /// confirm a navigation, or after the first call. + pub fn take_after_response_permit(&mut self) -> Option { + self.after_response_permit.0.take() + } + + /// Takes the navigation permit, for adapter tests. + #[cfg(any(test, feature = "test-utils"))] + pub fn take_navigation_permit_for_test(&mut self) -> Option { + self.take_navigation_permit() + } + + /// Stores validated provider lookup results for finalize. + pub fn set_lookup_results(&mut self, results: LookupResults) { + self.lookup_results = results; + } + + /// Provider lookup results collected for this navigation. + pub(crate) fn lookup_results(&self) -> &LookupResults { + &self.lookup_results + } + + /// Retargets this navigation's lookups from an orphaned EC ID to its + /// replacement; see [`LookupResults::retarget_orphan`]. + pub(crate) fn retarget_orphan_lookups(&mut self, orphan: &str, replacement: &str) { + self.lookup_results.retarget_orphan(orphan, replacement); + } + + /// Takes the row built by [`Self::generate_if_needed`]. + pub(crate) fn take_pending_entry(&mut self) -> Option { + self.pending_entry.take().map(|entry| *entry) + } + /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); @@ -665,6 +737,10 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -690,6 +766,10 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -718,6 +798,10 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } } @@ -728,7 +812,8 @@ impl EcContext { /// native and `wasm32-wasip1` targets and to a JS-backed clock on /// `wasm32-unknown-unknown` (Cloudflare Workers), where `std::time` is not /// available. -pub(crate) fn current_timestamp() -> u64 { +#[must_use] +pub fn current_timestamp() -> u64 { checked_current_timestamp().unwrap_or(0) } @@ -766,69 +851,9 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - use crate::ec::kv_backend::{ - EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, - }; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; - /// [`EcKvStore`] wrapper whose first `collisions` `Add` writes report a - /// precondition failure, forcing generation to retry with a fresh suffix. - struct AddCollidingEcKv { - inner: InMemoryEcKv, - collisions_remaining: std::sync::Mutex, - } - - impl AddCollidingEcKv { - fn new(collisions: u32) -> Self { - Self { - inner: InMemoryEcKv::new("add-colliding-store"), - collisions_remaining: std::sync::Mutex::new(collisions), - } - } - } - - impl EcKvStore for AddCollidingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::Add) { - let mut remaining = self - .collisions_remaining - .lock() - .expect("should lock collision counter"); - if *remaining > 0 { - *remaining -= 1; - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - fn granting_consent() -> ConsentContext { ConsentContext { jurisdiction: Jurisdiction::NonRegulated, @@ -838,37 +863,48 @@ mod tests { } #[test] - fn generate_if_needed_retries_id_collision_then_persists() { + fn generate_if_needed_defers_row_creation_to_the_page_load_write() { let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.5".to_owned())); - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(2)); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + let graph = KvIdentityGraph::in_memory("gen"); - ec.generate_if_needed(&settings, Some(&graph)) - .expect("should generate after bounded collisions"); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint EC"); - assert!(ec.ec_value().is_some(), "should allocate a fresh EC ID"); - assert!(ec.ec_generated(), "should mark the EC as generated"); + let ec_id = ec_context.ec_value().expect("should set EC").to_owned(); assert!( - matches!(ec.kv_snapshot(), EcKvSnapshot::Present { .. }), - "generation should seed a present snapshot" + graph.get(&ec_id).expect("should read").is_none(), + "should not write during generation" + ); + assert!( + ec_context.take_pending_entry().is_some(), + "should hold the new row for finalize" + ); + assert!( + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }), + "should record missing row" ); } #[test] - fn generate_if_needed_errors_after_collision_exhaustion() { - let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.6".to_owned())); - // Collide on every attempt so the bounded retry is exhausted. - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(u32::MAX)); + fn cloned_context_never_carries_the_navigation_permit() { + let mut ec_context = EcContext::default(); + ec_context.set_navigation_permit(NavigationPermit::for_test()); - let result = ec.generate_if_needed(&settings, Some(&graph)); + let mut clone = ec_context.clone(); - assert!(result.is_err(), "should fail after exhausting attempts"); assert!( - ec.ec_value().is_none() && !ec.ec_generated(), - "must not activate an EC ID it could not persist" + clone.take_navigation_permit().is_none(), + "should drop permit on clone" + ); + assert!( + ec_context.take_navigation_permit().is_some(), + "should keep permit on original" ); } diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs new file mode 100644 index 0000000000..f530127bcd --- /dev/null +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -0,0 +1,365 @@ +//! The one-KV-write-per-page-load rule, enforced by types. +//! +//! A request classified as a top-level document navigation receives a +//! [`NavigationPermit`]. Once the final response status is known it becomes a +//! [`PageLoadWrite`], which [`KvIdentityGraph::commit_page_load`] consumes. +//! Neither type is `Clone`, so one navigation makes at most one write before +//! its response. +//! +//! The one exception is [`AfterResponseWrite`], confirmed alongside the page +//! load's permit. It allows one follow-up update after the response has been +//! sent, carrying only the results of after-response identity lookups. +//! +//! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load + +use edgezero_core::body::Body as EdgeBody; +use http::{Method, Request, StatusCode}; + +use super::EcKvSnapshot; +use super::kv_types::KvEntry; + +/// Issued to a request classified as a page-load navigation. +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation: + /// `GET`, `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate`, and no + /// `prefetch` token in `Sec-Purpose`. + /// + /// Call once per request, from the publisher-proxy route only. + #[must_use] + pub fn classify(request: &Request) -> Option { + let headers = request.headers(); + let header_is = |name: &str, expected: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(expected)) + }; + let prefetch = headers + .get("sec-purpose") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split([';', ',']) + .any(|token| token.trim().eq_ignore_ascii_case("prefetch")) + }); + let navigation = request.method() == Method::GET + && header_is("sec-fetch-dest", "document") + && header_is("sec-fetch-mode", "navigate") + && !prefetch; + navigation.then_some(Self(())) + } + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect (301, 302, 303, 307, 308): the redirect + /// target is the page load. Other 3xx statuses, such as 304 Not Modified, + /// are the page load itself and keep the permit. + #[must_use] + pub fn confirm(self, status: StatusCode) -> Option { + (!Self::is_redirect(status)).then_some(PageLoadWrite(())) + } + + /// Confirms the permit like [`Self::confirm`] and also issues the + /// navigation's [`AfterResponseWrite`]. + /// + /// Returns `None` for a redirect, which makes neither write. + #[must_use] + pub fn confirm_with_after_response( + self, + status: StatusCode, + ) -> Option<(PageLoadWrite, AfterResponseWrite)> { + self.confirm(status) + .map(|permit| (permit, AfterResponseWrite(()))) + } + + /// Whether `status` is a redirect (301, 302, 303, 307, 308) on which the + /// page load's write is skipped. + /// + /// Callers use this to drop work, such as pending identity lookups, whose + /// only purpose is that write. + #[must_use] + pub fn is_redirect(status: StatusCode) -> bool { + matches!( + status, + StatusCode::MOVED_PERMANENTLY + | StatusCode::FOUND + | StatusCode::SEE_OTHER + | StatusCode::TEMPORARY_REDIRECT + | StatusCode::PERMANENT_REDIRECT + ) + } + + /// Builds a permit without a request, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +impl PageLoadWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} +} + +/// Proof that the current navigation may make one follow-up update after its +/// response has been sent. +/// +/// Issued only by [`NavigationPermit::confirm_with_after_response`] and +/// consumed by [`KvIdentityGraph::commit_after_response`]. It is not `Clone`, +/// so a navigation makes at most one follow-up write. +/// +/// [`KvIdentityGraph::commit_after_response`]: super::kv::KvIdentityGraph::commit_after_response +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +impl AfterResponseWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} + + /// Builds a permit without a navigation, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + +/// KV generation a conditional write expects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Generation(u64); + +impl Generation { + /// Reads the generation captured in `snapshot` for `ec_id`. + /// + /// Returns `None` when the snapshot has no generation, so no blind + /// overwrite can be planned. + #[must_use] + pub fn of(snapshot: &EcKvSnapshot, ec_id: &str) -> Option { + snapshot.generation_for(ec_id).map(Self) + } + + /// The raw generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { + /// The full new row. + entry: KvEntry, + /// Generation the row must still have. + expected: Generation, + }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + /// The row changed since it was read. + #[display("row changed since it was read")] + Conflict, + /// A create found an existing row. + #[display("row already exists")] + AlreadyExists, + /// The KV store rejected the write. + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::KvEntry; + + fn request(headers: &[(&str, &str)]) -> http::Request { + let mut builder = http::Request::builder() + .method(http::Method::GET) + .uri("https://www.example.com/"); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + const DOCUMENT: [(&str, &str); 2] = [ + ("sec-fetch-dest", "document"), + ("sec-fetch-mode", "navigate"), + ]; + + #[test] + fn classify_accepts_top_level_document_navigation() { + assert!( + NavigationPermit::classify(&request(&DOCUMENT)).is_some(), + "should permit navigation" + ); + } + + #[test] + fn classify_rejects_missing_fetch_metadata_iframes_and_prefetch() { + assert!( + NavigationPermit::classify(&request(&[])).is_none(), + "should reject without fetch metadata" + ); + assert!( + NavigationPermit::classify(&request(&[ + ("sec-fetch-dest", "iframe"), + ("sec-fetch-mode", "navigate") + ])) + .is_none(), + "should reject iframe" + ); + let mut prefetch = DOCUMENT.to_vec(); + prefetch.push(("sec-purpose", "prefetch;prerender")); + assert!( + NavigationPermit::classify(&request(&prefetch)).is_none(), + "should reject prefetch" + ); + } + + #[test] + fn confirm_rejects_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::FOUND) + .is_none(), + "should not write on redirect" + ); + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_FOUND) + .is_some(), + "should write on error page" + ); + } + + #[test] + fn confirm_keeps_not_modified_and_drops_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_MODIFIED) + .is_some(), + "should write on 304 Not Modified" + ); + for status in [ + http::StatusCode::FOUND, + http::StatusCode::TEMPORARY_REDIRECT, + ] { + assert!( + NavigationPermit::for_test().confirm(status).is_none(), + "should not write on redirect {status}" + ); + } + } + + #[test] + fn is_redirect_matches_only_write_skipping_statuses() { + for status in [ + http::StatusCode::MOVED_PERMANENTLY, + http::StatusCode::FOUND, + http::StatusCode::SEE_OTHER, + http::StatusCode::TEMPORARY_REDIRECT, + http::StatusCode::PERMANENT_REDIRECT, + ] { + assert!( + NavigationPermit::is_redirect(status), + "should treat {status} as a redirect" + ); + } + for status in [http::StatusCode::OK, http::StatusCode::NOT_MODIFIED] { + assert!( + !NavigationPermit::is_redirect(status), + "should not treat {status} as a redirect" + ); + } + } + + fn permit() -> PageLoadWrite { + NavigationPermit::for_test() + .confirm(http::StatusCode::OK) + .expect("should confirm") + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + #[test] + fn commit_create_reports_existing_row() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect("should create row"); + + let error = graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect_err("should refuse second create"); + + assert!( + matches!(error.current_context(), KvWriteError::AlreadyExists), + "should report existing row" + ); + } + + #[test] + fn commit_update_with_stale_generation_conflicts_without_retry() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let snapshot = graph.load_snapshot(&ec_id); + let expected = Generation::of(&snapshot, &ec_id).expect("should read generation"); + graph + .create_or_revive(&ec_id, &live_entry()) + .expect("should no-op on live row"); + graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect("should write at current generation"); + + let error = graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect_err("should reject stale generation"); + + assert!( + matches!(error.current_context(), KvWriteError::Conflict), + "should report conflict" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index d823cb32fb..768a431188 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,6 +157,7 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } + // Provider-owned sources stay in the list; the write planner skips them. dedupe_partner_updates(updates) } @@ -294,10 +295,7 @@ fn structured_cookie_eids_to_openrtb(entries: Vec) -> Vec) -> Vec { entries .into_iter() .filter(|entry| !entry.source.is_empty() && !entry.id.is_empty()) - .map(|entry| Eid { - source: entry.source, - uids: vec![Uid { - id: entry.id, - atype: (entry.atype >= 0).then_some(entry.atype), - ext: None, - }], + .map(|entry| { + Eid::new( + entry.source, + vec![Uid { + id: entry.id, + atype: (entry.atype >= 0).then_some(entry.atype), + ext: None, + }], + ) }) .collect() } @@ -636,22 +636,22 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_stale", "atype": 1}]} ])); let body_eids = vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_fresh".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]; let updates = collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs new file mode 100644 index 0000000000..9f479fa296 --- /dev/null +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -0,0 +1,926 @@ +//! Identity provider interface (spec §7). +//! +//! Providers are pure: they build a lookup request and parse its response. +//! They never perform HTTP, KV, cookie or clock work; the coordinator in +//! [`crate::ec::coordinator`] executes requests and applies consent, +//! transport, validation, expiry and persistence policy. +//! +//! Provider implementations live in the `trusted-server-identity-providers` +//! crate, which depends on core; core never depends on it. + +use core::fmt; +use std::borrow::Cow; +use std::collections::HashSet; +use std::net::IpAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize as _; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use url::Url; + +use crate::consent::ConsentContext; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::kv_types::{KvEid, KvEntry}; +use crate::ec::registry::PartnerRegistry; +use crate::error::TrustedServerError; +use crate::settings::{Ec, IdentityProviderConfig, ProviderSecrets}; + +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. Coordinators are built once and shared for the +/// application's lifetime, so implementations are `Send + Sync`. +pub trait IdentityProvider: Send + Sync { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be + /// parsed at all. Individually malformed entries are returned and dropped + /// by the coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; + + /// Creates IDs for due sources without a lookup, for a provider whose + /// IDs the operator owns rather than a vendor server. + /// + /// Runs when a source is due. A returned entry goes through the same + /// validation and write plan as a lookup result, and no request is built. + /// The default creates nothing. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + let _ = lookup; + None + } +} + +impl IdentityProvider for Box

{ + fn capabilities(&self) -> &ProviderCapabilities { + (**self).capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + (**self).build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + (**self).parse_response(response) + } + + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + (**self).generate(lookup) + } +} + +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +/// One source a provider can return. +#[derive(Debug, Clone)] +pub struct SourceCapability { + /// The source domain. + pub source: SourceDomain, + /// Visitor input the source needs. + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +/// Limits a provider declares for one response. +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + /// Maximum EID entries kept from one response. + pub max_eids: NonZeroUsize, + /// Maximum UIDs kept per EID entry. + pub max_uids_per_eid: NonZeroUsize, + /// Whether EID-level fields are kept. + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + /// The fields are validated and stored. + Preserved, + /// The fields are dropped without rejecting the entry. + Ignored, +} + +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +/// Client signals lent to providers. +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + /// Client IP address. + pub ip: Option, + /// Client `User-Agent`. + pub user_agent: Option<&'a str>, +} + +/// This provider's stored entries, borrowed from the row. +/// +/// Only entries whose every UID this provider wrote are visible, so another +/// provider's refresh state never reaches it. +#[derive(Debug, Clone, Copy)] +pub struct OwnedEntries<'a> { + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], + owner: Option<&'a ProviderId>, +} + +impl<'a> OwnedEntries<'a> { + /// Views the entries of `entry` for `sources` that `owner` wrote. + #[must_use] + pub fn new( + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], + owner: &'a ProviderId, + ) -> Self { + Self { + entry, + sources, + owner: Some(owner), + } + } + + /// A view holding no entries, for a provider with nothing stored. + #[must_use] + pub fn empty() -> Self { + Self { + entry: None, + sources: &[], + owner: None, + } + } + + /// Iterates `(source, entry)` pairs in source order. + pub fn iter(&self) -> impl Iterator + 'a { + let entry = self.entry; + let owner = self.owner; + self.sources.iter().flat_map(move |source| { + entry + .and_then(|entry| entry.ids.get_key_value(source.as_str())) + .into_iter() + .flat_map(move |(key, record)| { + record + .eids() + .iter() + .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == owner)) + .map(move |eid| (key.as_str(), eid)) + }) + }) + } +} + +/// A validated lowercase hex SHA-256 email digest. V1 never constructs one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HashedEmail(Box); + +impl HashedEmail { + /// Returns the digest. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + /// HTTP method. + pub method: http::Method, + /// Request URL; must be `https` with an allowlisted host. + pub url: Url, + /// Request headers. + pub headers: http::HeaderMap, + /// Request body. + pub body: bytes::Bytes, + /// Provider state returned with the response in + /// [`ProviderResponse::context`]; never sent. + pub context: Option, +} + +impl fmt::Debug for ProviderRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRequest") + .field("method", &self.method) + .field("host", &self.url.host_str()) + .finish_non_exhaustive() + } +} + +/// A provider response, borrowed from the transport. +/// +/// Its `Debug` output omits the body, headers and context, which can hold +/// identifiers or a reply-decryption key. +#[derive(Clone, Copy)] +pub struct ProviderResponse<'body> { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: &'body http::HeaderMap, + /// Response body. + pub body: &'body [u8], + /// The [`ProviderRequest::context`] of the request this answers. + pub context: Option<&'body str>, +} + +impl fmt::Debug for ProviderResponse<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderResponse") + .field("status", &self.status) + .field("body_len", &self.body.len()) + .field("has_context", &self.context.is_some()) + .finish_non_exhaustive() + } +} + +/// A parsed provider response. +#[derive(Debug)] +pub enum LookupOutcome<'body> { + /// Entries were returned. + Found(Vec>), + /// The provider knows no ID for this visitor. + NoMatch, + /// The visitor opted out with the provider; its stored UIDs are removed. + OptedOut, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + /// EID source domain. + pub source: Cow<'body, str>, + /// `OpenRTB` `inserter`. + pub inserter: Option>, + /// `OpenRTB` `matcher`. + pub matcher: Option>, + /// `OpenRTB` `mm`. + pub mm: Option, + /// EID-level `ext`. + pub ext: Option<&'body RawValue>, + /// UIDs of the entry. + pub uids: Vec>, +} + +/// One UID as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderUid<'body> { + /// UID value. + pub id: Cow<'body, str>, + /// Agent type; filled from partner config when missing. + pub atype: Option, + /// UID-level `ext`. + pub ext: Option<&'body RawValue>, + /// Expiry as the provider states it. + pub expiry: ProviderExpiry, + /// Private refresh state kept with the UID in the row and never served. + pub refresh: Option>, +} + +/// Refresh state a provider asks the framework to keep with a UID. +/// +/// The stored UID's source becomes due again at `from`, and the state is +/// handed back through [`LookupInput::current`]. `Debug` hides the state, +/// which may hold a credential. +pub struct ProviderRefresh<'body> { + /// Provider-defined state, such as a refresh token. + pub state: Cow<'body, str>, + /// When the provider should be asked to refresh the UID. + pub from: ProviderExpiry, +} + +impl fmt::Debug for ProviderRefresh<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } +} + +/// Expiry as a provider states it. The coordinator turns it into +/// [`crate::ec::eid_types::Expiry`] using its own clock, so providers never +/// read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + /// No expiry was stated. + Unspecified, + /// Expires at this time. + At(UnixSeconds), + /// Expires this long after the request. + After(Duration), +} + +/// Why a provider could not build a request or parse a response. +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + /// The input cannot form a request. + #[display("provider request could not be built")] + Request, + /// The body cannot be parsed at all. + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} + +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +#[derive(Debug, Clone, Copy)] +pub struct ProviderCommon<'a> { + /// Configured lookup endpoint. + pub endpoint: &'a Url, + /// Configured sources. + pub sources: &'a [SourceDomain], + /// Resolved secrets. + pub secrets: &'a ProviderSecrets, +} + +/// Why a provider configuration was rejected. +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + /// No implementation is registered for the configured `kind`. + #[display("provider kind is not registered")] + UnknownKind, + /// The configuration is unusable. + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} + +/// Parses `config.settings` for `K` and builds a boxed provider. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when the settings do not +/// deserialize into `K::Settings`, a source or the endpoint is invalid, or +/// [`ProviderKind::build`] rejects them. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + let settings = K::Settings::deserialize(serde_json::Value::Object(config.settings.clone())) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("settings for provider `{}` are invalid", config.id))?; + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + let sources = parse_sources(config)?; + let common = ProviderCommon { + endpoint: &endpoint, + sources: &sources, + secrets, + }; + let provider = K::build(&common, settings)?; + Ok(Box::new(provider)) +} + +/// Parses and normalizes a provider's configured sources. +pub(crate) fn parse_sources( + config: &IdentityProviderConfig, +) -> Result, Report> { + config + .sources + .iter() + .map(|source| SourceDomain::try_from(source.as_str())) + .collect::, _>>() + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("provider `{}` has an invalid source", config.id)) +} + +/// Whether `url` is `https` and its host is one of `allowed_domains`, +/// compared ASCII case-insensitively. +#[must_use] +pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S]) -> bool { + url.scheme() == "https" + && url.host_str().is_some_and(|host| { + allowed_domains + .iter() + .any(|allowed| allowed.as_ref().eq_ignore_ascii_case(host)) + }) +} + +/// Checks the configuration rules core owns for every enabled provider +/// (spec §12.1): the endpoint is an `https` URL whose host is in +/// `allowed_domains`, it claims at least one source, every source has an +/// `[[ec.partners]]` entry, no source is claimed by two enabled providers, and +/// every enabled provider has a non-empty id of its own. +/// +/// Unknown kinds and capability checks need the provider crate; the adapter +/// builds the coordinators at startup and fails to start on either. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] naming the first offending +/// provider. +pub fn validate_enabled_configs<'a>( + configs: impl IntoIterator, + partners: &PartnerRegistry, +) -> Result<(), Report> { + let mut claimed: HashSet = HashSet::new(); + let mut ids: HashSet<&str> = HashSet::new(); + for config in configs.into_iter().filter(|config| config.enabled) { + // The id keys UID ownership, opt-outs and the rate-limit bucket in + // the row, so two providers sharing one would read, refresh and clear + // each other's IDs. + let id = config.id.trim(); + if id.is_empty() || !ids.insert(id) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "provider id `{}` is empty or used by another enabled provider", + config.id + ))); + } + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + if !destination_allowed(&endpoint, &config.allowed_domains) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "endpoint for provider `{}` must be https with a host in allowed_domains", + config.id + ))); + } + let sources = parse_sources(config)?; + if sources.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach(format!("provider `{}` claims no sources", config.id))); + } + for source in sources { + if partners.find_by_source_domain(source.as_str()).is_none() { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` has no [[ec.partners]] entry", + config.id + ))); + } + if !claimed.insert(source.clone()) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is claimed by another enabled provider", + config.id + ))); + } + } + } + Ok(()) +} + +/// Fails when `[ec]` enables an identity provider on an adapter that has no +/// provider implementations. +/// +/// Ignoring the provider would leave its sources open to browser ingest with +/// no sign that it never runs. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming the first enabled +/// provider. +pub fn reject_enabled_providers(ec: &Ec, adapter: &str) -> Result<(), Report> { + match ec.identity_providers.iter().find(|config| config.enabled) { + Some(config) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "identity provider `{}` is enabled, but the {adapter} adapter does not run identity providers", + config.id + ), + })), + None => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + use serde_json::json; + + struct Fixed(ProviderCapabilities); + + impl IdentityProvider for Fixed { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Ok(LookupOutcome::NoMatch) + } + } + + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FixedSettings { + max_eids: usize, + } + + struct FixedKind; + + impl ProviderKind for FixedKind { + const KIND: &'static str = "fixed"; + type Settings = FixedSettings; + type Provider = Fixed; + + fn build( + common: &ProviderCommon<'_>, + settings: FixedSettings, + ) -> Result> { + let max_eids = NonZeroUsize::new(settings.max_eids) + .ok_or_else(|| Report::new(ProviderConfigError::Invalid))?; + Ok(Fixed(ProviderCapabilities { + sources: common + .sources + .iter() + .map(|source| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids, + max_uids_per_eid: max_eids, + eid_level_fields: EidLevelFields::Preserved, + }, + })) + } + } + + fn config(settings: &serde_json::Value) -> IdentityProviderConfig { + serde_json::from_value(json!({ + "id": "example-provider", + "kind": FixedKind::KIND, + "endpoint": "https://ids.example.com/v1", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "settings": settings, + })) + .expect("should parse provider config") + } + + fn partners() -> PartnerRegistry { + let partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn assert_invalid(error: &Report) { + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn build_kind_parses_typed_settings_and_boxes_provider() { + let provider = build_kind::( + &config(&json!({ "max_eids": 2 })), + &ProviderSecrets::default(), + ) + .expect("should build"); + + assert_eq!( + provider.capabilities().sources.len(), + 1, + "should claim configured source" + ); + assert_eq!( + provider.capabilities().shape.max_eids.get(), + 2, + "should apply typed settings" + ); + } + + #[test] + fn build_kind_rejects_unknown_settings() { + let error = build_kind::( + &config(&json!({ "unexpected": true })), + &ProviderSecrets::default(), + ) + .err() + .expect("should reject"); + + assert_invalid(&error); + } + + #[test] + fn config_parses_from_toml_with_a_settings_table() { + let parsed: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"fixed\"\nendpoint = \"https://ids.example.com/\"\n[settings]\nmax_eids = 3", + ) + .expect("should parse TOML config"); + + assert_eq!(parsed.kind(), "fixed", "should keep kind"); + assert!(parsed.enabled, "should default to enabled"); + assert_eq!( + parsed.settings.get("max_eids"), + Some(&json!(3)), + "should keep provider settings" + ); + } + + #[test] + fn request_debug_hides_query_and_headers() { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("Bearer example-secret"), + ); + let request = ProviderRequest { + method: http::Method::GET, + url: Url::parse("https://ids.example.com/v1?token=example-secret") + .expect("should parse"), + headers, + body: bytes::Bytes::new(), + context: None, + }; + + let printed = format!("{request:?}"); + + assert!( + !printed.contains("example-secret"), + "should not print secrets" + ); + assert!(printed.contains("ids.example.com"), "should print host"); + } + + #[test] + fn response_debug_hides_body_and_context() { + let headers = http::HeaderMap::new(); + let response = ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body: b"example-uid", + context: Some("example-key"), + }; + + let printed = format!("{response:?}"); + + assert!( + !printed.contains("example-uid"), + "should not print the body" + ); + assert!( + !printed.contains("example-key"), + "should not print the context" + ); + assert!(printed.contains("200"), "should print the status"); + } + + #[test] + fn validate_enabled_configs_accepts_valid_config() { + validate_enabled_configs([&config(&json!({}))], &partners()) + .expect("should accept valid config"); + } + + #[test] + fn validate_enabled_configs_rejects_endpoint_outside_allowlist() { + let mut outside = config(&json!({})); + outside.endpoint = "https://other.example.com/".to_owned(); + let mut plain_http = config(&json!({})); + plain_http.endpoint = "http://ids.example.com/".to_owned(); + + for rejected in [outside, plain_http] { + let error = validate_enabled_configs([&rejected], &partners()) + .expect_err("should reject endpoint"); + assert_invalid(&error); + } + } + + #[test] + fn validate_enabled_configs_rejects_source_without_partner() { + let mut unknown = config(&json!({})); + unknown.sources = vec!["unknown.example.com".to_owned()]; + + let error = validate_enabled_configs([&unknown], &partners()) + .expect_err("should reject source without partner"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_source_claimed_twice() { + let first = config(&json!({})); + let mut second = config(&json!({})); + second.id = "second-provider".to_owned(); + + let error = validate_enabled_configs([&first, &second], &partners()) + .expect_err("should reject double claim"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_a_provider_without_sources() { + let mut empty = config(&json!({})); + empty.sources = Vec::new(); + + let error = validate_enabled_configs([&empty], &partners()) + .expect_err("should reject a provider that claims no sources"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_shared_or_empty_id() { + let registry = PartnerRegistry::from_config(&[ + serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + serde_json::from_value(json!({ + "name": "other", "source_domain": "other.example.com" + })) + .expect("should build partner"), + ]) + .expect("should build registry"); + let first = config(&json!({})); + let mut shared = config(&json!({})); + shared.sources = vec!["other.example.com".to_owned()]; + let mut empty = config(&json!({})); + empty.id = " ".to_owned(); + + let error = validate_enabled_configs([&first, &shared], ®istry) + .expect_err("should reject a shared provider id"); + assert_invalid(&error); + let error = validate_enabled_configs([&empty], ®istry) + .expect_err("should reject an empty provider id"); + assert_invalid(&error); + } + + #[test] + fn reject_enabled_providers_fails_only_for_an_enabled_provider() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + let mut disabled = config(&json!({})); + disabled.enabled = false; + ec.identity_providers = vec![disabled]; + reject_enabled_providers(&ec, "Axum").expect("should accept a disabled provider"); + + ec.identity_providers.push(config(&json!({}))); + let error = + reject_enabled_providers(&ec, "Axum").expect_err("should reject an enabled provider"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::Configuration { .. } + ), + "should report a configuration error" + ); + } + + #[test] + fn validate_enabled_configs_ignores_disabled_providers() { + let first = config(&json!({})); + let mut disabled = config(&json!({})); + disabled.enabled = false; + disabled.endpoint = "https://other.example.com/".to_owned(); + + validate_enabled_configs([&first, &disabled], &partners()) + .expect("should ignore disabled provider"); + } + + #[test] + fn owned_entries_yields_only_configured_sources() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3, "owner": "example-provider" }] }] }, + "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3, "owner": "example-provider" }] }] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); + + let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources, &owner) + .iter() + .map(|(source, _)| source) + .collect(); + + assert_eq!( + owned, + ["ids.example.com"], + "should yield owned sources only" + ); + } + + #[test] + fn empty_owned_entries_yields_nothing() { + assert_eq!( + OwnedEntries::empty().iter().count(), + 0, + "should yield no entries" + ); + } + + #[test] + fn owned_entries_hides_entries_another_provider_wrote() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [ + { "uids": [{ "id": "u-1", "atype": 3, "owner": "former-provider" }] }, + { "uids": [{ "id": "u-2", "atype": 3 }] } + ] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); + + let count = OwnedEntries::new(Some(&entry), &sources, &owner) + .iter() + .count(); + + assert_eq!(count, 0, "should hide entries this provider did not write"); + } +} diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 191e6c6d24..ead979b312 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -560,6 +560,8 @@ mod tests { pull_sync_ttl_sec: ttl_sec, pull_sync_rate_limit: 20, ts_pull_token: Some(Redacted::new("token".to_owned())), + provider_owned: false, + owner_tcf_vendor_id: None, } } @@ -652,9 +654,7 @@ mod tests { if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid".to_owned()), ); } ec_context.set_kv_snapshot(snapshot); @@ -702,9 +702,7 @@ mod tests { entry.ids.insert( "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-b".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-b".to_owned()), ); assert!( @@ -801,12 +799,12 @@ mod tests { #[test] fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(513); + let long_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let body = format!("{{\"uid\":\"{long_uid}\"}}"); let response = make_response(200, body.as_bytes()); let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding 512 bytes"); + assert!(uid.is_none(), "should reject uid exceeding MAX_UID_LENGTH"); } #[test] @@ -1189,7 +1187,7 @@ mod tests { let (entry, _) = graph .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!( entry.ids.is_empty(), "no partner UID may be written back onto a tombstoned row" @@ -1213,9 +1211,7 @@ mod tests { entry.consent.ok = true; entry.ids.insert( "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "already-known".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("already-known".to_owned()), ); graph .create(&ec_id, &entry) diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 11df47296f..7cc86898b2 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -357,8 +357,8 @@ mod tests { fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { let mut settings = create_test_settings(); settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); (settings, registry) } @@ -375,9 +375,7 @@ mod tests { for domain in domains { entry.ids.insert( (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: format!("uid-{domain}"), - }, + crate::ec::kv_types::KvPartnerId::single(format!("uid-{domain}")), ); } EcKvSnapshot::Present { @@ -510,9 +508,7 @@ mod tests { let mut tombstone = KvEntry::tombstone(1_000); tombstone.ids.insert( "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "stale".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("stale".to_owned()), ); let snapshots = [ live_snapshot(EC_ID, &[]), diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index c4637b4310..c82f6c0bd7 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -5,12 +5,13 @@ //! lookup by source domain and API key hash. use std::collections::{HashMap, HashSet}; +use std::num::NonZeroU16; use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::redacted::Redacted; -use crate::settings::EcPartner; +use crate::settings::{Ec, EcPartner}; use super::partner::{hash_api_key, normalize_partner_source_domain}; @@ -47,6 +48,12 @@ pub struct PartnerConfig { pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, + /// Whether an enabled identity provider claims this source. Browser + /// ingest skips it and batch sync rejects it. + pub provider_owned: bool, + /// TCF vendor ID of the provider that owns this source, when configured. + /// Its UIDs leave the server only while that vendor has consent. + pub owner_tcf_vendor_id: Option, } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -205,6 +212,52 @@ impl PartnerRegistry { }) } + /// Builds the registry from `[ec]` settings without provider ownership. + /// + /// For adapters with no identity-provider implementations, which refuse + /// to start with an enabled provider: no source is provider-owned, so + /// browser-supplied IDs are ingested for every source. Use + /// [`Self::from_ec_with_kinds`] where kinds are known. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`]. + pub fn from_ec(ec: &Ec) -> Result> { + Self::from_config(&ec.partners) + } + + /// Builds the registry from `[ec]` settings and marks every source claimed + /// by an enabled identity provider whose kind is registered as + /// provider-owned. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`], and + /// [`TrustedServerError::Configuration`] for an enabled provider whose + /// kind is not registered: it could never run, and its sources would + /// silently go back to browser ingest. + pub fn from_ec_with_kinds( + ec: &Ec, + is_registered_kind: impl Fn(&str) -> bool, + ) -> Result> { + let mut registry = Self::from_config(&ec.partners)?; + for provider in ec.identity_providers.iter().filter(|p| p.enabled) { + if !is_registered_kind(provider.kind()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "identity provider `{}` has kind `{}`, which this build does not include", + provider.id, + provider.kind() + ), + })); + } + for source in &provider.sources { + registry.mark_provider_owned(source, provider.tcf_vendor_id); + } + } + Ok(registry) + } + /// Returns an empty registry (no partners configured). #[must_use] pub fn empty() -> Self { @@ -234,12 +287,41 @@ impl PartnerRegistry { self.get(domain) } - /// Returns all partners with `pull_sync_enabled = true`. + /// Marks `source` as owned by an enabled identity provider, recording the + /// provider's TCF vendor ID, if any. + /// + /// Returns `false` when the source has no `[[ec.partners]]` entry. + pub fn mark_provider_owned(&mut self, source: &str, tcf_vendor_id: Option) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.provider_owned = true; + partner.owner_tcf_vendor_id = tcf_vendor_id; + true + } + None => false, + } + } + + /// Whether an enabled identity provider owns `source`. + #[must_use] + pub fn is_provider_owned(&self, source: &str) -> bool { + self.find_by_source_domain(source) + .is_some_and(|partner| partner.provider_owned) + } + + /// Returns all partners with `pull_sync_enabled = true` whose source no + /// identity provider owns. + /// + /// A provider writes full-fidelity records for the sources it owns, so a + /// pull-sync write there would overwrite them with a single UID. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { self.by_source_domain .values() - .filter(|p| p.pull_sync_enabled) + .filter(|p| p.pull_sync_enabled && !p.provider_owned) .collect() } @@ -305,6 +387,8 @@ fn build_partner_config( pull_sync_ttl_sec: partner.pull_sync_ttl_sec, pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), + provider_owned: false, + owner_tcf_vendor_id: None, } } @@ -435,6 +519,68 @@ mod tests { } } + #[test] + fn from_ec_marks_enabled_provider_sources_only() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![ + make_partner("ids.example.com", &valid_api_token("ids-partner")), + make_partner("other.example.com", &valid_api_token("other-partner")), + ]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"k\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + toml::from_str( + "id = \"b\"\nkind = \"k\"\nenabled = false\nendpoint = \"https://ids.example.com/\"\nsources = [\"other.example.com\"]", + ) + .expect("should parse disabled provider"), + ]; + + let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); + + assert!( + registry.is_provider_owned("ids.example.com"), + "should mark enabled provider source" + ); + assert!( + !registry.is_provider_owned("other.example.com"), + "should not mark disabled provider source" + ); + } + + #[test] + fn from_ec_with_kinds_rejects_an_unregistered_kind() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![make_partner( + "ids.example.com", + &valid_api_token("ids-partner"), + )]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + ]; + + let error = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect_err("should reject an enabled provider with an unregistered kind"); + let plain = PartnerRegistry::from_ec(&ec).expect("should build registry"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::Configuration { .. } + ), + "should report a configuration error" + ); + assert!( + !plain.is_provider_owned("ids.example.com"), + "should not claim sources without provider implementations" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); @@ -569,6 +715,25 @@ mod tests { assert!(result.is_err(), "should reject invalid source domain"); } + #[test] + fn pull_enabled_partners_excludes_provider_owned_sources() { + let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); + pull_partner.api_token = None; + pull_partner.pull_sync_enabled = true; + pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); + pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; + pull_partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); + let mut registry = + PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); + + registry.mark_provider_owned("pull.example.com", None); + + assert!( + registry.pull_enabled_partners().is_empty(), + "should not pull sources an identity provider owns" + ); + } + #[test] fn pull_enabled_partners_filters_correctly() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs new file mode 100644 index 0000000000..66d6f39b15 --- /dev/null +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -0,0 +1,857 @@ +//! Plans a navigation's single row change. +//! +//! Combines browser ingest (fill missing) and validated provider results +//! (replace the provider's sources, or remove them on an opt-out), prunes expired UIDs and enforces +//! [`MAX_ENTRY_BYTES`]. Pure: no I/O. +//! +//! Browser ingest skips provider-owned sources entirely (spec §8.4), so a +//! source the provider cleared on an opt-out is never refilled from the +//! browser's `ts-eids` cookie. + +use super::EcKvSnapshot; +use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; +use super::eid_types::{SourceDomain, UnixSeconds}; +use super::kv::PartnerIdUpdate; +use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; +use super::page_load::{Generation, RowChange}; +use super::registry::PartnerRegistry; + +/// Inputs to [`plan_write`]. +#[derive(Debug, Clone, Copy)] +pub struct WritePlanInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read for this request. + pub snapshot: &'a EcKvSnapshot, + /// The new row when this navigation creates one (new EC or recovery). + pub new_entry: Option<&'a KvEntry>, + /// Browser updates from `ts-eids` and `sharedId`, already reduced by the + /// ingest rules and consent-gated. + pub browser_ingest: &'a [PartnerIdUpdate], + /// Validated provider results. + pub lookups: &'a LookupResults, + /// Partner registry, for provider ownership. + pub registry: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Plans the navigation's single row change. +/// +/// Returns `None` when nothing would change, when the row is a tombstone, or +/// when an update has no generation to guard it. +#[must_use] +pub fn plan_write(input: &WritePlanInput<'_>) -> Option { + let (base, expected) = match input.new_entry { + Some(entry) => (entry.clone(), None), + None => { + let entry = input.snapshot.entry_for(input.ec_id)?; + if !entry.consent.ok { + return None; + } + let Some(expected) = Generation::of(input.snapshot, input.ec_id) else { + log::info!( + "EC write planner: skipping update for '{}' (no_generation)", + super::log_id(input.ec_id) + ); + return None; + }; + (entry.clone(), Some(expected)) + } + }; + + // Prune first so an expired record frees its slot for browser ingest. + // Pruning alone never triggers a write; it rides along with a real change. + let mut entry = base; + for record in entry.ids.values_mut() { + record.prune_expired(input.now); + } + entry.prune_opt_outs(input.now); + entry.ids.retain(|_, record| !record.uid.is_empty()); + let pruned = entry.clone(); + + let lookups_apply = input.lookups.is_for(input.ec_id); + apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); + let written = if lookups_apply { + apply_lookups(&mut entry, input.lookups, input.now) + } else { + if !input.lookups.is_empty() { + log::info!( + "EC write planner: discarding lookups made for another EC ID than '{}'", + super::log_id(input.ec_id) + ); + } + Vec::new() + }; + enforce_size_cap(&mut entry, &written); + // Compared after the cap, which may trim the only real change away. + let changed = entry != pruned; + + match expected { + None => Some(RowChange::Create(entry)), + Some(_) if !changed => None, + Some(expected) => Some(RowChange::Update { entry, expected }), + } +} + +/// Fills missing sources from the browser, skipping provider-owned ones. +fn apply_browser_ingest( + entry: &mut KvEntry, + updates: &[PartnerIdUpdate], + registry: &PartnerRegistry, +) { + for update in updates { + if registry.is_provider_owned(&update.partner_id) { + log::trace!( + "EC write planner: skipping browser UID for provider-owned source '{}'", + update.partner_id + ); + continue; + } + entry + .ids + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + } +} + +fn apply_lookups( + entry: &mut KvEntry, + lookups: &LookupResults, + now: UnixSeconds, +) -> Vec { + let mut written = Vec::new(); + for lookup in lookups.iter() { + for (source, eids) in &lookup.entries { + if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { + entry.ids.insert(source.as_str().to_owned(), record); + written.push(source.clone()); + } + } + if lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) { + entry.remember_opt_out(&lookup.provider, now); + } + for source in &lookup.cleared { + if entry.ids.remove(source.as_str()).is_some() { + log::info!( + "EC write planner: removing '{source}' after provider '{}' opt-out", + lookup.provider + ); + } + } + } + written +} + +fn serialized_len(entry: &KvEntry) -> usize { + serde_json::to_vec(entry).map_or(usize::MAX, |bytes| bytes.len()) +} + +/// Serialized bytes one `ids` member adds to the row: its record, its +/// escaped key, the colon and a separating comma. +fn member_len(source: &str, record: &KvPartnerId) -> usize { + let key = serde_json::to_vec(source).map_or(usize::MAX, |bytes| bytes.len()); + let value = serde_json::to_vec(record).map_or(usize::MAX, |bytes| bytes.len()); + key.saturating_add(value).saturating_add(2) +} + +/// Shrinks `entry` until it serializes within [`MAX_ENTRY_BYTES`]. +/// +/// # Performance +/// +/// Tracks the row size by the bytes each drop removes and serializes the +/// whole row again only when that estimate reaches the cap, so the cost is +/// linear in the number of dropped UIDs and records. +fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { + let mut size = serialized_len(entry); + // Drop this navigation's provider UIDs first, newest source last-in first. + while size > MAX_ENTRY_BYTES { + let Some(source) = written.iter().rev().find(|source| { + entry + .ids + .get(source.as_str()) + .is_some_and(KvPartnerId::is_provider_written) + }) else { + break; + }; + let record = entry + .ids + .get_mut(source.as_str()) + .expect("should hold a written source"); + let before = member_len(source.as_str(), record); + record.drop_last_uid(); + let after = if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + 0 + } else { + member_len(source.as_str(), record) + }; + size = size.saturating_sub(before.saturating_sub(after)); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } + } + if size <= MAX_ENTRY_BYTES { + return; + } + // No provider UIDs from this navigation are left: drop whole records, + // largest first, so every later write still fits the cap. + let mut by_size: Vec<(usize, String)> = entry + .ids + .iter() + .map(|(source, record)| (member_len(source, record), source.clone())) + .collect(); + // A stable ascending sort pops the last of equally large records first. + by_size.sort_by_key(|(len, _)| *len); + while size > MAX_ENTRY_BYTES { + let Some((len, largest)) = by_size.pop() else { + log::warn!("EC write planner: row exceeds size cap with no partner IDs left to drop"); + return; + }; + log::warn!("EC write planner: dropping '{largest}' to fit the row size cap"); + entry.ids.remove(&largest); + size = size.saturating_sub(len); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid, OPT_OUT_MEMORY_SECONDS}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn partner(source: &str) -> EcPartner { + serde_json::from_value(serde_json::json!({ "name": source, "source_domain": source, "bidstream_enabled": true })) + .expect("should build partner") + } + + fn registry(owned: &[&str]) -> PartnerRegistry { + let mut registry = PartnerRegistry::from_config(&[ + partner("browser.example.com"), + partner("ids.example.com"), + ]) + .expect("should build registry"); + for source in owned { + assert!( + registry.mark_provider_owned(source, None), + "should mark {source}" + ); + } + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn present(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("plan"); + graph.create(EC_ID, entry).expect("should seed row"); + graph.load_snapshot(EC_ID) + } + + fn found(source: &str, ids: &[&str]) -> LookupResults { + let uids = ids + .iter() + .map(|id| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), uids)], + )], + cleared: Vec::new(), + }], + ) + } + + fn input<'a>( + snapshot: &'a EcKvSnapshot, + new_entry: Option<&'a KvEntry>, + ingest: &'a [PartnerIdUpdate], + lookups: &'a LookupResults, + registry: &'a PartnerRegistry, + ) -> WritePlanInput<'a> { + WritePlanInput { + ec_id: EC_ID, + snapshot, + new_entry, + browser_ingest: ingest, + lookups, + registry, + now: UnixSeconds::new(2000), + } + } + + #[test] + fn new_ec_creates_row_with_ingest_and_lookups() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = [ + PartnerIdUpdate::new("browser.example.com", "b-1"), + PartnerIdUpdate::new("ids.example.com", "client"), + ]; + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert_eq!( + entry.ids["browser.example.com"].uid, "b-1", + "should ingest browser uid" + ); + assert_eq!( + entry.ids["ids.example.com"].uid, "p-1", + "should store provider uid, not client uid" + ); + } + + #[test] + fn lookups_for_another_ec_id_are_discarded() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let lookups_for_old = LookupResults::new( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.OLD123", + found("ids.example.com", &["p-1"]).iter().cloned().collect(), + ); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups_for_old, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not attribute the old EC ID's lookups to the new row" + ); + } + + #[test] + fn unchanged_returning_row_plans_no_write() { + let registry = registry(&[]); + let mut entry = live_entry(); + entry + .ids + .insert("browser.example.com".to_owned(), KvPartnerId::single("b-1")); + let snapshot = present(&entry); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-2")]; + let lookups = LookupResults::default(); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn lookup_replaces_provider_source_with_cas_update() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1", "p-2"]); + + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert_eq!( + entry.ids["ids.example.com"].eids()[0].uids().len(), + 2, + "should keep both provider uids" + ); + } + + #[test] + fn tombstoned_row_plans_nothing() { + let registry = registry(&[]); + let snapshot = present(&KvEntry::tombstone(1000)); + let lookups = found("ids.example.com", &["p-1"]); + assert!( + plan_write(&input(&snapshot, None, &[], &lookups, ®istry)).is_none(), + "should never write over a tombstone" + ); + } + + #[test] + fn snapshot_without_generation_plans_nothing() { + let registry = registry(&[]); + let snapshot = EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(live_entry()), + generation: None, + }; + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let lookups = LookupResults::default(); + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite blindly" + ); + } + + #[test] + fn oversized_row_without_provider_uids_drops_records_until_it_fits() { + let registry = registry(&[]); + let mut entry = live_entry(); + for i in 0..40 { + entry.ids.insert( + format!("source{i:02}.example.com"), + KvPartnerId::single(format!("{i:03}{}", "x".repeat(500))), + ); + } + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let lookups = LookupResults::default(); + + let change = plan_write(&input(&snapshot, Some(&entry), &[], &lookups, ®istry)) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + assert!(!entry.ids.is_empty(), "should keep the records that fit"); + } + + #[test] + fn oversized_row_drops_provider_uids_until_it_fits() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let big: Vec = (0..40) + .map(|i| format!("{i:03}{}", "x".repeat(500))) + .collect(); + let ids: Vec<&str> = big.iter().map(String::as_str).collect(); + let lookups = found("ids.example.com", &ids); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + let kept = entry.ids["ids.example.com"].eids()[0].uids(); + assert!(kept.len() < 40, "should drop some uids"); + assert_eq!(kept[0].id().as_str(), ids[0], "should drop from the end"); + } + + fn ingest_owned() -> [PartnerIdUpdate; 1] { + [PartnerIdUpdate::new("ids.example.com", "browser-id")] + } + + fn failed_lookup(status: LookupStatus) -> LookupResults { + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status, + entries: Vec::new(), + cleared: Vec::new(), + }], + ) + } + + #[test] + fn opt_out_removes_the_provider_sources_and_blocks_browser_fallback() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + }], + ); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should remove the opted-out source without a browser fallback" + ); + } + + #[test] + fn opt_out_is_remembered_without_extending_it() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let provider = ProviderId::new("example-provider"); + let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + + // Act: the first opt-out is remembered. + let first = plan_write(&input( + &present(&provider_entry(&["p-1"])), + None, + &ingest, + &lookups, + ®istry, + )) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = first else { + panic!("should update existing row") + }; + let until = entry.opt_outs[&provider]; + assert_eq!( + until, + 2000 + OPT_OUT_MEMORY_SECONDS, + "should remember the opt-out for the memory period" + ); + + // Act: a skip because of the remembered opt-out changes nothing. + let again = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert + assert!(again.is_none(), "should not extend a remembered opt-out"); + } + + #[test] + fn lapsed_opt_out_alone_plans_no_write() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!(change.is_none(), "should not write only to prune"); + } + + #[test] + fn lapsed_opt_out_is_pruned_with_a_real_change() { + let registry = registry(&["ids.example.com"]); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!(entry.opt_outs.is_empty(), "should forget a lapsed opt-out"); + } + + #[test] + fn not_due_skip_with_an_expired_uid_plans_no_write() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!( + change.is_none(), + "should not write only to prune an expired UID" + ); + } + + #[test] + fn browser_ingest_fills_a_slot_freed_by_an_expired_record() { + // The source is not provider-owned (its provider was disabled), so the + // browser may fill it once the expired provider record is pruned. + let registry = registry(&[]); + let ingest = [PartnerIdUpdate::new("ids.example.com", "browser-1")]; + let lookups = LookupResults::default(); + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + let Some(RowChange::Update { entry, .. }) = change else { + panic!("should plan an update that fills the freed slot"); + }; + assert_eq!( + entry + .ids + .get("ids.example.com") + .map(|record| record.uid.as_str()), + Some("browser-1"), + "should store the browser UID in place of the expired record" + ); + } + + fn provider_entry(ids: &[&str]) -> KvEntry { + let mut entry = live_entry(); + let lookups = found("ids.example.com", ids); + let (_, eids) = lookups.iter().next().expect("should hold a lookup").entries[0].clone(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids(eids, UnixSeconds::new(1000)).expect("should build record"), + ); + entry + } + + #[test] + fn browser_ingest_skips_a_provider_owned_source() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + + for status in [ + LookupStatus::NoMatch, + LookupStatus::Timeout, + LookupStatus::Skipped(SkipReason::NotDue), + ] { + let lookups = failed_lookup(status); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not store a browser UID for a provider-owned source" + ); + } + } + + #[test] + fn opted_out_source_is_not_refilled_from_the_browser_on_the_next_navigation() { + // Arrange: the row after an opt-out holds nothing for the source. + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not restore an opted-out provider UID from the browser cookie" + ); + } + + #[test] + fn new_row_skips_browser_uid_for_a_provider_owned_source() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not store a browser UID when creating the row" + ); + } + + #[test] + fn browser_fallback_is_not_stored_when_provider_found_the_source() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should store the provider uid"); + assert!(record.is_provider_written(), "should keep provider eids"); + } + + #[test] + fn browser_fallback_never_overwrites_a_live_provider_uid() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should leave the live provider uid untouched" + ); + } + + #[test] + fn browser_fallback_does_not_overwrite_an_existing_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("older")); + let snapshot = present(&entry); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn later_provider_result_replaces_browser_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("browser-id"), + ); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + None, + &ingest_owned(), + &lookups, + ®istry, + )) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should replace the fallback"); + assert!(record.is_provider_written(), "should store provider eids"); + } +} diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c13260704f..33018ca519 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -7015,30 +7015,30 @@ external_bundle_sri = "sha384-AAAA" let provider = PrebidAuctionProvider::new(base_config()); let mut auction_request = create_test_auction_request(); auction_request.user.eids = Some(vec![ - crate::openrtb::Eid { - source: "liveramp.com".to_owned(), - uids: vec![crate::openrtb::Uid { + crate::openrtb::Eid::new( + "liveramp.com".to_owned(), + vec![crate::openrtb::Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, - crate::openrtb::Eid { - source: "id5-sync.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "id5-sync.com".to_owned(), + vec![crate::openrtb::Uid { id: "ID5_abc".to_owned(), atype: Some(1), ext: None, }], - }, - crate::openrtb::Eid { - source: "google.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "google.com".to_owned(), + vec![crate::openrtb::Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }, + ), ]); let settings = make_settings(); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 15a6486fd8..0460984238 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -10,11 +10,8 @@ use matchit::Router; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; -use crate::ec::EcContext; -use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; -use crate::http_util::is_navigation_request; use crate::platform::RuntimeServices; use crate::settings::Settings; use crate::streaming_processor::StreamProcessor; @@ -786,15 +783,11 @@ impl IntegrationMetadata { /// Inputs to [`IntegrationRegistry::handle_proxy`]. /// -/// Bundled into a struct so the dispatch surface stays within the project's -/// 7-argument cap; `ec_context` and `req` participate in the borrow so the -/// whole thing shares one lifetime. +/// Bundled into a struct so the dispatch surface shares one lifetime. pub struct ProxyDispatchInput<'a> { pub method: &'a Method, pub path: &'a str, pub settings: &'a Settings, - pub kv: Option<&'a KvIdentityGraph>, - pub ec_context: &'a mut EcContext, pub services: &'a RuntimeServices, pub req: Request, } @@ -1018,7 +1011,8 @@ impl IntegrationRegistry { /// Dispatch a proxy request when an integration handles the path. /// /// This method removes any caller-supplied `x-ts-ec` before proxying. - /// Response-side cookie mutation is centralized in EC finalize. + /// Response-side cookie mutation is centralized in EC finalize. It never + /// mints an EC: only a publisher navigation may create the EC row. #[must_use] pub async fn handle_proxy( &self, @@ -1028,24 +1022,12 @@ impl IntegrationRegistry { method, path, settings, - kv, - ec_context, services, mut req, } = input; if let Some((proxy, _)) = self.find_route(method, path) { - // Organic proxy handler: generate if needed (best effort). - // Only generate for document navigations — subresource requests - // may lack consent signals such as the Sec-GPC header. - if is_navigation_request(&req) { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { - log::warn!("EC generation failed for integration proxy: {err:?}"); - } - } else { - log::debug!( - "EC generation skipped for integration proxy: non-document request (path={path})", - ); - } + // Integration proxies never mint an EC: only the publisher + // navigation may create the row (one write per page load). // Remove any caller-supplied EC header rather than forwarding it. req.headers_mut().remove(HEADER_X_TS_EC.clone()); @@ -1835,14 +1817,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { method: &http::Method::GET, path: "/integrations/test/echo", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &noop_services(), req, })) @@ -2189,8 +2167,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = noop_services(); // Call handle_proxy (uses futures executor in test environment) @@ -2198,8 +2174,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); @@ -2239,8 +2213,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("evil;injected"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2248,8 +2220,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2288,8 +2258,6 @@ mod tests { )) .expect("should build Cookie header"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2297,8 +2265,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2334,8 +2300,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2343,8 +2307,6 @@ mod tests { method: &Method::POST, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 67cd7f6950..645031a7b0 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -70,10 +70,37 @@ pub struct ConsentedProvidersSettings { pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// `OpenRTB` `mm` (match method). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// EID-level extension data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, /// One or more user IDs from this provider. pub uids: Vec, } +impl Eid { + /// Builds an entry with no EID-level fields. + #[must_use] + pub fn new(source: impl Into, uids: Vec) -> Self { + Self { + source: source.into(), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } +} + /// A single user identifier within an [`Eid`] entry. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -405,16 +432,46 @@ mod tests { ); } + #[test] + fn eid_serializes_eid_level_fields_only_when_present() { + let plain = Eid::new("ids.example.com", vec![]); + assert_eq!( + serde_json::to_value(&plain).expect("should serialize"), + serde_json::json!({ "source": "ids.example.com", "uids": [] }), + "should omit absent EID-level fields" + ); + let full = Eid { + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(3), + ext: Some(serde_json::json!({ "k": 1 })), + ..Eid::new("ids.example.com", vec![]) + }; + let value = serde_json::to_value(&full).expect("should serialize"); + assert_eq!( + value["inserter"], + serde_json::json!("inserter.example.com"), + "should write inserter" + ); + assert_eq!(value["mm"], serde_json::json!(3), "should write mm"); + let back: Eid = serde_json::from_value(value).expect("should accept EID-level fields"); + assert_eq!( + back.matcher.as_deref(), + Some("matcher.example.com"), + "should read matcher" + ); + } + #[test] fn eid_serializes_correctly() { - let eid = Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5-abc123".to_owned(), atype: Some(1), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); assert_eq!(serialized["source"], "id5-sync.com", "source should match"); @@ -431,14 +488,14 @@ mod tests { #[test] fn eid_serializes_vendor_specific_atype() { - let eid = Eid { - source: "google.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "google.com".to_owned(), + vec![Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a670d31416..c02070a0ce 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3448,16 +3448,32 @@ fn request_head_snapshot(req: &Request) -> Request { struct EcSnapshotPreloadInput { is_navigation: bool, is_get: bool, + has_navigation_permit: bool, has_ec_id: bool, has_kv: bool, marker_valid: bool, auction_needs_row: bool, eid_cookie_may_need_persistence: bool, privacy_needs_row: bool, + snapshot_already_read: bool, } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { + // An earlier step of this request already settled the row snapshot + // (identity-provider dispatch, or EC generation's authoritative miss), so + // a read adds nothing. + if input.snapshot_already_read { + return false; + } let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; + // Without a permit (no Fetch Metadata, or a prefetch) the navigation + // cannot write, so browser EID persistence no longer needs the row; the + // auction, a withdrawal and pull sync (unless the marker proves it + // complete) still do. + if !input.has_navigation_permit { + return eligible + && (input.auction_needs_row || input.privacy_needs_row || !input.marker_valid); + } let marker_can_skip = input.marker_valid && !input.auction_needs_row && !input.eid_cookie_may_need_persistence @@ -4956,9 +4972,15 @@ pub async fn handle_publisher_request( // explicit withdrawal `ec_allowed` is false, so auction dispatch forwards no // EC while the origin-overlapped snapshot read still happens for the active // ID, keeping the withdrawal CAS off the post-origin latency path. + // + // An EC minted on this request has no row until finalize creates it, and + // that create can still fail or be skipped (a redirect), so it is not sent + // to bidders until a later request proves it was issued. let active_ec_id_owned = ec_context.ec_value().map(str::to_owned); let active_ec_id = active_ec_id_owned.as_deref(); - let ec_id_owned = active_ec_id_owned.clone().filter(|_| ec_allowed); + let ec_id_owned = active_ec_id_owned + .clone() + .filter(|_| ec_allowed && !ec_context.ec_generated()); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; if let Some(registry) = auction.registry { @@ -5298,6 +5320,7 @@ pub async fn handle_publisher_request( let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { is_navigation, is_get, + has_navigation_permit: ec_context.has_navigation_permit(), has_ec_id: active_ec_id.is_some(), has_kv: kv.is_some(), marker_valid: ec_context.pull_sync_marker().is_valid(), @@ -5307,6 +5330,12 @@ pub async fn handle_publisher_request( .is_some_and(|registry| !registry.is_empty()), eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), + // A freshly generated EC already holds an authoritative `Missing` + // snapshot: its row is only created by this navigation's write. + snapshot_already_read: !matches!( + ec_context.kv_snapshot(), + crate::ec::EcKvSnapshot::NotRead + ), }); let mut pending_origin = None; // A shared-template hit skips the origin entirely, and an in-flight request @@ -5349,18 +5378,9 @@ pub async fn handle_publisher_request( ); } if should_preload_ec && let (Some(graph), Some(active_ec_id)) = (kv, active_ec_id) { - let refreshed = graph.load_snapshot(active_ec_id); - // Never downgrade an in-request Add-confirmed Present snapshot: a - // freshly created row can read back Missing/Failed on an - // eventually-consistent store, and rotating or suppressing that - // just-generated identity would fragment it. Adopt the refresh only - // when it keeps or upgrades to a Present row (the intended - // generation-refresh) — otherwise retain the confirmed entry. - let keep_present = ec_context.kv_snapshot().entry_for(active_ec_id).is_some() - && refreshed.entry_for(active_ec_id).is_none(); - if !keep_present { - ec_context.set_kv_snapshot(refreshed); - } + // A new EC's row is created only in finalize, so no snapshot read + // before this point can be an in-request confirmed row to protect. + ec_context.set_kv_snapshot(graph.load_snapshot(active_ec_id)); } // Dispatch SSP bid requests while req still has the original client headers @@ -8383,12 +8403,14 @@ mod tests { let baseline = EcSnapshotPreloadInput { is_navigation: true, is_get: true, + has_navigation_permit: true, has_ec_id: true, has_kv: true, marker_valid: false, auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(should_preload_ec_snapshot(&baseline)); assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8407,6 +8429,55 @@ mod tests { has_kv: false, ..baseline })); + assert!( + !should_preload_ec_snapshot(&EcSnapshotPreloadInput { + snapshot_already_read: true, + ..baseline + }), + "should skip the preload when the snapshot was already read" + ); + } + + #[test] + fn ec_snapshot_preload_without_a_permit_ignores_browser_eid_persistence() { + let no_permit = EcSnapshotPreloadInput { + is_navigation: true, + is_get: true, + has_navigation_permit: false, + has_ec_id: true, + has_kv: true, + marker_valid: true, + auction_needs_row: false, + eid_cookie_may_need_persistence: true, + privacy_needs_row: false, + snapshot_already_read: false, + }; + + assert!( + !should_preload_ec_snapshot(&no_permit), + "should not read a row the navigation can never write" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + auction_needs_row: true, + ..no_permit + }), + "should read the row for the auction's stored EIDs" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + privacy_needs_row: true, + ..no_permit + }), + "should read the row for a withdrawal" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + marker_valid: false, + ..no_permit + }), + "should read the row for pull sync while the marker is not valid" + ); } #[test] @@ -8414,12 +8485,14 @@ mod tests { let marker_only = EcSnapshotPreloadInput { is_navigation: true, is_get: true, + has_navigation_permit: true, has_ec_id: true, has_kv: true, marker_valid: true, auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(!should_preload_ec_snapshot(&marker_only)); assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8684,6 +8757,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert!( ec_context.ec_allowed() && ec_context.ec_value().is_some(), "test precondition: an active, consent-allowed EC must exist" @@ -8844,8 +8918,8 @@ mod tests { create_test_settings() }; settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); let http = Arc::new(StubHttpClient::new()); http.set_concurrent_fanout(true); http.push_response(200, b"ok".to_vec()); @@ -8894,6 +8968,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), ); @@ -13655,6 +13730,8 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); let response = handle_publisher_request( &settings, &services, @@ -16285,6 +16362,74 @@ mod tests { } } + #[tokio::test] + async fn publisher_auction_withholds_an_ec_minted_on_this_request() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings).auction.providers = + crate::auction_config_types::AuctionConfig::legacy_provider_map(&[ + SCHEDULING_PROVIDER, + ]); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let graph = KvIdentityGraph::in_memory("minted-ec-store"); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let partners = PartnerRegistry::empty(); + queue_shareable_html(&stub); + let minted = format!("{}.Mint01", "c".repeat(64)); + let mut ec_context = EcContext::new_for_test(None, scheduling_consent()); + ec_context.replace_with_generated( + minted.clone(), + crate::ec::EcKvSnapshot::Missing { + ec_id: minted.clone(), + }, + ); + let captured = Arc::new(Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { + captured: Arc::clone(&captured), + http: Arc::clone(&stub), + lookups: Arc::new(AtomicUsize::new(0)), + })); + let orchestrator = Arc::new(orchestrator); + + let response = handle_publisher_request( + &settings, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: Some(&partners), + }, + cookie_policy_request(&[b"ab_bucket=A"]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should serve a first visit"); + let response = finalize_test_publisher_response( + response, + &settings, + &services, + ®istry, + orchestrator, + Finalizer::Buffered, + ) + .await; + let body = body_of(response).await; + + assert!(!body.is_empty(), "should render the page"); + let captured = captured.lock().expect("should lock captured auction"); + let auction = captured.as_ref().expect("should dispatch the auction"); + assert_eq!( + auction.request.user.id, None, + "should not send an EC whose row does not exist yet" + ); + } + #[tokio::test] async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { @@ -16350,6 +16495,9 @@ mod tests { scheduling_consent() }; let mut ec_context = EcContext::new_for_test(Some(identity.clone()), consent); + // A publisher page load: only a navigation may write the tombstone. + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert_eq!( ec_context.ec_allowed(), !withdrawn, @@ -20045,11 +20193,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } fn active_ec_context() -> EcContext { - EcContext::new_for_test_with_ip( + let mut ec_context = EcContext::new_for_test_with_ip( Some(format!("{}.CkId01", "b".repeat(64))), scheduling_consent(), Some("203.0.113.7".to_owned()), - ) + ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); + ec_context } #[tokio::test] diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 754acfc262..f623aa1852 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::num::{NonZeroU16, NonZeroU32}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -579,6 +580,90 @@ impl EcPartner { } } +/// One `[[ec.identity_providers]]` entry. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IdentityProviderConfig { + /// Unique provider ID; recorded as the owner of the UIDs it writes. + pub id: String, + /// Implementation kind, matched by the provider crate's `build`. + kind: String, + /// Whether lookups run. A disabled provider owns no sources. + #[serde( + default = "IdentityProviderConfig::default_enabled", + deserialize_with = "from_value_or_str" + )] + pub enabled: bool, + /// Lookup endpoint; must be `https` with a host in `allowed_domains`. + pub endpoint: String, + /// Exact hosts the coordinator may call for this provider. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub allowed_domains: Vec, + /// Source domains this provider owns; each needs an `[[ec.partners]]` entry. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub sources: Vec, + /// TCF vendor ID that must have vendor consent, when set. + #[serde(default)] + pub tcf_vendor_id: Option, + /// When the provider's lookup runs on a navigation. + #[serde(default)] + pub collect: CollectTiming, + /// Most lookups per EC hash per hour, when set; unset means unlimited. + /// ECs behind one client IP share the budget. + /// + /// The Fastly limiter enforces this as a per-minute budget of at least + /// one, so values below 60 behave like 60 per hour. + #[serde(default)] + pub rate_limit_per_hour: Option, + /// Secret Store key references, resolved at startup. + #[serde(default)] + pub secrets: ProviderSecrets, + /// Provider-specific settings, parsed by the provider kind. + /// + /// Held as a JSON object because settings travel through the JSON app + /// config pipeline. + #[serde(default)] + pub settings: serde_json::Map, +} + +impl IdentityProviderConfig { + const fn default_enabled() -> bool { + true + } + + /// The configured implementation kind. + #[must_use] + pub fn kind(&self) -> &str { + &self.kind + } +} + +/// When an identity provider's lookup runs on a navigation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectTiming { + /// Before the response, within `identity_budget_ms`; results reach the + /// navigation's own write and response. + #[default] + Page, + /// After the response has been sent, within + /// `identity_after_response_budget_ms`; results reach the row in one + /// follow-up write and are served from the next page load. + AfterResponse, +} + +/// Secret values for one identity provider. +/// +/// `EdgeZero` secret paths address fixed field names, so V1 supports one +/// optional bearer token per provider. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderSecrets { + /// Secret Store key whose value is the provider's token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token: Option>, +} + /// Edge Cookie (EC) configuration. /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, @@ -616,6 +701,47 @@ pub struct Ec { #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub partners: Vec, + + /// Shared time budget, in milliseconds, for all provider lookups on one + /// navigation. + /// + /// `Ec` denies unknown fields, so this and the other identity fields are + /// skipped from serialization at their defaults: a config blob without + /// providers then still loads in a runtime built before them. + #[serde( + default = "Ec::default_identity_budget_ms", + skip_serializing_if = "Ec::is_default_identity_budget_ms", + deserialize_with = "from_value_or_str" + )] + #[validate(range( + min = 1, + max = 1000, + message = "must be between 1 and 1000 milliseconds" + ))] + pub identity_budget_ms: u64, + + /// Time budget, in milliseconds, for lookups of providers with + /// `collect = "after_response"`, measured from their dispatch after the + /// response has been sent. + #[serde( + default = "Ec::default_identity_after_response_budget_ms", + skip_serializing_if = "Ec::is_default_identity_after_response_budget_ms", + deserialize_with = "from_value_or_str" + )] + #[validate(range( + min = 1, + max = 5000, + message = "must be between 1 and 5000 milliseconds" + ))] + pub identity_after_response_budget_ms: u64, + + /// Server-side identity providers. + #[serde( + default, + skip_serializing_if = "Vec::is_empty", + deserialize_with = "vec_from_seq_or_map" + )] + pub identity_providers: Vec, } impl Ec { @@ -646,6 +772,28 @@ impl Ec { 3600 } + /// Default shared identity lookup budget (200 ms). + #[must_use] + pub const fn default_identity_budget_ms() -> u64 { + 200 + } + + /// Default after-response identity lookup budget (2 s). + #[must_use] + pub const fn default_identity_after_response_budget_ms() -> u64 { + 2000 + } + + #[allow(clippy::trivially_copy_pass_by_ref)] + fn is_default_identity_budget_ms(value: &u64) -> bool { + *value == Self::default_identity_budget_ms() + } + + #[allow(clippy::trivially_copy_pass_by_ref)] + fn is_default_identity_after_response_budget_ms(value: &u64) -> bool { + *value == Self::default_identity_after_response_budget_ms() + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -3510,6 +3658,19 @@ impl Settings { )); } } + for provider in &self.ec.identity_providers { + if provider + .secrets + .token + .as_ref() + .is_some_and(|token| EcPartner::is_placeholder_api_token(token.expose())) + { + insecure_fields.push(format!( + "ec.identity_providers[{}].secrets.token", + provider.id + )); + } + } for handler in &self.handlers { if Handler::is_placeholder_password(handler.password.expose()) { insecure_fields.push(format!("handlers[{}].password", handler.path)); @@ -4292,6 +4453,53 @@ mod tests { ); } + /// `Ec` denies unknown fields, so a binary built before identity providers + /// existed must still accept a blob that configures none. + #[test] + fn default_identity_settings_are_omitted_for_rollback() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + + let serialized = serde_json::to_value(&settings.ec).expect("should serialize ec settings"); + + for field in [ + "identity_budget_ms", + "identity_after_response_budget_ms", + "identity_providers", + ] { + assert!( + serialized.get(field).is_none(), + "should omit default {field} for older binaries: {serialized}" + ); + } + } + + #[test] + fn identity_budgets_are_range_checked() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + for (page, after_response, invalid) in [ + (1, 1, false), + (1000, 5000, false), + (0, 2000, true), + (1001, 2000, true), + (200, 0, true), + (200, 5001, true), + ] { + let ec = Ec { + identity_budget_ms: page, + identity_after_response_budget_ms: after_response, + ..settings.ec.clone() + }; + + assert_eq!( + ec.validate().is_err(), + invalid, + "should enforce identity budget bounds for {page} and {after_response}" + ); + } + } + /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. /// That only holds while the flag is skipped during serialization. @@ -6591,6 +6799,31 @@ source_domain = "partner.example.com" .expect("should accept a realistic partner pull token"); } + #[test] + fn reject_placeholder_secrets_includes_identity_provider_tokens() { + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); + settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); + let mut provider: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse provider config"); + provider.secrets.token = Some(Redacted::new( + "partner-api-token-32-bytes-minimum".to_owned(), + )); + settings.ec.identity_providers = vec![provider]; + + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder provider token"); + + assert!( + format!("{err:?}").contains("ec.identity_providers[example-provider].secrets.token"), + "should mention the provider token field" + ); + } + #[test] fn reject_placeholder_secrets_reports_both_partner_tokens() { let mut settings = diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37d54614c0..187b22a276 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -7,6 +7,35 @@ pub mod tests { use crate::consent::types::{ConsentContext, ConsentSource, TcfConsent}; use crate::settings::Settings; + /// GDPR consent with TCF Purposes 1 and 4 and the given vendor consents. + #[must_use] + pub fn consent_granting_vendors(vendor_consents: Vec) -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents, + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + #[must_use] pub fn crate_test_settings_str() -> String { r#" diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml new file mode 100644 index 0000000000..1ff50dedf8 --- /dev/null +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "trusted-server-identity-providers" +description = "Identity provider implementations for Trusted Server." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +error-stack = { workspace = true } +trusted-server-core = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs new file mode 100644 index 0000000000..23197150ff --- /dev/null +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -0,0 +1,251 @@ +//! Identity provider implementations for Trusted Server. +//! +//! Each provider kind is a module implementing +//! [`trusted_server_core::ec::provider::ProviderKind`]. [`build`] maps a +//! configured `kind` to its implementation; [`build_coordinator`] and +//! [`build_after_response_coordinator`] build the coordinators for enabled +//! providers collected before and after the response. +//! +//! Dependencies point one way: this crate depends on core, and core never +//! depends on it. + +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; +use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; +use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::settings::{ + CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, +}; + +/// Whether `kind` has a registered implementation. +/// +/// No kinds are registered yet. +#[must_use] +pub fn is_registered_kind(_kind: &str) -> bool { + false +} + +/// Builds the partner registry, marking the sources claimed by enabled +/// providers as provider-owned. +/// +/// # Errors +/// +/// Returns the same errors as [`PartnerRegistry::from_ec_with_kinds`], +/// including for an enabled provider whose kind is not registered. +pub fn partner_registry(ec: &Ec) -> Result> { + PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) +} + +/// Builds the provider for a configured `kind`. +/// +/// No kinds are registered yet, so every kind is rejected. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`trusted_server_core::ec::provider::build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + _secrets: &ProviderSecrets, +) -> Result, Report> { + let kind = config.kind(); + Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))) +} + +/// Builds the coordinator for enabled providers collected before the +/// response (`collect = "page"`, the default). +/// +/// Returns `None` when no such provider is enabled. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or the +/// enabled providers, taken together, fail the coordinator's configuration +/// checks. +pub fn build_coordinator( + settings: &Settings, +) -> Result, Report> { + let partners = partner_registry(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + build_coordinator_with(settings, Arc::new(partners)) +} + +/// Builds the page-phase coordinator against an already built partner +/// registry. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. +pub fn build_coordinator_with( + settings: &Settings, + partners: Arc, +) -> Result, Report> { + build_phase_coordinator( + settings, + partners, + CollectTiming::Page, + Duration::from_millis(settings.ec.identity_budget_ms), + ) +} + +/// Builds the coordinator for enabled providers collected after the response +/// has been sent (`collect = "after_response"`), against an already built +/// partner registry. +/// +/// Its budget is `identity_after_response_budget_ms`. Returns `None` when no +/// such provider is enabled. +/// +/// # Errors +/// +/// Returns the same errors as [`build_coordinator_with`]. +pub fn build_after_response_coordinator( + settings: &Settings, + partners: Arc, +) -> Result, Report> { + build_phase_coordinator( + settings, + partners, + CollectTiming::AfterResponse, + Duration::from_millis(settings.ec.identity_after_response_budget_ms), + ) +} + +fn build_phase_coordinator( + settings: &Settings, + partners: Arc, + timing: CollectTiming, + budget: Duration, +) -> Result, Report> { + // Source ownership across both phases is checked when settings load + // (`config::validate_identity_providers`), so only this phase's providers + // are built and checked here. + let registered = settings + .ec + .identity_providers + .iter() + .filter(|config| config.enabled && config.collect == timing) + .map(|config| { + build(config, &config.secrets) + .map(|provider| RegisteredProvider::new(config.clone(), provider)) + }) + .collect::, Report>>()?; + if registered.is_empty() { + return Ok(None); + } + ProviderCoordinator::new(registered, partners, budget).map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "example-admin-password" + + [publisher] + domain = "www.example.com" + cookie_domain = ".example.com" + origin_url = "https://origin.example.com" + proxy_secret = "example-proxy-secret" + + [ec] + passphrase = "example-passphrase-32-bytes-minimum" + "#, + ) + .expect("should parse test settings") + } + + #[test] + fn unknown_kind_is_rejected() { + let config: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nallowed_domains = [\"ids.example.com\"]", + ) + .expect("should parse"); + + let error = build(&config, &ProviderSecrets::default()) + .err() + .expect("should reject unknown kind"); + + assert!( + matches!(error.current_context(), ProviderConfigError::UnknownKind), + "should report unknown kind" + ); + } + + #[test] + fn no_enabled_provider_builds_no_coordinator() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nenabled = false\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse"), + ]; + + let coordinator = build_coordinator(&settings).expect("should skip disabled providers"); + + assert!(coordinator.is_none(), "should build no coordinator"); + } + + #[test] + fn enabled_unknown_kind_fails_the_build() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse"), + ]; + + let result = build_coordinator(&settings); + + assert!( + result.is_err(), + "should fail instead of skipping an enabled provider of an unknown kind" + ); + } + + #[test] + fn rejects_an_unknown_collect_timing() { + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\ncollect = \"later\"", + ) + .expect_err("should reject an unknown `collect` value"); + } + + #[test] + fn parses_an_optional_hourly_rate_limit() { + let limited: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 2", + ) + .expect("should parse a rate limit"); + let unset: IdentityProviderConfig = + toml::from_str("id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"") + .expect("should parse without a rate limit"); + + assert_eq!( + limited.rate_limit_per_hour.map(std::num::NonZeroU32::get), + Some(2), + "should keep the hourly limit" + ); + assert!( + unset.rate_limit_per_hour.is_none(), + "should leave the limit unset by default" + ); + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 0", + ) + .expect_err("should reject a zero limit"); + } +} diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index 0a1f149c3b..1846fb9602 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -113,6 +113,7 @@ impl EcTestClient { .client .get(format!("{}{path}", self.base_url)) .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") .header("accept", "text/html"); for (key, value) in headers { builder = builder.header(*key, *value); diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 99059adba0..1ddd0ab4ae 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -363,8 +363,10 @@ Server-to-server batch sync endpoint for writing EC ID to partner UID mappings. **Contract:** The JSON body is limited to 2 MiB and 1,000 mappings. The per-partner, per-minute limit is `ec.partners[].batch_rate_limit`. Responses are `200` when every mapping is accepted or `207` when at least one is rejected; -item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, or -`kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; +item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, +`provider_owned` (the source belongs to a server-side identity provider), +`row_full` (the EC's row has no room for the UID; retrying gives the same +result), or `kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; rate exhaustion returns `429`; an oversized body returns `413`; too many mappings returns `400`. The endpoint emits JSON but defines no dedicated cache or CORS headers. It is Fastly-only and requires the EC KV store. diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index fb02fe2e68..c4b2c5c2b4 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -172,7 +172,8 @@ Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin currently accept at most one provider in an enabled auction. Every adapter runs target-aware fan-out and backend-name checks at startup. Auction collection uses an absolute cutoff for network waiting across headers and complete response bodies. -A nonblocking snapshot retains complete results after a slow origin. Fastly +A final snapshot retains complete results after a slow origin (on Fastly it may +wait up to 1 ms per partially received body). Fastly abandons pending backend requests because its SDK cannot cancel them; other adapters cancel their owned operations at cutoff. See [Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index c377157ce5..cd40f2a94d 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -828,15 +828,15 @@ Until that callback lands, startup remains the mandatory target-aware gate. Auction requests start at dispatch and progress while the publisher origin loads. Response collection waits for headers and complete upstream bodies only until the -original absolute deadline. At expiry, one nonblocking snapshot preserves complete -responses already available; unfinished providers become timeouts. A slow origin +original absolute deadline. At expiry, one final snapshot preserves complete +responses already available; unfinished providers become timeouts. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and keeps a response whose headers arrived by the cutoff. A slow origin never restarts the budget or causes an already-buffered bid to be discarded. Fastly has no upstream arrival timestamp, so this is a bound on network waiting, not bid eligibility based on an exact server-side arrival time. A mediator starts only with remaining budget and uses the earlier of the auction cutoff and its own logical timeout. -Provider bids that are already complete at the final nonblocking snapshot remain +Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 8413e77d09..acb84abaa8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -59,6 +59,7 @@ publisher, trusted-client-IP, EC, handler, Tinybird, DataDome, and S3 fields: - `ec.passphrase` - `ec.partners[*].api_token`, when inbound identify or batch sync is used - `ec.partners[*].ts_pull_token`, when pull sync is enabled +- `ec.identity_providers[*].secrets.token`, when a provider needs an API token - `handlers[*].password` - `tinybird.auction_token_secret`, when Tinybird auction telemetry is enabled - `integrations.datadome.server_side_key_secret_name`, when protection is enabled @@ -748,14 +749,17 @@ Legacy consent-store records are not read or migrated into `ec.ec_store`. Their `passphrase` is a key name in `trusted_server_secrets`; the resolved value must be at least 32 bytes. Keep it stable to preserve EC identifier continuity. -| Field | Type | Required | Description | -| ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | -| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | -| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | -| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | -| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | -| `partners` | Array | No | Static partner registry entries | +| Field | Type | Required | Description | +| ----------------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | +| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | +| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | +| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | +| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | +| `partners` | Array | No | Static partner registry entries | +| `identity_budget_ms` | Integer | No | Shared identity lookup budget, 1–1000 ms (default 200) | +| `identity_after_response_budget_ms` | Integer | No | After-response lookup budget, 1–5000 ms (default 2000) | +| `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. @@ -788,6 +792,102 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` +### Identity providers + +`[[ec.identity_providers]]` entries look up a visitor's IDs from a provider's +server on publisher page navigations. Lookups start before the origin fetch, +share the `identity_budget_ms` budget, and are collected before the response is +finalized. Results go into the navigation's single EC row write. A lookup that +times out or fails leaves the row unchanged. + +Lookups run only for an EC issued on an earlier request (an EC minted on this +navigation is looked up on the next one), only when consent allows EID +persistence, and, when `tcf_vendor_id` is set and TCF applies to the request, +only with TCF vendor consent for that vendor. The same vendor consent gates +egress: while it is withheld, the provider's stored UIDs are left out of +auction `user.eids`, `x-ts-eids` and `/_ts/api/v1/identify`. Without +`tcf_vendor_id`, a provider is gated by the EID-persistence purposes alone, so +set it for any provider that has a TCF vendor ID. + +Identity providers run only on Fastly. The Axum, Cloudflare and Spin adapters +refuse to start when an enabled provider is configured. On Fastly, an enabled +provider whose `kind` this build does not include, or whose `settings` its kind +rejects, also fails startup. Lookups always verify the provider's TLS +certificate, whatever `proxy.certificate_check` says. + +A provider owns its `sources`: its results replace whatever the row holds for +them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs +(`ts-eids`, `sharedId`) are never stored for an owned source, so a source a +provider cleared on an opt-out is not refilled from the browser. + +When a provider reports an opt-out, its sources are cleared and the row +remembers the opt-out for 30 days. The provider is not called for that EC +again until the 30 days pass. + +| Field | Type | Required | Description | +| --------------------- | ------- | -------- | ----------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation; must be one this build includes | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | +| `allowed_domains` | Array | Yes | Hosts the provider may call | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | + +**Example**: + +```toml +[ec] +identity_budget_ms = 200 + +[[ec.partners]] +name = "Example IDs" +source_domain = "ids.example.com" + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +sources = ["ids.example.com"] + +[ec.identity_providers.secrets] +token = "example_provider_token" +``` + +`collect = "after_response"` lookups run after the response is sent, within +`identity_after_response_budget_ms`, and only when `ec_store` is configured, +the response is not a redirect, the navigation's own write succeeded (or had +nothing to change), and the EC was not minted on this navigation. Their results +reach the row in a follow-up write. + +::: warning Rate limit granularity +`rate_limit_per_hour` is counted per EC hash, which is derived from the client +IP, so ECs behind one IP (a NAT or carrier gateway) share the budget. On +Fastly it is enforced as a per-minute budget rounded up to at least one lookup +per minute, so any value below 60 behaves like 60 per hour. Values of 60 and +above are enforced as `ceil(value / 60)` per minute. +::: + +**Validation**: configuration validation fails when an enabled provider's +`endpoint` is not HTTPS or its host is not in `allowed_domains`, when it claims +no sources, when a source has no `[[ec.partners]]` entry, when two enabled +providers claim the same source, or when its `id` is empty or shared with +another enabled provider. `identity_budget_ms` must be 1–1000 and +`identity_after_response_budget_ms` 1–5000. Deploy validation also rejects an +empty `secrets.token` key name. + +::: warning Rollback +A config blob that configures identity providers, or sets either identity +budget to a non-default value, cannot be loaded by a build from before +identity providers. Roll the runtime back only with a blob that leaves them +out. +::: + ### Field Details #### `passphrase` @@ -2448,11 +2548,11 @@ push as proof that a Cloudflare or Spin multi-provider plan can start. ### Deadline behavior Auction response collection waits only until an absolute deadline measured from -dispatch, including headers and complete response bodies. One nonblocking snapshot +dispatch, including headers and complete response bodies. One final snapshot retains complete available responses after a slow origin; unfinished providers become timeouts and mediation cannot start with an exhausted budget. Backend -timeout rounding does not extend network waiting. -Provider bids that are already complete at the final nonblocking snapshot remain +timeout rounding does not extend network waiting. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and keeps a response whose headers arrived by the cutoff. +Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..a3e8176a97 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -42,7 +42,7 @@ maps to a stable base. ### Request Lifecycle -Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. +Every request passes through four phases. A new EC is minted, and its row created, only on a top-level publisher navigation: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` token in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, and a prefetch or prerender (`Sec-Purpose: prefetch;prerender`) never mints or writes. Integration proxies and `/auction` use an existing EC but never mint one, and read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. ```mermaid sequenceDiagram @@ -133,7 +133,7 @@ Partner identities flow into the KV identity graph through three channels. Each ```mermaid flowchart LR subgraph Browser-initiated - Prebid["Prebid EIDs
/auction body + ts-eids + sharedId
Response-time ingestion"] + Prebid["Prebid EIDs
ts-eids + sharedId
Navigation-time ingestion"] end subgraph Server-initiated @@ -148,13 +148,12 @@ flowchart LR ### Prebid EID Flow -Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a response. It collects matched partner UIDs from three request-local sources, applied in this order so that, when two sources carry the same partner, the later source's UID is the candidate offered to KV: +Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a top-level document navigation, the only request that writes browser-resolved IDs to KV. It collects matched partner UIDs from two request-local sources, applied in this order so that, when both carry the same partner, the later source's UID is the candidate offered to KV: 1. the `ts-eids` cookie, -2. the `eids` array in the `/auction` request body, -3. the `sharedId` cookie. +2. the `sharedId` cookie. -The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. +The `eids` array in the `/auction` request body goes to bidders on that request only and is never stored. The `ts-eids` cookie bridges Prebid user ID modules with the identity graph on the next navigation. These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. @@ -173,8 +172,8 @@ sequenceDiagram TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] TSJS->>B: document.cookie = "ts-eids=..." - Note over B,TS: Next eligible EID-sync request - B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids + Note over B,TS: Next page load + B->>TS: Top-level document navigation TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) ``` @@ -183,11 +182,11 @@ Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shap The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. -Returning-user cookie persistence runs only on publisher document navigations, `POST /auction`, and admitted `GET /_ts/page-bids` SPA navigations. Static assets, analytics, integration requests, filter short circuits, and other subresources do not decode or persist these cookies. New EC creation remains eligible regardless of route because its backing row must include the request's initial IDs. +Cookie persistence runs only on top-level document navigations (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`). `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. -Each eligible request attempts at most one conditional cookie update. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. +Each navigation makes at most one conditional page-load write, which carries browser ingest and page-phase identity-provider results together. If another writer wins, the navigation's changes are dropped without a re-read; a later navigation offers them again. Two writes sit outside that rule: a consent withdrawal's tombstone is written on any request that carries the EC cookie, with bounded retries, and `collect = "after_response"` identity providers make one follow-up write after the response is sent. Batch sync and pull sync write through their own channels. -Browser cookies and the `/auction` request body do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. +Browser cookies do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. ### EID Seeding and Prebid Bidstream Forwarding @@ -263,7 +262,7 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner } ``` -Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). +Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs from the `ts-eids` or `sharedId` cookie only fill partner UIDs that are missing from KV (the `/auction` request body goes to bidders only and is never stored); a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). ### Pull-Sync Completeness Marker diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index b7a05ac7be..1f64f99e98 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -865,12 +865,10 @@ page or auction. When available, the opaque value follows the standard path: 2. The current `/auction` request includes that entry. 3. Trusted Server merges and consent-gates it, then forwards it to Prebid Server as `user.ext.eids`. -4. After the response, the same `/auction` request ingests the body entry into - an EC/KV partner configured with `source_domain = "liveramp.com"`, when that - partner UID is not already stored. -5. The browser also persists the opaque value in the bounded `ts-eids` cookie, - which lets requests without an EID body, such as `GET /_ts/page-bids` and - page navigations, ingest it. +4. The browser also persists the opaque value in the bounded `ts-eids` cookie. + The next top-level page navigation ingests it into an EC/KV partner + configured with `source_domain = "liveramp.com"`, when that partner UID is + not already stored. `/auction` and `GET /_ts/page-bids` never write KV. Trusted Server treats the RampID envelope as an opaque string. Do not log, decode, publish, or dimension metrics by the value. Source names, counts, @@ -958,9 +956,9 @@ Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions 2. **Server-side EIDs from the EC/KV identity graph** are resolved on the edge from the current EC ID. 3. Trusted Server **merges and deduplicates** both sets before calling Prebid Server. 4. The merged result is forwarded downstream as `user.ext.eids` in the OpenRTB request. -5. After the response, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `/auction` request-body EIDs, then the `sharedId` cookie. Because the body carries the untrimmed EID set, `/auction` ingestion is not limited by the size-capped `ts-eids` cookie. +5. On the next top-level page navigation, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `sharedId` cookie. The `/auction` request-body EIDs go to bidders on that request only and are never stored, so ingestion is limited by the size-capped `ts-eids` cookie. -The `ts-eids` cookie still matters for requests without an EID body, such as `GET /_ts/page-bids` and page navigations, where it is the source for both EID fallback and identity-graph ingestion. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. +The `ts-eids` cookie also supplies the EID fallback for requests without an EID body, such as `GET /_ts/page-bids`. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. This means Prebid auctions get same-request transparency for browser-resolved IDs without giving up the durability of the server-managed EC identity graph. diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md new file mode 100644 index 0000000000..733c167518 --- /dev/null +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -0,0 +1,1625 @@ +# Server-to-Server EID Collection — Design Spec + +**Date:** 2026-10-30 +**Status:** Draft for Tech Lead review +**Source of intent:** `S2S_EID_Collection_Requirements.md` (sections referenced +as "R§n") + +## 1. Purpose + +Trusted Server (TS) collects external IDs (EIDs) only through the browser and +one fixed pull-sync protocol, and a single page load can write the EC KV row +several times. This spec defines: + +- a Rust trait that any identity provider implements so TS can collect IDs + server-to-server, with implementations isolated in a dedicated crate; +- one shared coordinator that owns eligibility, consent, transport, time + budget, validation, merging, persistence and the mirror cookie; +- a strict rule of **at most one KV write per page load**, made on the top-level + document navigation and never retried; an opt-in after-response phase + (§5.2.1) may add one guarded follow-up update with provider results only; +- a full-fidelity, ownership- and expiry-aware KV representation for + provider-written IDs, and how auctions and other consumers read it. + +Client-side collection (`ts-eids`, the `/auction` body) keeps working alongside +server-side collection, and both feed the same consent-gated KV row. + +## 2. Scope + +### 2.1 In scope + +- `IdentityProvider` trait, its input/output types and capability description. +- Dedicated provider crate boundary and registration. +- Identity coordinator: provider selection, consent gating, allowlisted + outbound HTTP, shared time budget, validation, merge, single write. +- Page-load write permit: navigation-only, single-attempt KV writes. +- Additive KV schema for multiple EIDs/UIDs with EID-level and UID-level + fields, owner and expiry. +- `__Host-ts-eids-mirror` first-party cookie. +- Consumer changes: `/auction`, `/_ts/page-bids`, `x-ts-eids`, + `/_ts/api/v1/identify`, browser ingest, batch sync on provider-owned sources. +- Re-expressing the existing pull sync as the first provider on the trait. +- Operator configuration, secrets, logs and metrics. + +### 2.2 Non-goals (firm, from R§6) + +- Logged-in identity and email contribution: no hashed-email intake, no + `ts-hem` cookie, no `tsjs.setEmail()`, no publisher sign-in changes. The + trait carries a typed `HashedEmail` slot that is always empty in V1; a source + that requires it is skipped before any call. TS never receives a raw email + from the page. +- Proxying vendor identity JavaScript through TS. +- Concrete vendor implementations (each gets its own spec, see §13.2 F1). +- Adapters other than Fastly. KV identity remains Fastly-only; Axum, + Cloudflare and Spin get this design when EC identity is ported to them. +- Cross-browser linking. +- Vendor segment products and ad-server targeting. +- Batch sync and admin write paths, except how batch sync treats a + provider-owned source (§8.5). +- Removing `ts-eids` or `x-ts-eids`. +- Storing the browser's complete EID set. Browser ingest keeps today's rules + (§8.4); full fidelity applies only to provider-written IDs. +- Unregistered sources for providers. A provider claims only sources that have + an `[[ec.partners]]` entry. + +### 2.3 Future tasks (R§5, not in this scope) + +Listed in §13.2 in their original order, stacked on `main`, each a Tech Lead +decision. + +## 3. Agreed decisions + +These were agreed before drafting and are binding on the rest of the spec. + +| # | Topic | Decision | +| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | +| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | +| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | +| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | +| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | +| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of `bidstream_enabled` sources only. Never trusted by the server over KV. Filled deterministically up to its size limit. | +| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | +| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | +| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | +| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | +| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | +| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | +| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | +| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | + +§5.2.1 (opt-in `collect = "after_response"`) and §5.6 (refresh state) extend +D2 and D3 additively; defaults are unchanged. + +### 3.1 Choices this spec makes to apply those decisions + +Reviewers should check these; each follows from a decision above. + +- **Navigation test (D1).** A request is a page-load navigation when it is a + `GET` on the publisher-proxy route with `Sec-Fetch-Dest: document` and + `Sec-Fetch-Mode: navigate`, and its `Sec-Purpose` header, if any, has no + `prefetch` token. Requests without Fetch Metadata never write. A navigation + whose final response is a redirect (3xx) does not write; the redirect target + is the page load. This is stricter than the existing `is_navigation_request` + (which falls back to `Accept: text/html`); that helper keeps its other uses, + and the write permit uses the stricter test. +- **Integration-proxy EC minting stops (D1).** Integration-proxy requests can no + longer create a KV row, so they no longer mint a new EC. Today a new EC is + only issued when its row exists, so this keeps that invariant. +- **Withdrawal is the one exception to the one-write rule (D1, D9).** Any + finalized request that carries a consent withdrawal and the EC cookie writes + the tombstone, navigation or not, keeping today's behaviour. It tombstones the + cookie's EC ID and the active EC ID (when they differ), using a bounded CAS + retry and, when a store listing proves the key exists, an unconditional + tombstone. It serves no IDs and expires the mirror cookie, and it expires the + EC cookie only when every tombstone succeeded, so a failed write is retried on + the next request instead of orphaning the row. Deferring the tombstone to the + next navigation would leave a withdrawn visitor's row live for as long as they + only make non-navigation requests, which privacy does not allow. +- **One owner per source (D8).** Config validation rejects two enabled providers + claiming the same source, so "the owning provider" of a source is always + unambiguous. +- **No retry loops on the navigation write (D4).** Existing loops on this path + (EC-ID collision on create, orphan recovery) become a single attempt. + +## 4. Terms + +| Term | Meaning | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| page load | One top-level document navigation and every request the page makes because of it: `/auction`, `/_ts/page-bids` (SPA view changes), integration proxies. | +| navigation | The request that starts a page load (§3.1). | +| row | The EC KV identity-graph entry keyed by EC ID ([`KvEntry`]). | +| source | An EID source domain registered in `[[ec.partners]]`. | +| provider | A configured `IdentityProvider` instance with an operator-chosen `ProviderId`. | +| owner | The provider that wrote a UID. Browser and batch UIDs have no owner. | +| provider-owned | A source claimed by an enabled provider. A source whose owner is disabled is not provider-owned; its owned UIDs serve until they expire or are pruned. | +| due source | A source an enabled provider claims whose owned UIDs are missing or all expired, or one of whose live owned UIDs has refresh state with `from <= now` (§5.6). | +| live UID | A stored UID with no `exp`, or with `exp` greater than now. | +| mirror | The `__Host-ts-eids-mirror` cookie. | + +## 5. End-to-end flow + +### 5.1 Inputs + +| Input | Arrives on | Persisted? | +| --------------------------------- | ----------------------------------------- | ------------------------------------------- | +| `ts-eids` cookie | every request | Navigation only, via browser ingest (§8.4) | +| `sharedId` cookie | every request | Navigation only, applied last in ingest | +| `/auction` body `user.eids` | `/auction` | Never; forwarded to bidders on that request | +| Provider lookup responses | navigation (outbound) | Navigation write | +| After-response provider responses | navigation, after the response (outbound) | Follow-up CAS update (§5.2.1) | +| Batch sync | its own endpoint | Its own write path (unchanged) | + +### 5.2 Navigation (the only writer) + +``` +browser TS (navigation) origin providers KV + | GET / (document) | | | | + |------------------>| read EC cookie, consent | | | + | | read row snapshot ---------------------------------->| + | | classify: navigation => PageLoadWrite permit | + | | coordinator.dispatch(): due sources, consent, | + | | allowlist ---------------------------> lookups | + | | fetch origin ------------------------>| (concurrent) | + | |<-------------------- origin headers --| | | + | | coordinator.collect(): wait until all done or | + | | dispatch + budget (≈200 ms); drop late responses | + | | validate + merge: browser ingest + provider results | + | | if anything changed: one write (permit consumed) ---->| + | |<---------------------------------------- ok / failed | + | | on ok: Set-Cookie EC (new rows) | +| | mirror from committed row, else snapshot row | + |<------------------| response streams | +``` + +Notes: + +- The budget is measured from dispatch. If the origin takes longer than the + budget, provider collection adds no latency. +- The added delay never exceeds the budget. Collection uses + `PlatformHttpClient::select_until` (added by PR #1433), which bounds both + header waits and body reads by an absolute deadline on every adapter, so a + slow or byte-trickling provider cannot hold the response. +- Response headers, including cookies, depend on the write result, so the write + completes before headers are sent. This is today's behaviour for new ECs. +- A navigation with nothing due and nothing new from the browser makes no + provider calls and no write. + +### 5.2.1 After-response collection (opt-in) + +A provider can be slower than any acceptable page budget (for example, one +that takes about a second). Setting `collect = "after_response"` on that +provider moves its lookup out of the page path: + +``` +browser TS (navigation) providers KV + | | page phase as in §5.2, without after-response providers + |<------------------| response sent (send_to_client) | | + | | dispatch after-response providers --------->| | + | | collect until after-response budget (≈2 s) | | + | | read row + generation --------------------------->| + | | one CAS update with their results (if changed) -->| +``` + +- Default is `collect = "page"`; with no provider set to `after_response`, + behaviour is exactly §5.2. No existing behaviour changes. +- Dispatch happens only after the response is sent, because `select_until` + abandons requests still pending at the page deadline on every adapter. +- The budget is `ec.identity_after_response_budget_ms` (default 2000), + measured from the after-response dispatch. Late responses are dropped as in + §8.1.1. +- Only a navigation whose permit is confirmed (not a redirect) and whose page + phase succeeded (consent allows the EC; the page write landed or had nothing + to change) runs the phase, with an `AfterResponseWrite` permit (§9.1). + Redirects and non-navigation requests get neither permit. +- The follow-up write contains provider results only; browser ingest stays in + the page write. It never creates or revives a row: a missing, tombstoned or + withdrawn row, or one whose EC ID changed during the request, is skipped. +- The write is a compare-and-swap on the generation read just before it. On + conflict the results are dropped; the source stays due and is retried on the + next navigation. +- Results are not mirrored in the already-sent response. They are served from + KV from the next page load. A browser fallback UID for an owned source is + replaced by the provider's UID, as in §8.3. +- Source ownership is validated across both phases: a source claimed by a + page provider and an after-response provider is rejected at startup. +- Cost: the instance stays busy after the response for up to the + after-response budget, which lowers per-instance throughput when sandboxes + are reused. Keep the budget no larger than the slowest provider needs. +- Only the Fastly adapter runs this phase (§2.2). Other adapters do not yet + run it, so their `after_response` providers make no lookups. +- Cookie reads and generated IDs of an after-response provider also land only + in the follow-up write, so cookie-only providers should stay `page`. + +### 5.3 What one navigation write contains + +Exactly one of these, chosen from the request snapshot: + +| Situation | Write | +| --------------------------------------- | ----------------------------------------------------------------------------- | +| Consent withdrawn, live row | Tombstone. No lookups, no ingest. EC cookie and mirror expired. | +| Row tombstoned, no fresh consent | None. No lookups. Nothing served. | +| New EC (or orphaned EC being recovered) | Create the row with browser ingest + valid provider results. | +| Returning EC, something changed | CAS update with browser ingest + valid provider results; expired UIDs pruned. | +| Returning EC, nothing changed | None. | + +### 5.4 Later requests in the same page load + +`/auction`, `/_ts/page-bids` and integration proxies read the row snapshot and +never write. They do not receive a permit, so the write API cannot be called +(§9.1). EIDs in the `/auction` body go to bidders on that request only. + +### 5.5 Lifecycle per ID origin + +| ID origin | Collected | Persisted | Mirrored | Served from KV to `/auction` | +| --------------------- | ------------------------------ | ----------------------------- | ---------------------- | --------------------------------------- | +| `ts-eids`/`sharedId` | Navigation carrying cookies | That navigation's write | Never | Next page load (same page via body) | +| `/auction` body | `/auction` | Never | Never | That request only, from the body | +| Provider lookup | Navigation where source is due | That navigation's write | Same response (§10) | Next page load; same page if write done | +| After-response lookup | Navigation where source is due | Follow-up CAS update (§5.2.1) | Never on that response | Next page load | +| Batch sync | Batch endpoint | Its own write path | Never | After its write | + +### 5.6 Expiry and refresh + +- A UID with `exp <= now` is never served, mirrored or forwarded. +- An expired owned UID makes its source due; the owning provider is asked again + on the next navigation. No other provider and no browser input can refresh it. +- Expired UIDs stay in the row until the next write on any navigation, which + prunes them. Pruning alone never triggers a write. +- UIDs without `exp` and without refresh state are never refreshed; they stay until replaced or removed. +- A provider may keep private refresh state with a UID it writes (§6.3). Once + the state's `from` time passes, the source is due even though the UID is + still live, and the provider builds its refresh from the stored state + (`LookupInput::current`). For example, a token provider keeps the vendor's + refresh token and refreshes the token server side without any login input. + Vendor cookies are read first on every navigation; a usable cookie ID is + used instead of a refresh request. Refresh state that is empty, over 2048 + bytes or has no stated time is dropped and the UID is kept. +- A UID is refresh-due only while it is still live. A failing refresh is + therefore retried at most until the UID expires, and the provider's + `rate_limit_per_hour` also bounds it. +- A refresh reply may report an opt-out (`LookupOutcome::OptedOut`). It is + handled like a cookie opt-out: the provider's sources are cleared from the + row. +- A provider whose IDs have no vendor server may generate an ID instead of a + lookup (`IdentityProvider::generate`), for a due source when its cookies hold + none. The ID is validated and written like a lookup result; no request is + sent and no cookie is set. For example, a publisher common ID. Generation is + not rate-limited, and a source holding a live UID is not regenerated. +- A request may carry provider context (`ProviderRequest::context`) that is + never sent and is handed back with its response, for example a key that + decrypts the reply. + +### 5.7 Consent withdrawal + +- On any finalized request with the EC cookie: the tombstone is written outside + the page-load permit (§3.1); provider calls are not made; any lookup already + dispatched is discarded; no IDs are served and the mirror is expired. +- A concurrent navigation in another tab that tries to write collection results + after the tombstone fails its CAS and is not retried, so the tombstone wins. A + tombstone that loses its CAS to such a write retries, so the withdrawal still + wins. +- After fresh consent, the existing revive path creates a new row; previous IDs + are not restored. + +## 6. Type design + +### 6.1 Principles + +- **Every seam is a trait.** Five traits make the solution pluggable; each has + one production implementation and can be replaced in tests or future work + without touching shared policy. + + | Trait | Varies by | Production implementation | Other implementations | + | --------------------- | --------------------------- | ------------------------- | --------------------------------- | + | `IdentityProvider` | vendor | one per provider kind | test providers | + | `ProviderKind` | vendor (typed construction) | one per provider kind | — | + | `LookupTransport` | runtime / environment | `PlatformLookupTransport` | test transport; F4 mock transport | + | `IdentityCoordinator` | orchestration policy | `ProviderCoordinator

` | test fake for navigation tests | + | `EcKvStore` (exists) | KV backend | Fastly KV | existing in-memory test store | + +- **Static dispatch on the request path.** The coordinator, transport and + navigation code are generic. The new traits use native `async fn` (with + `#[allow(async_fn_in_trait)]`), not `async_trait`, so they add no boxed + futures; the existing `PlatformHttpClient` underneath keeps its own + `async_trait` boxing. The only trait objects are the configured providers: operators choose a + heterogeneous set, so each is boxed once at startup. `ProviderCoordinator

` + stays generic over `P`, so tests and single-provider builds use a concrete + type with no boxing. +- **Borrow per-request data.** Provider input borrows the request and the row + snapshot. Providers parse responses into types that borrow the response body + (`Cow<'body, str>`, `&'body RawValue`). Only entries that pass validation and + will be stored become owned. +- **Share config-derived identifiers.** `ProviderId` and `SourceDomain` wrap + `Arc`; clones made from config cost a reference count. `Serialize` and + `Deserialize` are implemented by hand: serializing writes the string + directly, and deserializing visits a `&str` and allocates the `Arc` + once. +- **Keep opaque JSON opaque in storage.** `ext` objects are stored as `RawJson`, + a `RawValue` newtype: never parsed into a map between provider response and + KV. They are parsed once, into `serde_json::Value`, only when an OpenRTB + request or header is built, because `crate::openrtb` already uses `Value` for + `ext`. This needs `serde_json`'s `raw_value` feature; no new crate is added. +- **Parse, don't validate.** Raw provider output converts into stored types + through `TryFrom`; past that boundary an invalid UID, agent type or match + method cannot be represented. +- **No micro-optimisation.** Small `Vec`s sized by provider or UID count are + fine; no arenas, no small-vector crates. + +### 6.2 Value types + +```rust +/// Unix time in whole seconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +/// Operator-chosen identifier of a configured provider instance. +/// +/// `Serialize` and `Deserialize` are implemented by hand (§6.1). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +/// Normalized EID source domain. +/// +/// `Serialize` and `Deserialize` are implemented by hand (§6.1); +/// deserializing normalizes and rejects invalid domains. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +/// A non-empty UID of at most [`MAX_UID_LENGTH`] bytes. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl TryFrom for UidValue { + type Error = EidValidationError; + // ... +} + +/// Non-negative OpenRTB agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32")] +pub struct AgentType(i32); + +/// An opaque JSON object, compared by its bytes. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RawJson(Box); + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + #[display("UID is empty")] + EmptyUid, + #[display("UID is too long")] + UidTooLong, + #[display("agent type is negative")] + NegativeAgentType, + #[display("match method is unknown")] + UnknownMatchMethod, + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + #[display("source is not claimed by this provider")] + UnclaimedSource, + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// OpenRTB 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(into = "i32", try_from = "i32")] +pub enum MatchMethod { + Unknown, + NoMatching, + CookieSync, + Authenticated, + Observed, + Inference, +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(exp) => now < exp, + } + } + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} +``` + +`Expiry` is stored as an optional `exp` field: absent for `Never`. + +```rust +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +mod optional_exp { + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(exp) => exp.serialize(serializer), + // Never reached: the field is skipped when `Expiry::is_never`. + Expiry::Never => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} +``` + +Sketches in this spec omit doc comments on self-describing fields and elide +bodies with `// ...`; implementations follow the documentation rules in +`AGENTS.md`. + +### 6.3 Stored row + +All additions are optional `serde` fields on schema `v: 1`. Rows written before +this change read unchanged; rows written after it stay readable by older code +through the retained `uid` field. + +`uid` is rewritten on every write to the first live UID of `eids`, and new code +ignores it when `eids` is non-empty. A known limitation of staying on `v: 1` +(D12): during a rollout or rollback, older code reads only `uid`, so it can +serve that UID after its `exp`, and an older writer that re-saves the row drops +`eids`. Both end at the next write by new code. Stored values are built only by validation +(§8.2) or by deserializing rows TS wrote, so their fields are private to the +`ec` module and exposed through accessors. + +```rust +/// A partner ID record within a KV entry, keyed by source domain. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvPartnerId { + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. + uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +/// One stored EID entry with its EID-level fields. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvEid { + #[serde(default, skip_serializing_if = "Option::is_none")] + inserter: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + matcher: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + mm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + uids: Vec, +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + /// Provider value, or the source's configured `openrtb_atype`. + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + /// Provider that wrote this UID; `None` for browser and batch UIDs. + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde(default, rename = "exp", with = "optional_exp", skip_serializing_if = "Expiry::is_never")] + expiry: Expiry, + /// Owning provider's private refresh state (§5.6); never served. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. `Debug` hides +/// `state`, which may hold a credential. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework (at most 2048 bytes). + state: String, + /// When the owning provider should be asked to refresh the UID. + from: UnixSeconds, +} +``` + +Outbound EIDs (auction, mirror, `x-ts-eids`) are built from a UID's `id`, +`atype` and `ext` only, so `refresh` never leaves the row. Refresh state is +private: it is never served, mirrored, forwarded or shown. The authenticated +admin view (`/_ts/admin/ec/{id}`) shows each `refresh.state` as `[redacted]` +and keeps `from` visible. It is removed with the UID on opt-out, withdrawal or +expiry pruning. Rows without it read unchanged. + +A UID whose refresh state is empty or over 2048 bytes, or has no stated time, +keeps the UID and drops only the refresh state. + +The source domain stays the map key in `KvEntry::ids`, so one provider can own +several sources and each UID still records its owner. + +### 6.4 Reading the row + +Consumers read through borrowed views and build owned OpenRTB values once, at +the point they serialize a request or header: + +```rust +impl KvEntry { + /// Iterates entries for `bidstream_enabled` sources, yielding only live + /// UIDs and borrowing from the row. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + now: UnixSeconds, + ) -> impl Iterator> + 'row; +} + +/// A borrowed EID entry: either a single legacy UID or a full-fidelity entry. +pub enum EidView<'row> { + Single { source: &'row str, uid: &'row str, atype: AgentType }, + Full { source: &'row str, eid: &'row KvEid, now: UnixSeconds }, +} + +impl From> for crate::openrtb::Eid { + // ... +} +``` + +The in-process `crate::openrtb::Eid` gains optional `inserter`, `matcher`, +`mm` and `ext` fields so full-fidelity entries reach bidders unchanged. + +## 7. Provider interface and crate boundary + +### 7.1 Where things live + +| Location | Contents | +| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `trusted-server-core` `ec::eid_types` | Value types and stored row types (§6). | +| `trusted-server-core` `ec::provider` | `IdentityProvider`, `ProviderKind`, the generic `build_kind` helper, input and output types, errors. No vendor code. | +| `trusted-server-core` `ec::coordinator` | `IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`, validation, merge, mirror encoding. | +| `crates/trusted-server-identity-providers` | One module per provider kind, each implementing `ProviderKind` and `IdentityProvider`, and one `build` function that matches `kind` to a `ProviderKind`. | +| `trusted-server-adapter-fastly` | Builds providers through the provider crate and a `ProviderCoordinator` at startup, and drives the navigation path with `PlatformLookupTransport`. | + +Dependencies point one way: the provider crate depends on core; core never +depends on the provider crate. Adding a provider means adding a module and one +`build` match arm in the provider crate. Shared consent, KV, cookie and auction +code never gains provider-specific branches. + +### 7.2 `IdentityProvider` + +Providers are pure: they build a request and parse a response. They never do +HTTP, KV, cookie or clock work. The trait is dyn-compatible so a configured set +can hold `Box`. + +```rust +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. +pub trait IdentityProvider { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request(&self, lookup: &LookupInput<'_>) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be parsed + /// at all. Individually malformed entries are returned and dropped by the + /// coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; + + /// Generates IDs for due sources instead of a lookup (§5.6). + /// + /// The default returns `None`: the provider makes a lookup. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + None + } +} + +impl IdentityProvider for Box

{ + // Delegates each method to `**self`. +} +``` + +Capabilities: + +```rust +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +#[derive(Debug, Clone)] +pub struct SourceCapability { + pub source: SourceDomain, + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + pub max_eids: NonZeroUsize, + pub max_uids_per_eid: NonZeroUsize, + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + Preserved, + Ignored, +} +``` + +Input and request: + +```rust +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + pub ec_id: &'a str, + pub consent: &'a ConsentContext, + pub client: ClientSignals<'a>, + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + pub ip: Option, + pub user_agent: Option<&'a str>, +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + pub method: http::Method, + pub url: url::Url, + pub headers: http::HeaderMap, + pub body: bytes::Bytes, + /// Never sent; handed back with the response (§5.6). + pub context: Option, +} +``` + +`OwnedEntries<'a>` is a borrowed view over the row that yields this provider's +`(&str, &KvEid)` pairs, keyed by the row's source strings. Provider secrets and settings are not in the +input: they are given to the provider once, when it is built (§7.3). + +`LookupInput` also lends the provider the request's cookies (`cookies`), which +`IdentityProvider::read_request_cookies` reads into a `CookieRead`. These +predate this work and are unchanged. + +Response and parsed output: + +```rust +#[derive(Debug, Clone, Copy)] +pub struct ProviderResponse<'body> { + pub status: http::StatusCode, + pub headers: &'body http::HeaderMap, + pub body: &'body [u8], + /// The request's `context`, handed back. + pub context: Option<&'body str>, +} + +#[derive(Debug)] +pub enum LookupOutcome<'body> { + Found(Vec>), + NoMatch, + /// The visitor opted out (§8.3). + OptedOut, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + pub source: Cow<'body, str>, + pub inserter: Option>, + pub matcher: Option>, + pub mm: Option, + pub ext: Option<&'body RawValue>, + pub uids: Vec>, +} + +#[derive(Debug)] +pub struct ProviderUid<'body> { + pub id: Cow<'body, str>, + pub atype: Option, + pub ext: Option<&'body RawValue>, + pub expiry: ProviderExpiry, + /// Private refresh state to keep with the UID (§5.6). + pub refresh: Option>, +} + +#[derive(Debug)] +pub struct ProviderRefresh<'body> { + pub state: Cow<'body, str>, + /// When to refresh; resolved like `exp`. + pub from: ProviderExpiry, +} + +/// Expiry as a provider states it. The coordinator turns it into [`Expiry`] +/// using its own clock, so providers never read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + Unspecified, + At(UnixSeconds), + After(Duration), +} + +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + #[display("provider request could not be built")] + Request, + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} +``` + +Providers typically deserialize their wire format into private structs with +`#[serde(borrow)]` and map them into `ProviderEid<'body>`. + +ms8lur5ZG51TgstVd7Khm8vkBRRHWu4H +zc_4380a19dcee84488447f6996dca327d4e6f7d2d547dd53db961888620f590805 + +### 7.3 `ProviderKind` and registration + +Construction is typed per provider; only the final value is erased. + +```rust +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +pub struct ProviderCommon<'a> { + pub endpoint: &'a url::Url, + pub sources: &'a [SourceDomain], + pub secrets: &'a ProviderSecrets, +} + +/// Parses `settings` for `K` and builds a boxed provider. Lives in core. +/// +/// `secrets` are resolved by the adapter at startup through the existing +/// secret path, from the key names in `config`. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when `settings` does not +/// deserialize into `K::Settings` or [`ProviderKind::build`] rejects it. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report>; +``` + +The provider crate's single entry point, shown with the `pull-sync` arm that +PR5 adds (PR3 ships only the fallback arm): + +```rust +/// Builds the provider for a configured `kind`. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + match config.kind() { + PullSync::KIND => build_kind::(config, secrets), + kind => Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))), + } +} +``` + +`ProviderSecrets` holds `Redacted` values resolved through the existing +secret path and is passed only at build time. + +```rust +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + #[display("provider kind is not registered")] + UnknownKind, + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} +``` + +`IdentityProviderConfig` is the deserialized `[[ec.identity_providers]]` entry +(§12.1). `HashedEmail` is a newtype over a validated lowercase hex SHA-256 +digest with no constructor reachable from request data in V1. + +## 8. Coordinator + +### 8.1 Responsibilities + +The coordinator owns, for every provider: + +1. **Eligibility.** Provider enabled; EC present or being created; row not + tombstoned; at least one due source; sources needing `HashedEmail` removed. +2. **Consent.** The existing EID-persistence check passes, and the provider's + `tcf_vendor_id`, when configured, has vendor consent in the effective TCF + string. A failure skips that provider only, before any request is built. +3. **Destination.** The built URL must be `https` and its host must be in the + provider's configured `allowed_domains`. The URL comes from operator config; + browser data never selects a destination. A rejected destination skips the + provider and is counted. +4. **Transport.** All eligible requests are started through a + `LookupTransport` and gathered until all complete or the shared deadline + passes. Late responses are dropped (§8.1.1). +5. **Validation** (§8.2), **merge** (§8.3) and preparing the single write. +6. **Mirror** encoding (§10) from the row as it stands after the navigation. +7. **Logs and metrics** (§12). + +#### 8.1.1 `LookupTransport` + +The transport sends provider requests and gathers responses within a deadline. +It is the only coordinator dependency that touches the network, so swapping it +is how tests and the future debug mode (F4) run real providers and the real +coordinator against mock responses. + +```rust +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + pub provider: ProviderIndex, + pub value: T, +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + pub status: http::StatusCode, + pub headers: http::HeaderMap, + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderIndex(u16); + +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + #[display("lookup request could not be sent")] + Send, + #[display("lookup did not complete before the deadline")] + Timeout, + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} +``` + +`Deadline` wraps the `web_time::Instant` that `select_until` takes, because +core is built into the Cloudflare Worker (CI wasm clock lint). +`PlatformLookupTransport<'s>` borrows `RuntimeServices` and is a thin adapter +over the platform client: `start` calls `send_async`, and `gather` calls +`select_until` with the shared deadline while requests remain in flight, +stopping when none remain (it rejects an empty list) or it returns `Ok(None)`, +and tags each result with its provider. It adds no timing logic of its own. Response +bodies are capped at 64 KiB. + +#### 8.1.2 `IdentityCoordinator` + +The navigation path depends on the `IdentityCoordinator` trait, generic over +the coordinator and the transport. The trait covers the coordinator's whole +surface on a navigation: start lookups, collect them, plan the single row +change, and build the mirror. Its production implementation is +`ProviderCoordinator

`. Tests of coordinator rules (validation, merge, +mirror) run `ProviderCoordinator` with test providers and a test transport; a +fake `IdentityCoordinator` is used only for navigation wiring tests, where +those rules are not under test. + +```rust +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget, validation, merging and mirror encoding. Providers never see +/// KV, cookies, HTTP or the clock. Implementations never write KV: they return +/// the change and the caller spends the [`PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when nothing is due, consent does not + /// permit EID collection, or no provider is enabled. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; + + /// Merges browser ingest and lookup results into the navigation's single + /// row change. + /// + /// Returns `None` when the row would not change, so no write is made. + fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; + + /// Builds the mirror cookie from the row as it stands after this + /// navigation: the committed row, or the snapshot when no write landed. + fn mirror(&self, row: Option<&KvEntry>, consent: &ConsentContext, now: UnixSeconds) -> MirrorCookie; +} +``` + +Supporting types: + +```rust +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + pub ec_id: &'a str, + pub snapshot: &'a EcKvSnapshot, + pub consent: &'a ConsentContext, + pub client: ClientSignals<'a>, + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. Only the coordinator module can +/// create a non-empty value. `Default` (empty) and `Debug` (counts only) are +/// implemented by hand so they do not require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// Request time, for validating `exp` and resolving `ProviderExpiry::After`. + now: UnixSeconds, + skipped: Vec>, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Default)] +pub struct LookupResults { + lookups: Vec, +} + +#[derive(Debug)] +pub struct ProviderLookup { + pub provider: ProviderId, + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `status` is `Found`. + pub entries: Vec<(SourceDomain, Vec)>, +} + +/// One outcome per provider per navigation, used for logs and metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + Found { kept: u16, dropped: DropCounts }, + NoMatch, + Timeout, + HttpError(http::StatusCode), + TransportFailed, + InvalidResponse, + Skipped(SkipReason), +} + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + pub invalid_uid: u16, + pub invalid_field: u16, + pub unclaimed_source: u16, + pub expired: u16, + pub over_limit: u16, + pub duplicate: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + Disabled, + Consent, + NotDue, + Input, + Destination, + RateLimited, + OptedOut, +} + +/// Inputs to [`IdentityCoordinator::plan_write`]. +/// +/// `PartnerIdUpdate` (today `pub(crate)` in `ec::kv`) becomes `pub` so this +/// public trait input does not leak a private type. +pub struct WritePlanInput<'a> { + pub snapshot: &'a EcKvSnapshot, + pub consent: &'a ConsentContext, + pub new_entry: Option<&'a KvEntry>, + /// Updates from `ts-eids` and `sharedId`, already reduced by today's rules. + pub browser_ingest: &'a [PartnerIdUpdate], + pub lookups: &'a LookupResults, + pub now: UnixSeconds, +} + +/// What to do with the mirror cookie on this response. +#[derive(Debug)] +pub enum MirrorCookie { + Set(MirrorValue), + Expire, +} + +/// An encoded mirror value within the size limit (§10). +#[derive(Debug)] +pub struct MirrorValue(String); +``` + +The production coordinator: + +```rust +/// The production coordinator, built once at startup. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation, `bidstream_enabled` + /// and config order for the mirror. + partners: Arc, + budget: Duration, +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated at startup. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +impl IdentityCoordinator for ProviderCoordinator

{ + // ... +} +``` + +`rate_limit_per_hour` is not a new traffic control: it carries over pull sync's +existing per-EC, per-partner hourly limit (PR5), which needs the `RateLimiter` +state that pure providers cannot hold. Operators may also set it per provider +with the optional `rate_limit_per_hour` config field (§12.1), for example to +bound repeat lookups for sources a provider never resolves; it is unset by +default. Broader traffic controls remain future work (F2, F3). + +The navigation path, in order: + +```rust +async fn run_navigation(coordinator: &C, transport: &T, navigation: Navigation<'_>) -> Response +where + C: IdentityCoordinator, + T: LookupTransport, +{ + let Navigation { permit, request, graph, ec, .. } = navigation; + let pending = coordinator.dispatch(transport, &ec.lookup_context()).await; + let mut response = fetch_origin(request).await; + let lookups = coordinator.collect(transport, pending).await; + let mut row = ec.snapshot_entry(); + let change = permit + .confirm(response.status()) + .zip(coordinator.plan_write(&ec.write_plan(&lookups))); + if let Some((permit, change)) = change { + // One attempt, never retried (§9.2). + match graph.commit_page_load(permit, ec.id(), change) { + Ok(committed) => row = Some(committed), + Err(report) => ec.record_write_failure(&report), + } + } + ec.apply_cookies(&mut response, row.as_ref(), coordinator.mirror(row.as_ref(), ec.consent(), ec.now())); + response +} +``` + +### 8.2 Validation + +Validation keeps valid entries and drops only invalid ones. Each drop is +counted by reason. + +| Level | Rule | +| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| EID | `source` normalizes to a source this provider claims in config; otherwise dropped. | +| EID | `inserter`/`matcher` at most 255 bytes; `mm` within OpenRTB values; `ext` a JSON object of at most 1 KiB. EID-level fields are dropped (not the entry) when `eid_level_fields` is `Ignored`. | +| UID | `id` non-empty after trimming and at most `MAX_UID_LENGTH` (1024) bytes; `ext` a JSON object of at most 1 KiB. | +| UID | Refresh `state` non-empty and at most 2048 bytes, with a stated `from`; otherwise the refresh state is dropped and the UID kept. `from` is resolved like `exp`. | +| UID | `atype` missing is filled from the source's `openrtb_atype`. | +| UID | `exp`, when present, must be greater than now; otherwise dropped. | +| Counts | Entries beyond `max_eids`, or UIDs beyond `max_uids_per_eid`, are dropped from the end. | +| Dedupe | UIDs with the same (`source`, `id`) keep the first occurrence, as in §11.1. | +| Row size | If the merged row would exceed 16 KiB serialized, new provider UIDs are dropped in reverse provider-config order, last UID first, until it fits. | + +An EID entry left with no UIDs is dropped. A response that cannot be parsed at +all is a failure for that provider; other providers and browser input are +unaffected. + +### 8.3 Merge into the row + +For each provider with a `Found` outcome and each source in its response: + +- The validated entries replace every UID of that source in the row, including + unowned ones (D8). Each written UID records `owner` and `exp`. +- Sources the provider claims but did not return stay unchanged. +- `NoMatch`, failures, timeouts and skipped providers leave the row unchanged. +- `OptedOut` (a cookie or a refresh reply) clears every source the provider + claims; generated and cookie IDs merge like `Found`. + +Browser ingest (§8.4) is applied in the same in-memory merge, before provider +results, so the write order never lets browser data overwrite provider data. + +### 8.4 Browser ingest (navigation only) + +Unchanged rules: registered sources only; one UID per source, the first valid +UID of the last matching entry, with `sharedId` applied last; no EID-level +fields; fill missing sources only, where a source holding only expired UIDs +counts as holding UIDs until a write prunes them. New rule: a provider-owned +source is skipped entirely. + +### 8.5 Batch sync + +Batch sync keeps its own write path. A record for a source claimed by an +enabled provider is rejected and counted as `provider_owned`; other records in +the same batch are processed as today. + +### 8.6 Disabled providers + +A disabled provider makes no outbound requests and is not considered for +eligibility, and its sources stop being provider-owned. UIDs it wrote earlier +keep serving until `exp`, or until replaced or removed; once pruned, browser +ingest may fill the source again. Other providers and client-side collection +are unaffected. + +## 9. KV write counting, failure and retry + +### 9.1 Permit + +The write limit is enforced by types. A navigation receives a +`NavigationPermit`; it becomes a `PageLoadWrite` only when the final response +is not a redirect; the write consumes it. Neither type is `Clone` or +`Copy`, `classify` is `pub(crate)` and called once by the router, and +`commit_page_load` consumes the permit, so a navigation cannot write twice and +no other request can write at all. + +```rust +/// Issued to a request classified as a page-load navigation (§3.1). +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation. + /// + /// Called once per request, by the router only. + #[must_use] + pub(crate) fn classify(request: &http::Request) -> Option; + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect: the redirect target is the page load. + #[must_use] + pub fn confirm(self, status: http::StatusCode) -> Option; +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +// Also on the navigation permit: confirms it together with the permit for the +// after-response phase (§5.2.1). +// pub fn confirm_with_after_response(self, status: http::StatusCode) +// -> Option<(PageLoadWrite, AfterResponseWrite)>; + +/// Proof for the one follow-up update; consumed by `commit_after_response_lookups`. +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { entry: KvEntry, expected: Generation }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + #[display("row changed since it was read")] + Conflict, + #[display("row already exists")] + AlreadyExists, + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +impl KvIdentityGraph { + /// Applies the navigation's single row change in one attempt. + /// + /// # Errors + /// + /// Returns [`KvWriteError`] on CAS conflict, an existing row on create, or + /// a store failure. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result>; +} +``` + +`Generation` is a newtype over the KV generation captured in +`EcKvSnapshot::Present`. When the snapshot carries no generation, +`plan_write` returns `None` for updates and the navigation counts +`no_generation`: TS never overwrites a row blindly. + +After PR2 all browser-driven KV writes except withdrawal tombstones go through +`commit_page_load`: new EC creation, orphan recovery and browser ingest; PR3 +adds provider results. Withdrawal tombstones keep their existing path (§3.1). The post-send pull-sync write is the one exception until +PR5 removes it. + +### 9.2 Failure behaviour + +| Failure | Effect | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| CAS conflict or store error | No retry. Row unchanged. Mirror built from the unchanged row. New EC cookie not set. Counted. | +| Tombstone write fails | Bounded CAS retry first. Mirror expired and no IDs served (consent gating). EC cookie kept so the next request retries it. | +| EC-ID collision on create | Same as above; the next navigation generates a new EC. | +| Provider timeout/outage/invalid | That provider's results are dropped; the write proceeds with everything else that is valid. | +| All providers fail | The write proceeds if browser ingest changed anything; otherwise no write. | +| Coordinator budget exceeded | Late responses dropped; the response is never held past the budget. | +| After-response row missing, withdrawn or EC changed | Nothing written (`no_row`); results dropped. | +| After-response write conflict | Results dropped; the source stays due and is retried on the next navigation. The sent response is unaffected. | + +Failed lookups are retried only by a later navigation where the source is +still due. + +## 10. Mirror cookie + +| Attribute | Value | +| --------- | -------------------------------------------------------------------------------------------- | +| Name | `__Host-ts-eids-mirror` | +| Flags | `Path=/; Secure; SameSite=Lax`; no `Domain`; not `HttpOnly` | +| Max-Age | Same as the EC cookie | +| Value | base64url of `{"v":1,"eids":[...]}`; each entry is an OpenRTB EID whose UIDs may carry `exp` | +| Size | Encoded value at most 3,584 bytes | + +- **Contents:** live, provider-owned UIDs of `bidstream_enabled` sources, with + their EID-level and UID-level fields. +- **Fill order:** sources in `[[ec.partners]]` config order; entries and UIDs in + stored order. Each UID is added if the encoded value still fits; otherwise it + is skipped and filling continues. KV keeps the complete set. +- **Set** on every consented navigation response from the row as it stands + after the navigation: the committed row, or the snapshot when no write + landed. Setting a cookie is not a KV write, and this restores a mirror the + browser lost. +- **Expired** (`Max-Age=0`) on any response where EID consent is absent or + withdrawn, whatever happened to the KV write, and on a navigation whose + mirror would be empty. +- **The server never reads it.** Readers on the page must drop UIDs whose `exp` + has passed and strip `exp` before use. + +## 11. Consumers + +All consumers read the row snapshot and gate by current consent and +`bidstream_enabled`, as today. None write KV. + +| Consumer | Behaviour | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/auction` | Resolves the row to EIDs, full-fidelity for provider entries, dropping UIDs that are not live. Merges with the body EIDs (§11.1). Server IDs need no browser resend. | +| `/_ts/page-bids` | Same as `/auction`. | +| `x-ts-eids` header | Built from the merged set with the existing truncation flag. | +| `/_ts/api/v1/identify` | Includes provider UIDs that are live, under existing authentication and partner gating. | +| Integration proxies | Use the same resolver where they forward EIDs. | + +### 11.1 Merge for auctions + +1. KV entries first, then body EIDs. +2. EID entries are distinct when (`source`, `inserter`, `matcher`, `mm`) differ; + both are kept. +3. A UID whose `id` already appears under the same source is a duplicate; the + first occurrence (KV) keeps its fields. +4. Distinct UIDs are kept under their entry. + +## 12. Configuration, secrets and observability + +### 12.1 Configuration + +```toml +[ec] +# Shared time budget for all provider lookups on one navigation. +identity_budget_ms = 200 +# Budget for providers with collect = "after_response", run after the +# response is sent (§5.2.1). +identity_after_response_budget_ms = 2000 + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +enabled = true +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry and must be in the kind's capabilities. +sources = ["ids.example.com"] +# Optional: require TCF vendor consent for this provider. +tcf_vendor_id = 9999 +# Optional: "page" (default) or "after_response" (§5.2.1). +collect = "page" +# Optional: most lookups per EC per hour (§8.1); unset means unlimited. +rate_limit_per_hour = 4 + +# Secret Store key names, resolved like other secret references. +[ec.identity_providers.secrets] +token = "example_provider_token" + +[ec.identity_providers.settings] +# Provider-specific settings, parsed by the provider implementation. +``` + +Startup validation rejects: unknown `kind`; a source with no `[[ec.partners]]` +entry; a source outside the kind's capabilities; a source claimed by two +enabled providers; an `endpoint` host outside `allowed_domains`; placeholder +secrets in deploy validation. + +The operator CLI's config checks (`ts config validate`, `push` and `diff`) +also build every enabled provider of a registered kind, as the adapter does +per navigation, so kind-specific setting errors fail before deploy. Core's +own validation cannot do this because provider kinds live outside core. + +Secrets are Secret Store key references, resolved at startup into +`Redacted` values through the existing secret path. They are never written to cookies or logs, and source-controlled +examples use placeholders. + +### 12.2 Logs and metrics + +Per provider and navigation, one outcome: `found`, `no_match`, `timeout`, +`http_error`, `transport_error`, `invalid_response`, `skipped_disabled`, `skipped_consent`, +`skipped_not_due`, `skipped_input`, `skipped_rate_limited`, `skipped_opted_out`, +`destination_rejected`; plus counts of +entries and UIDs kept and dropped by reason. Per navigation, one write outcome: +`written`, `no_change`, `conflict`, `error`, `not_permitted`, +`no_generation`. A navigation that runs the after-response phase (§5.2.1) has +one more write outcome: `written`, `unchanged`, `no_row`, `conflict` or +`failed`. Logs include the +provider ID, source, counts and durations, and the EC ID only in its existing +truncated form. Raw UIDs, EID values and encrypted envelopes are never logged. + +## 13. PR breakdown + +### 13.1 Required + +The work below is delivered as two pull requests: + +- **PR A** — the work units PR1, PR2 and PR3, stacked on the open platform PR + #1433, which provides the deadline-bounded `select_until` that + `PlatformLookupTransport` uses. +- **PR B** — the work units PR4, PR5 and PR6, stacked on PR A. + +PR A and PR B ship in the same release, because PR A alone leaves the old +post-send pull-sync write as a second write per page load. Within each pull +request the units are separate, ordered commits so they stay reviewable. + +| PR | Title | Contents | Depends on | +| --- | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| PR0 | Spec | This document. | — | +| PR1 | Full-fidelity EID storage and resolution | `ec::eid_types` value types (§6.2); `KvEid`/`KvUid` additive schema; EID-level fields on `crate::openrtb::Eid`; resolver drops UIDs that are not live; auction merge (§11.1); `x-ts-eids` and identify use it. No new writers. | PR0 | +| PR2 | Page-load write permit | Navigation classifier; `PageLoadWrite` and `commit_page_load`; single attempt; non-navigation requests read-only except withdrawal tombstones; browser ingest on navigation only; withdrawal handling (§3.1, §5.7); integration-proxy EC minting stops. | PR0 | +| PR3 | Provider trait and coordinator | `ec::provider` (`IdentityProvider`, `ProviderKind`), `ec::coordinator` (`IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`), provider crate with no kinds, config and validation, consent, allowlist, budgeted fan-out, validation, merge into the navigation write, provider-owned skip in ingest, metrics, example config. | PR1, PR2, #1433 | +| PR4 | Batch sync on provider-owned sources | Reject and count records for sources an enabled provider claims. | PR3 | +| PR5 | Pull sync as a provider | `pull-sync` kind built from existing `[[ec.partners]]` pull-sync settings with no config change; existing per-partner rate limit kept as provider-scoped eligibility; post-send dispatcher and completeness marker removed, leftover marker cookies expired. Behaviour change for existing pull-sync partners: their sources become provider-owned, so browser ingest skips them and, with PR4, batch sync rejects them; once PR6 lands their IDs are mirrored when `bidstream_enabled`. | PR3 | +| PR6 | Mirror cookie | Adds `IdentityCoordinator::mirror` (PR3 ships the trait without it); encoding, fill order, set and expiry rules (§10). | PR3 | + +### 13.2 Optional and future (Tech Lead decision) + +| ID | Task | Notes | Depends on | +| --- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| O1 | tsjs reads the mirror (D6b) | Reads the mirror, drops expired UIDs, strips `exp`, passes entries to Prebid user IDs. Separate PR. | PR6 | +| F1 | Concrete provider implementations | R§5.1. One spec and one PR per vendor named in R§6, each a module and `build` arm in the provider crate. | PR3 | +| F2 | Provider lookup traffic controls | R§5.2. Best-effort dedup per EC and provider, bounded no-match caching kept distinct from failures, refresh jitter. Strict cross-instance dedup is a separate decision. | PR3 | +| F3 | Per-provider cooldown | R§5.3. Pause after repeated failures, then probe for recovery. | PR3 | +| F4 | Provider debug mode | R§5.4. A mock `LookupTransport` runs real providers and the real coordinator against configurable responses; no real provider calls, no production KV or cookie mutation. | PR3 | + +## 14. Acceptance scenarios + +1. **First navigation, consent granted.** No EC. One provider is enabled and + due. The provider answers within budget. → One KV create holding the + provider UIDs (with owner and `exp`) and browser-ingested UIDs; the EC cookie + and mirror are set on the response; the provider is called once. +2. **One write per page load.** A navigation is followed by three `/auction` + calls, one `/_ts/page-bids` call and two integration-proxy calls, all + carrying new `ts-eids` content. → At most one KV write, on the navigation. +3. **Same-page and next-page availability.** Provider IDs written by the + navigation → the next page load's `/auction` contains them without the + browser resending them; a same-page `/auction` contains them only if it + read the row after the write. +4. **Slow provider.** Provider A answers in 50 ms, provider B in 500 ms, budget + 200 ms. → A's UIDs are written; B's response is dropped and counted as + `timeout`; the response is not held past the budget. +5. **Partial invalid response.** A provider returns three UIDs, one empty and + one over 1024 bytes. → One UID is stored; two drops are counted. +6. **Write failure.** The navigation's CAS fails. → No retry; mirror built from + the unchanged row; + on a new EC no EC cookie is set; the next navigation looks up again. +7. **Expiry.** A provider UID's `exp` passes. → It is excluded from `/auction`, + `x-ts-eids`, identify and the mirror; the next navigation asks only its + owning provider. +8. **No expiry.** A provider UID without `exp` → served until the provider + replaces it or consent is withdrawn; never refreshed unless it carries refresh state + (scenario 24). +9. **Withdrawal on a navigation.** → Tombstone written; no provider calls; EC + cookie and mirror expired; no IDs served. +10. **Withdrawal on `/auction`.** → Tombstone written (§3.1), EC cookie and + mirror expired, no IDs served. +11. **Tombstone beats a concurrent write.** Tab A writes a tombstone while tab + B's navigation is collecting provider results. → B's CAS fails and is not + retried; the provider IDs are never stored or served. +12. **Provider-owned source.** A browser `ts-eids` UID and a batch record arrive + for a source an enabled provider claims. → Browser UID ignored; batch + record rejected and counted; provider UIDs unchanged. +13. **Multi-source provider.** One provider returns entries for two claimed + sources with two UIDs each and EID-level fields. → Both sources stored with + all four UIDs and fields, each owned by that provider; `/auction` forwards + them at full fidelity. +14. **Disabled provider.** One of two providers is disabled. → No outbound + request to it; the other provider and browser ingest behave as before. +15. **Destination rule.** A provider builds a URL whose host is not in + `allowed_domains`. → No request is sent; counted as `destination_rejected`. +16. **Hashed-email source.** A source requires `HashedEmail`. → Skipped before + any call, counted as `skipped_input`. +17. **Mirror trimming.** Provider UIDs exceed 3,584 encoded bytes. → The mirror + holds the deterministic config-order fill of §10; KV holds all of them. +18. **No secrets or IDs in logs.** Every outcome above produces logs and metrics + without raw UIDs, envelopes or secrets. +19. **Nothing due.** A returning visitor with all owned UIDs live and no new + browser input. → No provider calls, no KV write. +20. **Redirect.** The navigation's origin response is a 302. → No KV write; the + redirect target navigation may write. +21. **Pull-sync partner after PR5.** A partner with pull sync enabled also + receives a browser `ts-eids` UID and a batch record. → Both are ignored or + rejected as for any provider-owned source; the pull-sync provider's UID is + stored and served. +22. **After-response provider.** Provider A is `collect = "page"` and answers + in 50 ms; provider B is `collect = "after_response"` and answers in + 900 ms; page budget 200 ms. → The response is sent without waiting for B; + the page write holds A's UIDs and browser ingest; after the response, one + CAS update adds B's UIDs with owner and `exp`; the next page load serves + them. +23. **After-response on a withdrawn or changed row.** The row is tombstoned, + or written by another request, between the page write and B's result. → + No row is created or revived; on a generation conflict B's results are + dropped and B is asked again on the next navigation. +24. **Provider refresh.** A provider-written UID is live until 5000 and holds + refresh state with `from` 900; the navigation is at 1000 and the request carries no usable vendor + cookie. → The source is + due; the provider's request is built from the stored state; the refreshed + UID and its new refresh state are written. With `from` 2000 the source is + not due and nothing is sent. +25. **Refresh reports an opt-out.** A refresh reply reports an opt-out. → The + provider's sources are cleared from the row; nothing is stored for them. +26. **Generated ID.** A provider that generates IDs has a due source and no + cookie ID. → No request is sent; the generated UID is validated and written + with its owner; on the next navigation the source holds a live UID and is + not regenerated. +27. **Refresh state is never served.** A row holds a UID with refresh state. + → `/auction`, the mirror cookie and `x-ts-eids` carry the UID's `id`, + `atype` and `ext` only. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 99afa869dd..31a608ec38 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -86,6 +86,12 @@ passphrase = "ec_passphrase" ec_store = "ec_identity_store" # Max concurrent partner pull-sync requests. pull_sync_concurrency = 3 +# Optional identity-provider budgets (see [[ec.identity_providers]] below). +# Every page-phase provider shares one budget (1-1000 ms): +# identity_budget_ms = 200 +# Budget for providers with collect = "after_response", which run after the +# response is sent and write their results in a follow-up write (1-5000 ms): +# identity_after_response_budget_ms = 2000 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds @@ -107,6 +113,36 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false +# Optional server-side identity providers (Fastly only), looked up on publisher +# navigations before the origin fetch. Their budgets are set in [ec] above. An +# enabled provider whose kind this build does not include fails startup. +# +# [[ec.identity_providers]] +# id = "example-provider" +# kind = "example-kind" +# enabled = false +# endpoint = "https://ids.example.com/v1/lookup" +# The endpoint host must be listed here. +# allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry (for example one with +# source_domain = "ids.example.com") and must be claimed by only one enabled +# provider. +# sources = ["ids.example.com"] +# Optional: require TCF vendor consent for lookups and for sending this +# provider's stored IDs to bidders. +# tcf_vendor_id = 9999 +# "page" (default) collects before the response; "after_response" collects +# after it is sent, only for an EC issued on an earlier request. +# collect = "page" +# Optional: most lookups per EC hash (client IP) per hour. On Fastly this is +# enforced per minute with a minimum of one, so values below 60 behave like 60. +# rate_limit_per_hour = 60 +# Secret Store key name, resolved like other secret references. +# [ec.identity_providers.secrets] +# token = "example_provider_token" +# Provider-specific settings, parsed by the provider kind. +# [ec.identity_providers.settings] + # ============================================================================= # OPTIONAL — Core features (disabled/omitted by default)