From 5b867588dd4b84154d64194dbf5cd5bc48711797 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 30 Oct 2026 07:03:15 +0530 Subject: [PATCH 01/58] Store and serve full-fidelity provider EIDs Add EID value types, an additive KV schema for provider-written EIDs with owner and expiry, EID-level fields on the OpenRTB Eid, and a resolver that serves only live UIDs to auctions, x-ts-eids, identify and the admin auction preview. Auction EIDs merge per (source, inserter, matcher, mm, ext) with first-occurrence UID dedupe. KV writes over 16 KiB are refused. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 246 +++++--- .../src/auction/formats.rs | 8 +- .../src/auction/test_support.rs | 8 +- crates/trusted-server-core/src/ec/admin.rs | 86 ++- .../trusted-server-core/src/ec/eid_types.rs | 594 ++++++++++++++++++ crates/trusted-server-core/src/ec/eids.rs | 213 ++++++- crates/trusted-server-core/src/ec/finalize.rs | 36 +- crates/trusted-server-core/src/ec/identify.rs | 111 +++- crates/trusted-server-core/src/ec/kv.rs | 119 ++-- crates/trusted-server-core/src/ec/kv_types.rs | 415 +++++++++++- crates/trusted-server-core/src/ec/mod.rs | 2 + .../trusted-server-core/src/ec/prebid_eids.rs | 37 +- .../trusted-server-core/src/ec/pull_sync.rs | 12 +- .../src/ec/pull_sync_marker.rs | 8 +- .../src/integrations/prebid.rs | 24 +- crates/trusted-server-core/src/openrtb.rs | 73 ++- 16 files changed, 1733 insertions(+), 259 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/eid_types.rs diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 086515fa20..b0121f7490 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -14,7 +14,8 @@ use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; -use crate::ec::eids::{resolve_partner_ids, to_eids}; +use crate::ec::current_timestamp; +use crate::ec::eid_types::UnixSeconds; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::MAX_UID_LENGTH; use crate::ec::prebid_eids::parse_prebid_eids_cookie; @@ -535,8 +536,8 @@ pub(crate) fn resolve_auction_eids( return Some(Vec::new()); } - let resolved = resolve_partner_ids(registry, entry); - Some(to_eids(&resolved)) + let now = UnixSeconds::new(current_timestamp()); + Some(entry.live_eids(registry, now).map(Eid::from).collect()) } pub(crate) fn resolve_client_auction_eids( @@ -600,7 +601,7 @@ fn parse_client_auction_eids(raw: Option<&JsonValue>) -> Option> { continue; } - eids.push(Eid { source, uids }); + eids.push(Eid::new(source, uids)); } if eids.is_empty() { None } else { Some(eids) } @@ -635,8 +636,9 @@ pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { - let mut merged = Vec::new(); + let mut merged: Vec = Vec::new(); + // Server-resolved entries come first so their fields win on duplicates. for eid in resolved_eids .into_iter() .flatten() @@ -645,50 +647,55 @@ pub(crate) fn merge_auction_eids( if eid.source.is_empty() { continue; } - - let source_index = match merged - .iter() - .position(|existing: &Eid| existing.source == eid.source) - { - Some(index) => index, - None => { - merged.push(Eid { - source: eid.source.clone(), - uids: Vec::new(), - }); - merged.len() - 1 - } - }; - - for uid in eid.uids { + let Eid { + source, + inserter, + matcher, + mm, + ext, + uids, + } = eid; + let mut kept = Vec::with_capacity(uids.len()); + for uid in uids { if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { continue; } - - if let Some(existing_uid) = merged[source_index] - .uids - .iter_mut() - .find(|existing| existing.id == uid.id) - { - if existing_uid.atype.is_none() { - existing_uid.atype = uid.atype; - } - if existing_uid.ext.is_none() { - existing_uid.ext = uid.ext; - } - } else { - merged[source_index].uids.push(uid); + let duplicate = merged + .iter() + .filter(|existing| existing.source == source) + .flat_map(|existing| existing.uids.iter()) + .chain(kept.iter()) + .any(|existing: &Uid| existing.id == uid.id); + if !duplicate { + kept.push(uid); } } + if kept.is_empty() { + continue; + } + // Entries are distinct when any EID-level field differs, including + // `ext` (compared by JSON value), so no entry's fields are lost. + let same_entry = merged.iter_mut().find(|existing| { + existing.source == source + && existing.inserter == inserter + && existing.matcher == matcher + && existing.mm == mm + && existing.ext == ext + }); + match same_entry { + Some(existing) => existing.uids.extend(kept), + None => merged.push(Eid { + source, + inserter, + matcher, + mm, + ext, + uids: kept, + }), + } } - merged.retain(|eid| !eid.uids.is_empty()); - - if merged.is_empty() { - None - } else { - Some(merged) - } + (!merged.is_empty()).then_some(merged) } #[cfg(test)] @@ -2058,33 +2065,132 @@ mod tests { ); } + #[test] + fn merge_auction_eids_keeps_distinct_eid_level_entries_per_source() { + let resolved = Some(vec![Eid { + inserter: Some("a.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![ + Eid { + inserter: Some("b.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }, + Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(2), + ext: None, + }], + ), + ]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!( + merged.len(), + 2, + "should keep entries with different inserters" + ); + assert_eq!( + merged[0].uids.len(), + 1, + "should drop duplicate u1 from client" + ); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should keep server fields for duplicate" + ); + assert_eq!( + merged[1].uids[0].id, "u2", + "should keep distinct client uid" + ); + } + + #[test] + fn merge_auction_eids_keeps_entries_with_different_ext_separate() { + let resolved = Some(vec![Eid { + ext: Some(json!({ "k": "server" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![Eid { + ext: Some(json!({ "k": "client" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!(merged.len(), 2, "should keep entries with different ext"); + assert_eq!( + merged[0].ext, + Some(json!({ "k": "server" })), + "should keep server entry ext" + ); + assert_eq!( + merged[1].ext, + Some(json!({ "k": "client" })), + "should keep client entry ext" + ); + assert_eq!(merged[1].uids[0].id, "u2", "should keep client uid"); + } + #[test] fn merge_auction_eids_deduplicates_client_and_resolved_ids() { - let client_eids = Some(vec![Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }]); + )]); let resolved_eids = Some(vec![ - Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_string(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_string(), + vec![Uid { id: "LR_xyz".to_string(), atype: Some(3), ext: None, }], - }, + ), ]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2098,22 +2204,22 @@ mod tests { #[test] fn merge_auction_eids_preserves_multiple_uids_per_source() { - let client_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_client".to_string(), atype: None, ext: None, }], - }]); - let resolved_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_server".to_string(), atype: Some(3), ext: None, }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2125,22 +2231,22 @@ mod tests { #[test] fn merge_auction_eids_prefers_server_resolved_metadata_on_conflict() { - let client_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(1), ext: Some(json!({ "provider": "client" })), }], - }]); - let resolved_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(3), ext: Some(json!({ "provider": "server" })), }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index b684c80dde..15076adb6a 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -850,14 +850,14 @@ mod tests { #[test] fn response_includes_eid_headers_when_eids_present() { let mut request = make_auction_request(); - request.user.eids = Some(vec![Eid { - source: "ssp.com".to_owned(), - uids: vec![Uid { + request.user.eids = Some(vec![Eid::new( + "ssp.com".to_owned(), + vec![Uid { id: "uid-1".to_owned(), atype: Some(3), ext: None, }], - }]); + )]); let settings = make_settings(); let result = make_empty_result(); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..c839a582eb 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -92,14 +92,14 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { raw_ac_string: Some("fictional-ac".to_string()), ..Default::default() }), - eids: Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { + eids: Some(vec![Eid::new( + "identity.example".to_string(), + vec![Uid { id: "fictional-uid".to_string(), atype: Some(1), ext: None, }], - }]), + )]), }, device: Some(DeviceInfo { user_agent: Some("Fictional Browser".to_string()), diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e21fab702c..383edddf15 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -31,7 +31,8 @@ use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; use crate::openrtb::Eid; -use super::eids::{resolve_partner_ids, to_eids}; +use super::current_timestamp; +use super::eid_types::UnixSeconds; use super::generation::is_valid_ec_id; use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; @@ -251,8 +252,8 @@ struct AuctionEidsView { struct SkippedPartnerId { /// Partner namespace key in the entry's `ids` map. source_domain: String, - /// Why the auction resolution skips it: `empty_uid`, `not_in_registry`, - /// or `bidstream_disabled`. + /// Why the auction resolution skips it: `empty_uid`, `expired` (provider + /// record with no live UID), `not_in_registry`, or `bidstream_disabled`. reason: &'static str, } @@ -479,22 +480,25 @@ fn iso_timestamp(unix_seconds: u64) -> Option { .map(|datetime| datetime.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) } -/// Derives the auction EID view for a valid entry, mirroring the filters in -/// [`resolve_partner_ids`] and reporting why each stored ID was skipped. +/// Derives the auction EID view for a valid entry through the same +/// [`KvEntry::live_eids`] resolution the live auction uses, reporting why each +/// stored ID was skipped. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { - let resolved = resolve_partner_ids(registry, entry); - let eids = to_eids(&resolved); + let now = UnixSeconds::new(current_timestamp()); + let eids = entry.live_eids(registry, now).map(Eid::from).collect(); let mut skipped = Vec::new(); for (source_domain, partner_uid) in &entry.ids { - let reason = if partner_uid.uid.is_empty() { - "empty_uid" - } else { + let reason = if partner_uid.has_live_uid(now) { match registry.get(source_domain) { None => "not_in_registry", Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", Some(_) => continue, } + } else if partner_uid.is_provider_written() { + "expired" + } else { + "empty_uid" }; skipped.push(SkippedPartnerId { source_domain: source_domain.clone(), @@ -682,9 +686,10 @@ mod tests { use crate::ec::kv::TombstoneOutcome; use super::*; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -778,15 +783,11 @@ mod tests { let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); entry.ids.insert( "disabled.example".to_owned(), - KvPartnerId { - uid: "uid-disabled".to_owned(), - }, + KvPartnerId::single("uid-disabled".to_owned()), ); entry.ids.insert( "unknown.example".to_owned(), - KvPartnerId { - uid: "uid-unknown".to_owned(), - }, + KvPartnerId::single("uid-unknown".to_owned()), ); entry } @@ -1533,4 +1534,55 @@ mod tests { assert_eq!(matched[0]["source_domain"], "sharedid.org"); assert_eq!(matched[0]["uid"], "shared-uid-123"); } + + #[test] + fn auction_view_matches_live_auction_for_provider_records() { + let registry = PartnerRegistry::from_config(&[ + make_test_partner("expired.example", true), + make_test_partner("live.example", true), + ]) + .expect("should build registry"); + let provider_record = |id: &str, expiry: Expiry| { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(0), + ) + .expect("should build record") + }; + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert( + "expired.example".to_owned(), + provider_record("old", Expiry::At(UnixSeconds::new(1))), + ); + entry.ids.insert( + "live.example".to_owned(), + provider_record("new", Expiry::Never), + ); + + let view = build_auction_view(®istry, &entry); + + let view_json = serde_json::to_value(&view.eids).expect("should serialize view eids"); + let live: Vec = entry + .live_eids(®istry, UnixSeconds::new(current_timestamp())) + .map(Eid::from) + .collect(); + assert_eq!( + view_json, + serde_json::to_value(&live).expect("should serialize live eids"), + "should show exactly what the live auction resolves" + ); + assert_eq!(view.eids.len(), 1, "should omit the expired provider UID"); + assert_eq!(view.eids[0].uids[0].id, "new", "should keep the live UID"); + assert_eq!(view.skipped.len(), 1, "should report the expired record"); + assert_eq!(view.skipped[0].source_domain, "expired.example"); + assert_eq!(view.skipped[0].reason, "expired", "should report expiry"); + } } diff --git a/crates/trusted-server-core/src/ec/eid_types.rs b/crates/trusted-server-core/src/ec/eid_types.rs new file mode 100644 index 0000000000..9b1a647f59 --- /dev/null +++ b/crates/trusted-server-core/src/ec/eid_types.rs @@ -0,0 +1,594 @@ +//! Value types for full-fidelity EIDs stored in the EC identity graph. +//! +//! Provider output is converted into these types at the trust boundary +//! (coordinator validation); past that point an invalid UID, agent type or +//! match method cannot be represented. + +use core::fmt; +use std::sync::Arc; + +use serde::de::{self, Deserializer, Visitor}; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::value::RawValue; + +use super::kv_types::MAX_UID_LENGTH; +use super::partner::normalize_partner_source_domain; + +/// Maximum serialized size of an `ext` object, in bytes. +pub const MAX_EXT_BYTES: usize = 1024; + +/// Maximum size of `inserter` and `matcher`, in bytes. +pub const MAX_EID_FIELD_BYTES: usize = 255; + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + /// The UID is blank. + #[display("UID is empty")] + EmptyUid, + /// The UID exceeds [`MAX_UID_LENGTH`]. + #[display("UID is too long")] + UidTooLong, + /// The agent type is negative. + #[display("agent type is negative")] + NegativeAgentType, + /// The match method is not an `OpenRTB` 2.6 value. + #[display("match method is unknown")] + UnknownMatchMethod, + /// An `ext` is not a JSON object within [`MAX_EXT_BYTES`]. + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + /// An `inserter` or `matcher` exceeds [`MAX_EID_FIELD_BYTES`]. + #[display("EID field is too long")] + FieldTooLong, + /// The source is not a valid domain or not claimed by the provider. + #[display("source is not claimed by this provider")] + UnclaimedSource, + /// The expiry is not after the request time. + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// Unix time in whole seconds. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + derive_more::Display, +)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +impl UnixSeconds { + /// Wraps a Unix timestamp in seconds. + #[must_use] + pub const fn new(seconds: u64) -> Self { + Self(seconds) + } + + /// Returns the timestamp in seconds. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } + + /// Adds `seconds`, saturating at `u64::MAX`. + #[must_use] + pub const fn saturating_add_secs(self, seconds: u64) -> Self { + Self(self.0.saturating_add(seconds)) + } +} + +struct ArcStrVisitor; + +impl<'de> Visitor<'de> for ArcStrVisitor { + type Value = Arc; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, value: &str) -> Result { + Ok(Arc::from(value)) + } +} + +/// Operator-chosen identifier of a configured provider instance. +/// +/// Clones share one allocation. Serialized as a plain string. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +impl ProviderId { + /// Creates a provider ID from configuration. + #[must_use] + pub fn new(id: &str) -> Self { + Self(Arc::from(id)) + } + + /// Returns the ID as a string slice. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Serialize for ProviderId { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ProviderId { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_str(ArcStrVisitor).map(Self) + } +} + +/// Normalized EID source domain. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +impl SourceDomain { + /// Returns the normalized domain. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom<&str> for SourceDomain { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + normalize_partner_source_domain(value) + .map(|normalized| Self(Arc::from(normalized))) + .map_err(|_| EidValidationError::UnclaimedSource) + } +} + +impl Serialize for SourceDomain { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SourceDomain { + fn deserialize>(deserializer: D) -> Result { + let raw = deserializer.deserialize_str(ArcStrVisitor)?; + Self::try_from(raw.as_ref()).map_err(de::Error::custom) + } +} + +/// A non-blank UID of at most [`MAX_UID_LENGTH`] bytes, kept verbatim. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl UidValue { + /// Returns the UID. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + fn check(value: &str) -> Result<(), EidValidationError> { + if value.trim().is_empty() { + return Err(EidValidationError::EmptyUid); + } + if value.len() > MAX_UID_LENGTH { + return Err(EidValidationError::UidTooLong); + } + Ok(()) + } +} + +impl TryFrom<&str> for UidValue { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + Self::check(value)?; + Ok(Self(Box::from(value))) + } +} + +impl TryFrom for UidValue { + type Error = EidValidationError; + + fn try_from(value: String) -> Result { + Self::check(&value)?; + Ok(Self(value.into_boxed_str())) + } +} + +/// Non-negative `OpenRTB` agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub struct AgentType(i32); + +impl AgentType { + /// Returns the raw agent type. + #[must_use] + pub const fn get(self) -> i32 { + self.0 + } +} + +impl TryFrom for AgentType { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + if value < 0 { + return Err(EidValidationError::NegativeAgentType); + } + Ok(Self(value)) + } +} + +impl From for i32 { + fn from(value: AgentType) -> Self { + value.0 + } +} + +/// `OpenRTB` 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub enum MatchMethod { + /// `0`: unknown. + Unknown, + /// `1`: no matching. + NoMatching, + /// `2`: cookie sync. + CookieSync, + /// `3`: authenticated. + Authenticated, + /// `4`: observed. + Observed, + /// `5`: inference. + Inference, +} + +impl TryFrom for MatchMethod { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + match value { + 0 => Ok(Self::Unknown), + 1 => Ok(Self::NoMatching), + 2 => Ok(Self::CookieSync), + 3 => Ok(Self::Authenticated), + 4 => Ok(Self::Observed), + 5 => Ok(Self::Inference), + _ => Err(EidValidationError::UnknownMatchMethod), + } + } +} + +impl From for i32 { + fn from(value: MatchMethod) -> Self { + match value { + MatchMethod::Unknown => 0, + MatchMethod::NoMatching => 1, + MatchMethod::CookieSync => 2, + MatchMethod::Authenticated => 3, + MatchMethod::Observed => 4, + MatchMethod::Inference => 5, + } + } +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(expires_at) => now < expires_at, + } + } + + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} + +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +pub mod optional_exp { + use serde::{Deserialize as _, Deserializer, Serialize as _, Serializer}; + + use super::{Expiry, UnixSeconds}; + + /// Serializes `At` as its seconds; `Never` is skipped by the caller. + /// + /// # Errors + /// + /// Returns the serializer's error. + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(expires_at) => expires_at.serialize(serializer), + Expiry::Never => serializer.serialize_none(), + } + } + + /// Reads an optional number; absent or `null` means [`Expiry::Never`]. + /// + /// # Errors + /// + /// Returns the deserializer's error for a non-numeric value. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} + +/// An opaque JSON object of at most [`MAX_EXT_BYTES`], compared by bytes. +/// +/// Deserializing applies the same checks as [`RawJson::try_from`], so a +/// stored row cannot carry an oversized or non-object `ext`. +#[derive(Debug, Clone)] +pub struct RawJson(Box); + +impl RawJson { + /// Returns the raw JSON text. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.get() + } + + /// Parses the raw JSON into a [`serde_json::Value`] for `OpenRTB` output. + /// + /// # Panics + /// + /// Never in practice: the bytes were validated as JSON when stored. + #[must_use] + pub fn to_value(&self) -> serde_json::Value { + serde_json::from_str(self.0.get()).expect("should parse stored raw JSON") + } +} + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +impl RawJson { + fn check(value: &RawValue) -> Result<(), EidValidationError> { + let text = value.get(); + if text.len() > MAX_EXT_BYTES || !text.trim_start().starts_with('{') { + return Err(EidValidationError::InvalidExt); + } + Ok(()) + } +} + +impl TryFrom<&RawValue> for RawJson { + type Error = EidValidationError; + + fn try_from(value: &RawValue) -> Result { + Self::check(value)?; + Ok(Self(value.to_owned())) + } +} + +impl TryFrom> for RawJson { + type Error = EidValidationError; + + fn try_from(value: Box) -> Result { + Self::check(&value)?; + Ok(Self(value)) + } +} + +impl Serialize for RawJson { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for RawJson { + fn deserialize>(deserializer: D) -> Result { + let raw = Box::::deserialize(deserializer)?; + Self::try_from(raw).map_err(de::Error::custom) + } +} + +/// EID-level fields shared by all UIDs of one stored entry. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct EidFields { + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option>, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option>, + /// `OpenRTB` `mm`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// `OpenRTB` EID `ext`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn uid_value_rejects_blank_and_oversized() { + assert!(UidValue::try_from(" ").is_err(), "should reject blank UID"); + let long = "a".repeat(MAX_UID_LENGTH + 1); + assert!( + UidValue::try_from(long.as_str()).is_err(), + "should reject oversized UID" + ); + let uid = UidValue::try_from("ID-123").expect("should accept valid UID"); + assert_eq!(uid.as_str(), "ID-123", "should keep UID verbatim"); + } + + #[test] + fn agent_type_rejects_negative() { + assert!( + AgentType::try_from(-1).is_err(), + "should reject negative atype" + ); + assert_eq!( + AgentType::try_from(571_187) + .expect("should accept vendor atype") + .get(), + 571_187, + "should keep vendor atype" + ); + } + + #[test] + fn match_method_round_trips_openrtb_values() { + for value in 0..=5 { + let method = MatchMethod::try_from(value).expect("should accept OpenRTB mm"); + assert_eq!(i32::from(method), value, "should round-trip mm {value}"); + } + assert!( + MatchMethod::try_from(6).is_err(), + "should reject unknown mm" + ); + } + + #[test] + fn source_domain_normalizes() { + let source = SourceDomain::try_from("IDS.Example.COM").expect("should accept domain"); + assert_eq!( + source.as_str(), + "ids.example.com", + "should lowercase source domain" + ); + } + + #[test] + fn provider_id_round_trips_as_string() { + let id = ProviderId::new("example-provider"); + let encoded = serde_json::to_string(&id).expect("should serialize provider id"); + assert_eq!( + encoded, "\"example-provider\"", + "should serialize as plain string" + ); + let decoded: ProviderId = + serde_json::from_str(&encoded).expect("should deserialize provider id"); + assert_eq!(decoded, id, "should round-trip provider id"); + } + + #[test] + fn expiry_liveness() { + let now = UnixSeconds::new(100); + assert!( + Expiry::Never.is_live(now), + "should serve UIDs without expiry" + ); + assert!( + Expiry::At(UnixSeconds::new(101)).is_live(now), + "should serve before expiry" + ); + assert!( + !Expiry::At(UnixSeconds::new(100)).is_live(now), + "should not serve at expiry" + ); + } + + #[test] + fn raw_json_accepts_small_objects_only() { + let object = + serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode object"); + let raw = RawJson::try_from(object.as_ref()).expect("should accept object"); + assert_eq!(raw.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + let array = serde_json::value::to_raw_value(&json!([1])).expect("should encode array"); + assert!( + RawJson::try_from(array.as_ref()).is_err(), + "should reject non-object" + ); + let big = serde_json::value::to_raw_value(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + RawJson::try_from(big.as_ref()).is_err(), + "should reject oversized ext" + ); + } + + #[test] + fn raw_json_compares_by_bytes() { + let a = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + let b = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + assert_eq!( + RawJson::try_from(a.as_ref()).expect("should accept"), + RawJson::try_from(b.as_ref()).expect("should accept"), + "should compare equal bytes as equal" + ); + } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct WithExpiry { + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, + } + + #[test] + fn optional_exp_omits_never_and_reads_missing() { + let never = serde_json::to_value(WithExpiry { + expiry: Expiry::Never, + }) + .expect("should serialize"); + assert_eq!(never, json!({}), "should omit exp for Never"); + let at: WithExpiry = + serde_json::from_value(json!({ "exp": 42 })).expect("should deserialize exp"); + assert_eq!( + at.expiry, + Expiry::At(UnixSeconds::new(42)), + "should read exp" + ); + let missing: WithExpiry = + serde_json::from_value(json!({})).expect("should deserialize empty"); + assert_eq!(missing.expiry, Expiry::Never, "should default to Never"); + } + + #[test] + fn raw_json_deserialize_applies_same_checks() { + let ok: RawJson = serde_json::from_str(r#"{"k":1}"#).expect("should accept small object"); + assert_eq!(ok.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + assert!( + serde_json::from_str::("[1]").is_err(), + "should reject non-object on deserialize" + ); + let big = serde_json::to_string(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + serde_json::from_str::(&big).is_err(), + "should reject oversized ext on deserialize" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 1359f75091..904594c934 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -10,7 +10,8 @@ use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; -use super::kv_types::KvEntry; +use super::eid_types::{RawJson, UnixSeconds}; +use super::kv_types::{KvEid, KvEntry, KvPartnerId}; use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. @@ -64,17 +65,120 @@ pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec Vec { resolved .iter() - .map(|item| Eid { - source: item.source_domain.clone(), - uids: vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], + .map(|item| { + Eid::new( + item.source_domain.clone(), + vec![Uid { + id: item.uid.clone(), + atype: Some(item.openrtb_atype), + ext: None, + }], + ) }) .collect() } +/// A borrowed EID entry: a single legacy UID or a full-fidelity provider entry. +#[derive(Debug, Clone, Copy)] +pub enum EidView<'row> { + /// A record holding one UID and no EID-level fields. + Single { + /// Source domain. + source: &'row str, + /// The UID. + uid: &'row str, + /// The partner's configured agent type. + atype: i32, + }, + /// A provider-written entry; only live UIDs are emitted. + Full { + /// Source domain. + source: &'row str, + /// The stored entry. + eid: &'row KvEid, + /// Request time used to filter UIDs. + now: UnixSeconds, + }, +} + +impl From> for Eid { + fn from(view: EidView<'_>) -> Self { + match view { + EidView::Single { source, uid, atype } => Eid::new( + source, + vec![Uid { + id: uid.to_owned(), + atype: Some(atype), + ext: None, + }], + ), + EidView::Full { source, eid, now } => { + let fields = eid.fields(); + let uids = eid + .uids() + .iter() + .filter(|uid| uid.is_live(now)) + .map(|uid| Uid { + id: uid.id().as_str().to_owned(), + atype: Some(uid.atype().get()), + ext: uid.ext().map(RawJson::to_value), + }) + .collect(); + Eid { + inserter: fields.inserter.as_deref().map(str::to_owned), + matcher: fields.matcher.as_deref().map(str::to_owned), + mm: fields.mm.map(i32::from), + ext: fields.ext.as_ref().map(RawJson::to_value), + ..Eid::new(source, uids) + } + } + } + } +} + +/// Yields the live views of one stored record. +/// +/// Legacy records yield one [`EidView::Single`]; provider records yield one +/// [`EidView::Full`] per entry that still has a live UID. +pub fn source_views<'row>( + source: &'row str, + record: &'row KvPartnerId, + default_atype: i32, + now: UnixSeconds, +) -> impl Iterator> + 'row { + let single = + (!record.is_provider_written() && !record.uid.is_empty()).then_some(EidView::Single { + source, + uid: record.uid.as_str(), + atype: default_atype, + }); + let full = record + .eids() + .iter() + .filter(move |eid| eid.uids().iter().any(|uid| uid.is_live(now))) + .map(move |eid| EidView::Full { source, eid, now }); + single.into_iter().chain(full) +} + +impl KvEntry { + /// Iterates live entries for registered, `bidstream_enabled` sources in + /// source order, borrowing from the row. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + now: UnixSeconds, + ) -> impl Iterator> + 'row { + self.ids.iter().flat_map(move |(source, record)| { + let partner = registry + .get(source) + .filter(|partner| partner.bidstream_enabled); + partner + .into_iter() + .flat_map(move |partner| source_views(source, record, partner.openrtb_atype, now)) + }) + } +} + /// Builds a base64-encoded EID header value, truncating if needed. /// /// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds @@ -145,6 +249,8 @@ pub fn encode_eids_header(eids: &[Eid]) -> Result<(String, bool), Report PartnerRegistry { + let mut partner = make_test_partner(source); + partner.bidstream_enabled = bidstream; + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn provider_record(ids: &[(&str, Expiry)]) -> KvPartnerId { + let uids = ids + .iter() + .map(|(id, expiry)| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + *expiry, + ) + }) + .collect(); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, uids)], UnixSeconds::new(0)) + .expect("should build record") + } + + #[test] + fn live_eids_skip_expired_uids_and_keep_eid_fields() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[ + ("old", Expiry::At(UnixSeconds::new(10))), + ("new", Expiry::Never), + ]), + ); + + let eids: Vec = entry + .live_eids(®istry, UnixSeconds::new(20)) + .map(Eid::from) + .collect(); + + assert_eq!(eids.len(), 1, "should emit one entry"); + assert_eq!( + eids[0].inserter.as_deref(), + Some("inserter.example.com"), + "should keep inserter" + ); + let ids: Vec<&str> = eids[0].uids.iter().map(|uid| uid.id.as_str()).collect(); + assert_eq!(ids, vec!["new"], "should drop expired uid"); + } + + #[test] + fn live_eids_respect_bidstream_flag() { + let registry = registry_with("ids.example.com", false); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + assert_eq!( + entry.live_eids(®istry, UnixSeconds::new(0)).count(), + 0, + "should hide non-bidstream source" + ); + } + + #[test] + fn live_eids_emit_legacy_single_uid() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + let eids: Vec = entry + .live_eids(®istry, UnixSeconds::new(0)) + .map(Eid::from) + .collect(); + assert_eq!(eids[0].uids[0].id, "v", "should emit legacy uid"); + assert_eq!(eids[0].uids[0].atype, Some(3), "should use partner atype"); + } } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 019ef2aa40..ce4c831602 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1115,14 +1115,14 @@ mod tests { ); ec_context.set_recovery_eligible(true); ec_context.set_eid_sync_source(EidSyncSource::Navigation); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1517,22 +1517,22 @@ mod tests { .expect("should serialize test cookie payload"), ); ec_context.set_client_eids(vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_from_body".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_from_body".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]); let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1595,14 +1595,14 @@ mod tests { ec_id: ec_id.clone(), }); ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -2315,9 +2315,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "partner-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("partner-uid".to_owned()), ); ec_context.set_kv_snapshot(EcKvSnapshot::Present { ec_id, diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 94464d1a57..26a6e195e4 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -12,13 +12,15 @@ use url::Url; use super::auth::authenticate_bearer; use super::consent::ec_consent_granted; use crate::error::TrustedServerError; -use crate::openrtb::{Eid, Uid}; +use crate::openrtb::Eid; use crate::settings::Settings; -use super::EcContext; +use super::eid_types::UnixSeconds; +use super::eids::source_views; use super::kv::KvIdentityGraph; use super::log_id; use super::registry::PartnerRegistry; +use super::{EcContext, current_timestamp}; /// Handles `GET /_ts/api/v1/identify`. /// @@ -84,7 +86,7 @@ pub fn handle_identify( }; let mut degraded = false; - let mut uid: Option = None; + let mut eid: Option = None; let mut cluster_size: Option = None; match kv.get(ec_id) { @@ -95,12 +97,16 @@ pub fn handle_identify( // write back with the live-entry TTL. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { - // Extract only this partner's UID. - if let Some(partner_uid) = entry.ids.get(&partner.source_domain) - && !partner_uid.uid.is_empty() - { - uid = Some(partner_uid.uid.clone()); - } + // Extract only this partner's first live entry. + let now = UnixSeconds::new(current_timestamp()); + eid = entry + .ids + .get(&partner.source_domain) + .and_then(|record| { + source_views(&partner.source_domain, record, partner.openrtb_atype, now) + .next() + }) + .map(Eid::from); // Evaluate cluster size lazily for identify responses. Existing // stored cluster_size values are reused without a prefix-list call. @@ -124,14 +130,10 @@ pub fn handle_identify( } } - let eid = uid.as_ref().map(|u| Eid { - source: partner.source_domain.clone(), - uids: vec![Uid { - id: u.clone(), - atype: Some(partner.openrtb_atype), - ext: None, - }], - }); + let uid = eid + .as_ref() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id.clone()); let body = IdentifyResponse { ec: ec_id.to_owned(), @@ -334,6 +336,8 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -756,4 +760,77 @@ mod tests { "should vary on identity request inputs for preflight" ); } + + fn provider_record(id: &str, expiry: Expiry) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, vec![uid])], UnixSeconds::new(0)) + .expect("should build record") + } + + fn identify_body_for_record(record: KvPartnerId) -> serde_json::Value { + let settings = create_test_settings(); + let kv = KvIdentityGraph::in_memory("identify_store"); + let partners = vec![make_test_partner("ssp.example.com", VALID_API_TOKEN)]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); + entry.ids.insert("ssp.example.com".to_owned(), record); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + let req = Request::builder() + .method("GET") + .uri("https://edge.test-publisher.com/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + + let response = handle_identify(&settings, &kv, ®istry, &req, &ec_context) + .expect("should construct identify response"); + + assert_eq!(response.status(), StatusCode::OK, "should return 200"); + serde_json::from_slice::( + &response.into_body().into_bytes().unwrap_or_default(), + ) + .expect("should decode identify response JSON") + } + + #[test] + fn handle_identify_omits_expired_provider_uid() { + let body = + identify_body_for_record(provider_record("old", Expiry::At(UnixSeconds::new(1)))); + + assert!( + body.get("uid").is_none(), + "should omit expired provider uid" + ); + assert!( + body.get("eid").is_none(), + "should omit eid without a live uid" + ); + } + + #[test] + fn handle_identify_returns_live_provider_eid_with_eid_fields() { + let body = identify_body_for_record(provider_record("live", Expiry::Never)); + + assert_eq!(body["uid"], "live", "should return live provider uid"); + assert_eq!( + body["eid"]["inserter"], "inserter.example.com", + "should keep provider EID-level fields" + ); + assert_eq!( + body["eid"]["uids"][0]["atype"], 1, + "should use the provider atype" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0eb83d3fa..9daaac243c 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,7 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; -use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; +use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -157,9 +157,7 @@ fn apply_cookie_partner_id_updates( entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); changed = true; } @@ -205,9 +203,7 @@ pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerId entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); changed = true; } @@ -284,6 +280,15 @@ impl KvIdentityGraph { store_name: store_name.to_owned(), message: "Failed to serialize KV entry body".to_owned(), })?; + if body.len() > MAX_ENTRY_BYTES { + return Err(Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Refusing to write KV entry of {} bytes (limit {MAX_ENTRY_BYTES})", + body.len() + ), + })); + } let meta = KvMetadata::from_entry(entry); let meta_str = serde_json::to_string(&meta).change_context(TrustedServerError::KvStore { @@ -920,9 +925,7 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -992,9 +995,7 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -1618,7 +1619,9 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_types::{KvEid, KvUid}; /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition /// and deletes the key, simulating a concurrent withdrawal that removes the @@ -1737,9 +1740,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + ), ); let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); @@ -1772,9 +1775,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + ), ); let err = KvIdentityGraph::serialize_entry(&entry, "test-store") @@ -1786,6 +1789,48 @@ mod tests { ); } + #[test] + fn create_refuses_oversized_provider_entry_before_write() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let uids = (0..40) + .map(|index| { + let id = format!("{index:03}{}", "x".repeat(480)); + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + let record = crate::ec::kv_types::KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), uids)], + UnixSeconds::new(0), + ) + .expect("should build record"); + let mut entry = live_entry(); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!( + entry.validate().is_ok(), + "should be a valid entry apart from its size" + ); + + let err = graph + .create(&ec_id, &entry) + .expect_err("should refuse oversized entry"); + + assert!( + format!("{err}").contains("Refusing to write KV entry"), + "should report the size limit" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "should not write the oversized entry" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; @@ -1796,9 +1841,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "concurrent.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), ); entry } @@ -2131,9 +2174,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-1".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-1".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; @@ -2159,9 +2200,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "old-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("old-uid".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; @@ -2191,9 +2230,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "original".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("original".to_owned()), ); let updates = vec![ PartnerIdUpdate::new("ssp_x", "intermediate"), @@ -2215,9 +2252,7 @@ mod tests { let mut concurrent = live_entry(); concurrent.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "desired-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("desired-uid".to_owned()), ); let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -2333,9 +2368,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "latest-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("latest-uid".to_owned()), ); let updates = [ PartnerIdUpdate::new("ssp_x", "older-uid"), @@ -2354,9 +2387,7 @@ mod tests { let mut concurrent = live_entry(); concurrent.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), ); let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -2439,9 +2470,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), ); graph.create(&ec_id, &entry).expect("should seed entry"); @@ -2466,9 +2495,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), ); graph.create(&ec_id, &entry).expect("should seed entry"); diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 589f280c11..2a638dc219 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -10,6 +10,10 @@ use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; +use super::eid_types::{ + AgentType, EidFields, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, UidValue, + UnixSeconds, optional_exp, +}; use crate::consent::ConsentContext; use crate::geo::GeoInfo; @@ -97,11 +101,233 @@ pub struct KvGeo { pub dma: Option, } -/// A partner user ID within a KV entry. +/// Maximum serialized size of a KV entry body written by a page load. +pub const MAX_ENTRY_BYTES: usize = 16 * 1024; + +/// A partner ID record within a KV entry, keyed by source domain. +/// +/// `uid` is always present so older readers keep working. Records written by +/// an identity provider also carry full-fidelity `eids`; new code reads +/// `eids` whenever it is non-empty and treats `uid` as derived. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvPartnerId { - /// The partner's user identifier. + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. pub uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +impl KvPartnerId { + /// Builds a single-UID record (browser ingest, batch sync, pull sync). + #[must_use] + pub fn single(uid: impl Into) -> Self { + Self { + uid: uid.into(), + eids: Vec::new(), + } + } + + /// Builds a provider record, keeping only entries with live UIDs. + /// + /// Returns `None` when no live UID remains. + #[must_use] + pub fn from_eids(eids: Vec, now: UnixSeconds) -> Option { + let mut record = Self { + uid: String::new(), + eids, + }; + record.prune_expired(now); + (!record.eids.is_empty()).then_some(record) + } + + /// Full-fidelity provider entries; empty for single-UID records. + #[must_use] + pub fn eids(&self) -> &[KvEid] { + &self.eids + } + + /// Whether this record was written by an identity provider. + #[must_use] + pub fn is_provider_written(&self) -> bool { + !self.eids.is_empty() + } + + /// Whether the record holds a UID that may be served at `now`. + #[must_use] + pub fn has_live_uid(&self, now: UnixSeconds) -> bool { + if self.eids.is_empty() { + return !self.uid.is_empty(); + } + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.is_live(now)) + } + + /// Drops expired UIDs and empty entries, then resyncs `uid`. + /// + /// Returns whether anything changed. Single-UID records never change. + pub fn prune_expired(&mut self, now: UnixSeconds) -> bool { + if self.eids.is_empty() && !self.uid.is_empty() { + return false; + } + let before = self.clone(); + for eid in &mut self.eids { + eid.uids.retain(|uid| uid.is_live(now)); + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + *self != before + } +} + +/// One stored EID entry with its EID-level fields. +/// +/// Stored flat (`{"inserter": .., "uids": [..]}`). Serialization flattens +/// [`EidFields`]; deserialization goes through a flat private representation +/// because serde's `flatten` cannot deserialize the `RawValue` behind `ext`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(from = "KvEidRepr")] +pub struct KvEid { + #[serde(flatten)] + fields: EidFields, + uids: Vec, +} + +/// Flat stored shape of [`KvEid`], used only to deserialize it. +#[derive(Deserialize)] +struct KvEidRepr { + #[serde(default)] + inserter: Option>, + #[serde(default)] + matcher: Option>, + #[serde(default)] + mm: Option, + #[serde(default)] + ext: Option, + uids: Vec, +} + +impl From for KvEid { + fn from(repr: KvEidRepr) -> Self { + let KvEidRepr { + inserter, + matcher, + mm, + ext, + uids, + } = repr; + Self { + fields: EidFields { + inserter, + matcher, + mm, + ext, + }, + uids, + } + } +} + +impl KvEid { + /// Builds an entry from validated parts. + #[must_use] + pub fn new(fields: EidFields, uids: Vec) -> Self { + Self { fields, uids } + } + + /// EID-level fields. + #[must_use] + pub fn fields(&self) -> &EidFields { + &self.fields + } + + /// Stored UIDs, live or not. + #[must_use] + pub fn uids(&self) -> &[KvUid] { + &self.uids + } +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, +} + +impl KvUid { + /// Builds a UID from validated parts. + #[must_use] + pub fn new( + id: UidValue, + atype: AgentType, + ext: Option, + owner: Option, + expiry: Expiry, + ) -> Self { + Self { + id, + atype, + ext, + owner, + expiry, + } + } + + /// The UID value. + #[must_use] + pub fn id(&self) -> &UidValue { + &self.id + } + + /// The agent type. + #[must_use] + pub fn atype(&self) -> AgentType { + self.atype + } + + /// The UID-level `ext`. + #[must_use] + pub fn ext(&self) -> Option<&RawJson> { + self.ext.as_ref() + } + + /// The provider that wrote this UID. + #[must_use] + pub fn owner(&self) -> Option<&ProviderId> { + self.owner.as_ref() + } + + /// When the UID stops being served. + #[must_use] + pub fn expiry(&self) -> Expiry { + self.expiry + } + + /// Whether the UID may be served at `now`. + #[must_use] + pub fn is_live(&self, now: UnixSeconds) -> bool { + self.expiry.is_live(now) + } } /// Publisher property metadata captured when an EC entry is created. @@ -303,12 +529,7 @@ impl KvEntry { #[must_use] pub fn minimal(partner_id: &str, uid: &str, now: u64) -> Self { let mut ids = BTreeMap::new(); - ids.insert( - partner_id.to_owned(), - KvPartnerId { - uid: uid.to_owned(), - }, - ); + ids.insert(partner_id.to_owned(), KvPartnerId::single(uid.to_owned())); Self { v: SCHEMA_VERSION, created: now, @@ -387,6 +608,23 @@ impl KvEntry { partner_uid.uid.len() )); } + let oversized_field = partner_uid.eids().iter().any(|eid| { + let fields = eid.fields(); + [fields.inserter.as_deref(), fields.matcher.as_deref()] + .into_iter() + .flatten() + .any(|field| field.len() > MAX_EID_FIELD_BYTES) + }); + if oversized_field { + return Err(format!( + "partner ID '{partner_id}' has an EID inserter or matcher over MAX_EID_FIELD_BYTES" + )); + } + if partner_uid.eids().iter().any(|eid| eid.uids().is_empty()) { + return Err(format!( + "partner ID '{partner_id}' has an EID entry with no UIDs" + )); + } } if let Some(pub_properties) = &self.pub_properties { @@ -463,6 +701,9 @@ impl KvGeo { #[cfg(test)] mod tests { use super::*; + use crate::ec::eid_types::{ + AgentType, EidFields, Expiry, MatchMethod, ProviderId, RawJson, UidValue, UnixSeconds, + }; fn sample_consent_context() -> ConsentContext { ConsentContext { @@ -492,9 +733,7 @@ mod tests { let mut entry = KvEntry::new(&consent, Some(&geo), 1_741_824_000, "example.com"); entry.ids.insert( "liveramp".to_owned(), - KvPartnerId { - uid: "LR_xyz".to_owned(), - }, + KvPartnerId::single("LR_xyz".to_owned()), ); let json = serde_json::to_string(&entry).expect("should serialize KvEntry"); @@ -737,9 +976,7 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - KvPartnerId { - uid: "x".repeat(MAX_UID_LENGTH + 1), - }, + KvPartnerId::single("x".repeat(MAX_UID_LENGTH + 1)), ); let err = entry @@ -1206,4 +1443,154 @@ mod tests { "known_browser should default to None" ); } + + fn provider_uid(id: &str, expiry: Expiry) -> KvUid { + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ) + } + + #[test] + fn legacy_partner_id_round_trips_without_eids() { + let legacy: KvPartnerId = serde_json::from_value(serde_json::json!({ "uid": "abc" })) + .expect("should read legacy record"); + assert_eq!(legacy, KvPartnerId::single("abc"), "should read legacy uid"); + assert_eq!( + serde_json::to_value(&legacy).expect("should serialize"), + serde_json::json!({ "uid": "abc" }), + "should not add eids to legacy records" + ); + } + + #[test] + fn from_eids_sets_uid_to_first_live_uid() { + let now = UnixSeconds::new(100); + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("expired", Expiry::At(UnixSeconds::new(50))), + provider_uid("live", Expiry::Never), + ], + ); + let record = KvPartnerId::from_eids(vec![eid], now).expect("should keep live uid"); + assert_eq!(record.uid, "live", "should mirror first live uid into uid"); + assert!(record.is_provider_written(), "should mark provider record"); + } + + #[test] + fn prune_expired_drops_dead_uids_and_resyncs_uid() { + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("first", Expiry::At(UnixSeconds::new(150))), + provider_uid("second", Expiry::Never), + ], + ); + let mut record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(100)).expect("should build"); + assert_eq!(record.uid, "first", "should start on first uid"); + + let changed = record.prune_expired(UnixSeconds::new(200)); + + assert!(changed, "should report pruning"); + assert_eq!(record.eids()[0].uids().len(), 1, "should drop expired uid"); + assert_eq!(record.uid, "second", "should resync uid"); + } + + #[test] + fn provider_record_round_trips_with_owner_and_exp() { + let eid = KvEid::new( + EidFields::default(), + vec![provider_uid("v", Expiry::At(UnixSeconds::new(500)))], + ); + let record = KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build"); + let json = serde_json::to_value(&record).expect("should serialize"); + assert_eq!( + json, + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "owner": "example-provider", "exp": 500 }] }] }), + "should store owner and exp" + ); + let back: KvPartnerId = serde_json::from_value(json).expect("should deserialize"); + assert_eq!(back, record, "should round-trip"); + } + + #[test] + fn validate_rejects_empty_provider_eid() { + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [{ "uids": [] }] })) + .expect("should parse"); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!(entry.validate().is_err(), "should reject EID with no UIDs"); + } + + #[test] + fn provider_record_round_trips_eid_level_fields() { + let ext = serde_json::value::to_raw_value(&serde_json::json!({ "k": 1 })) + .expect("should encode ext"); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + matcher: Some("matcher.example.com".into()), + mm: Some(MatchMethod::Authenticated), + ext: Some(RawJson::try_from(ext.as_ref()).expect("should accept ext")), + }; + let eid = KvEid::new(fields, vec![provider_uid("v", Expiry::Never)]); + let record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build record"); + + let json = serde_json::to_string(&record).expect("should serialize"); + let back: KvPartnerId = serde_json::from_str(&json).expect("should deserialize"); + + assert_eq!( + serde_json::from_str::(&json).expect("should parse"), + serde_json::json!({ "uid": "v", "eids": [{ + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 3, + "ext": { "k": 1 }, + "uids": [{ "id": "v", "atype": 1, "owner": "example-provider" }] + }] }), + "should store EID-level fields flat beside uids" + ); + assert_eq!(back, record, "should round-trip EID-level fields"); + } + + #[test] + fn provider_record_rejects_invalid_stored_ext() { + let big = "x".repeat(crate::ec::eid_types::MAX_EXT_BYTES); + let cases = [ + serde_json::json!({ "uid": "v", "eids": [{ "ext": [1], "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "ext": { "k": big }, "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": { "k": big } }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": "text" }] }] }), + ]; + + for case in cases { + let text = serde_json::to_string(&case).expect("should encode case"); + assert!( + serde_json::from_str::(&text).is_err(), + "should reject invalid stored ext: {text}" + ); + } + } + + #[test] + fn validate_rejects_oversized_inserter_and_matcher() { + let long = "a".repeat(crate::ec::eid_types::MAX_EID_FIELD_BYTES + 1); + for field in ["inserter", "matcher"] { + let mut eid = serde_json::json!({ "uids": [{ "id": "v", "atype": 1 }] }); + eid[field] = serde_json::Value::String(long.clone()); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [eid] })) + .expect("should parse record"); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry.ids.insert("ids.example.com".to_owned(), record); + + assert!(entry.validate().is_err(), "should reject oversized {field}"); + } + } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f24c489edd..e815c43a19 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -26,6 +26,7 @@ //! - [`rate_limiter`] — Rate limiting abstraction (implemented by adapters) //! - [`identify`] — Identity read endpoint (`GET /_ts/api/v1/identify`) //! - [`eids`] — Shared EID resolution and formatting helpers +//! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes @@ -36,6 +37,7 @@ pub mod batch_sync; pub mod consent; pub mod cookies; pub mod device; +pub mod eid_types; pub mod eids; pub mod finalize; pub mod generation; diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index d823cb32fb..3e2fbee219 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -294,10 +294,7 @@ fn structured_cookie_eids_to_openrtb(entries: Vec) -> Vec) -> Vec { entries .into_iter() .filter(|entry| !entry.source.is_empty() && !entry.id.is_empty()) - .map(|entry| Eid { - source: entry.source, - uids: vec![Uid { - id: entry.id, - atype: (entry.atype >= 0).then_some(entry.atype), - ext: None, - }], + .map(|entry| { + Eid::new( + entry.source, + vec![Uid { + id: entry.id, + atype: (entry.atype >= 0).then_some(entry.atype), + ext: None, + }], + ) }) .collect() } @@ -636,22 +635,22 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_stale", "atype": 1}]} ])); let body_eids = vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_fresh".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]; let updates = collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 191e6c6d24..784c217e51 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -652,9 +652,7 @@ mod tests { if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid".to_owned()), ); } ec_context.set_kv_snapshot(snapshot); @@ -702,9 +700,7 @@ mod tests { entry.ids.insert( "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-b".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-b".to_owned()), ); assert!( @@ -1213,9 +1209,7 @@ mod tests { entry.consent.ok = true; entry.ids.insert( "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "already-known".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("already-known".to_owned()), ); graph .create(&ec_id, &entry) diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 11df47296f..7b3c853074 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -375,9 +375,7 @@ mod tests { for domain in domains { entry.ids.insert( (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: format!("uid-{domain}"), - }, + crate::ec::kv_types::KvPartnerId::single(format!("uid-{domain}")), ); } EcKvSnapshot::Present { @@ -510,9 +508,7 @@ mod tests { let mut tombstone = KvEntry::tombstone(1_000); tombstone.ids.insert( "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "stale".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("stale".to_owned()), ); let snapshots = [ live_snapshot(EC_ID, &[]), diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c13260704f..33018ca519 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -7015,30 +7015,30 @@ external_bundle_sri = "sha384-AAAA" let provider = PrebidAuctionProvider::new(base_config()); let mut auction_request = create_test_auction_request(); auction_request.user.eids = Some(vec![ - crate::openrtb::Eid { - source: "liveramp.com".to_owned(), - uids: vec![crate::openrtb::Uid { + crate::openrtb::Eid::new( + "liveramp.com".to_owned(), + vec![crate::openrtb::Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, - crate::openrtb::Eid { - source: "id5-sync.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "id5-sync.com".to_owned(), + vec![crate::openrtb::Uid { id: "ID5_abc".to_owned(), atype: Some(1), ext: None, }], - }, - crate::openrtb::Eid { - source: "google.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "google.com".to_owned(), + vec![crate::openrtb::Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }, + ), ]); let settings = make_settings(); diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 67cd7f6950..645031a7b0 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -70,10 +70,37 @@ pub struct ConsentedProvidersSettings { pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// `OpenRTB` `mm` (match method). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// EID-level extension data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, /// One or more user IDs from this provider. pub uids: Vec, } +impl Eid { + /// Builds an entry with no EID-level fields. + #[must_use] + pub fn new(source: impl Into, uids: Vec) -> Self { + Self { + source: source.into(), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } +} + /// A single user identifier within an [`Eid`] entry. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -405,16 +432,46 @@ mod tests { ); } + #[test] + fn eid_serializes_eid_level_fields_only_when_present() { + let plain = Eid::new("ids.example.com", vec![]); + assert_eq!( + serde_json::to_value(&plain).expect("should serialize"), + serde_json::json!({ "source": "ids.example.com", "uids": [] }), + "should omit absent EID-level fields" + ); + let full = Eid { + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(3), + ext: Some(serde_json::json!({ "k": 1 })), + ..Eid::new("ids.example.com", vec![]) + }; + let value = serde_json::to_value(&full).expect("should serialize"); + assert_eq!( + value["inserter"], + serde_json::json!("inserter.example.com"), + "should write inserter" + ); + assert_eq!(value["mm"], serde_json::json!(3), "should write mm"); + let back: Eid = serde_json::from_value(value).expect("should accept EID-level fields"); + assert_eq!( + back.matcher.as_deref(), + Some("matcher.example.com"), + "should read matcher" + ); + } + #[test] fn eid_serializes_correctly() { - let eid = Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5-abc123".to_owned(), atype: Some(1), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); assert_eq!(serialized["source"], "id5-sync.com", "source should match"); @@ -431,14 +488,14 @@ mod tests { #[test] fn eid_serializes_vendor_specific_atype() { - let eid = Eid { - source: "google.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "google.com".to_owned(), + vec![Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); From 00fb3b652411fbc55adc9e45ba78cdc4e2f4acb1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 30 Oct 2026 07:33:40 +0530 Subject: [PATCH 02/58] Make every browser-driven EC write a single page-load write Top-level document navigations on the publisher route receive a single-use write permit; finalize plans one row change (browser ingest, provider results, prune, size cap) and commits it once without retry. Non-navigation requests and redirects never write, integration proxies no longer mint ECs, and new ECs are created by the page-load write instead of at generation. Withdrawal tombstones only the active EC and keeps the EC cookie when the tombstone does not land; the withdrawal completion marker is the one deliberate extra write. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 76 + .../src/auction/endpoints.rs | 32 +- .../src/ec/coordinator/mod.rs | 3 + .../src/ec/coordinator/results.rs | 147 + crates/trusted-server-core/src/ec/finalize.rs | 1204 +++++---- crates/trusted-server-core/src/ec/kv.rs | 2372 +++-------------- crates/trusted-server-core/src/ec/kv_types.rs | 15 + crates/trusted-server-core/src/ec/mod.rs | 277 +- .../trusted-server-core/src/ec/page_load.rs | 320 +++ .../trusted-server-core/src/ec/prebid_eids.rs | 5 +- .../trusted-server-core/src/ec/pull_sync.rs | 1 + crates/trusted-server-core/src/ec/registry.rs | 27 + .../trusted-server-core/src/ec/write_plan.rs | 354 +++ .../src/integrations/registry.rs | 22 +- crates/trusted-server-core/src/publisher.rs | 3 + 15 files changed, 2154 insertions(+), 2704 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/coordinator/mod.rs create mode 100644 crates/trusted-server-core/src/ec/coordinator/results.rs create mode 100644 crates/trusted-server-core/src/ec/page_load.rs create mode 100644 crates/trusted-server-core/src/ec/write_plan.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ce06ce7e30..7f75267188 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -114,6 +114,7 @@ use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; @@ -850,6 +851,11 @@ async fn dispatch_fallback( let is_navigation = is_navigation_request(&req); if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); + // The page-load write permit uses the stricter Fetch Metadata test + // (spec §3.1); requests without it never write KV. + if let Some(permit) = NavigationPermit::classify(&req) { + ec.ec_context.set_navigation_permit(permit); + } } let is_publisher_navigation = ec.is_real_browser && is_navigation; if is_publisher_navigation @@ -2578,6 +2584,76 @@ mod tests { } } + fn navigation_request(path: &str) -> Request { + let mut request = browser_request(Method::GET, path, "document"); + request + .headers_mut() + .insert("sec-fetch-mode", HeaderValue::from_static("navigate")); + request + } + + fn take_finalize_state(response: &mut Response) -> super::EcFinalizeState { + response + .extensions_mut() + .remove::() + .expect("should attach finalize state") + } + + #[test] + fn publisher_navigation_receives_navigation_permit() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_some(), + "should issue a permit" + ); + } + + #[test] + fn publisher_request_without_fetch_metadata_navigation_gets_no_permit() { + let router = test_router(); + let mut prefetch = navigation_request("/article"); + prefetch + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for request in [ + browser_request(Method::GET, "/article", "document"), + prefetch, + ] { + let mut response = route(&router, request); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit without a top-level navigation" + ); + } + } + + #[test] + fn integration_proxy_never_generates_ec() { + let router = test_router(); + + let mut response = route( + &router, + navigation_request("/integrations/prebid/bundle.js"), + ); + + let mut state = take_finalize_state(&mut response); + assert!( + !state.ec_context.ec_generated(), + "should not mint an EC on an integration proxy" + ); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit to an integration proxy" + ); + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index b0121f7490..b261add7b3 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -789,10 +789,9 @@ mod tests { #[tokio::test] async fn auction_endpoint_snapshot_is_reused_by_response_finalization() { // `/auction` loads the identity-graph row to resolve server-side EIDs. - // Finalization runs afterwards on the same EC context and ingests - // `ts-eids`/`sharedId` updates. Both must be served by a single billable - // read: before the snapshot was shared, finalization saw `NotRead` and - // paid for a second lookup. + // Finalization runs afterwards on the same EC context. It must not pay + // for a second lookup, and it never writes KV: only the page load's + // navigation may write (spec D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -876,21 +875,17 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "the sharedId update must still be ingested from the shared snapshot" + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[tokio::test] - async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { - // Regression test for #1184: `/auction` sends every EID it has in the - // request body, but response finalization used to ingest identity - // graph updates only from the `ts-eids` cookie — which the browser - // caps in size and may not have sent at all. `handle_auction` must - // hand its parsed body EIDs to `ec_context` so finalization ingests - // them regardless of what the cookie carried. + async fn auction_body_eids_reach_context_but_never_kv() { + // `handle_auction` hands its parsed body EIDs to `ec_context`, but body + // EIDs go to bidders on that request only: finalization of an + // `/auction` request never writes them to KV (spec §5.1, D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -969,10 +964,9 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the body's EID must be ingested into KV without a ts-eids cookie" + assert!( + stored.ids.is_empty(), + "the body's EIDs must not be persisted from an auction request" ); } diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs new file mode 100644 index 0000000000..7fe8da9015 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -0,0 +1,3 @@ +//! Server-side identity lookup coordination. + +pub mod results; diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs new file mode 100644 index 0000000000..e9bc32a2d1 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -0,0 +1,147 @@ +//! Per-provider outcomes of one navigation's lookups. + +use crate::ec::eid_types::{EidValidationError, ProviderId, SourceDomain}; +use crate::ec::kv_types::KvEid; + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + /// Blank or oversized UIDs, or negative agent types. + pub invalid_uid: u16, + /// Invalid `ext`, `inserter`, `matcher` or `mm`. + pub invalid_field: u16, + /// Entries for sources the provider does not claim. + pub unclaimed_source: u16, + /// UIDs whose expiry is not after the request time. + pub expired: u16, + /// Entries or UIDs beyond the provider's shape limits. + pub over_limit: u16, + /// Duplicate UIDs within one source. + pub duplicate: u16, +} + +impl DropCounts { + /// Counts one validation failure. + pub fn record(&mut self, reason: EidValidationError) { + let counter = match reason { + EidValidationError::EmptyUid + | EidValidationError::UidTooLong + | EidValidationError::NegativeAgentType => &mut self.invalid_uid, + EidValidationError::UnknownMatchMethod + | EidValidationError::InvalidExt + | EidValidationError::FieldTooLong => &mut self.invalid_field, + EidValidationError::UnclaimedSource => &mut self.unclaimed_source, + EidValidationError::Expired => &mut self.expired, + }; + *counter = counter.saturating_add(1); + } + + /// Total drops. + #[must_use] + pub fn total(&self) -> u16 { + self.invalid_uid + .saturating_add(self.invalid_field) + .saturating_add(self.unclaimed_source) + .saturating_add(self.expired) + .saturating_add(self.over_limit) + .saturating_add(self.duplicate) + } +} + +/// Why a provider was not called. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + /// The provider is disabled. + Disabled, + /// Consent does not permit this provider. + Consent, + /// None of its sources is due. + NotDue, + /// Every due source needs input V1 never has (hashed email). + Input, + /// The built URL is not `https` or not allowlisted. + Destination, + /// The provider's hourly rate limit is reached. + RateLimited, +} + +/// One outcome per provider per navigation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + /// Valid entries were returned. + Found { + /// UIDs kept after validation. + kept: u16, + /// UIDs or entries dropped by validation. + dropped: DropCounts, + }, + /// The provider knows no ID for this visitor. + NoMatch, + /// No response before the shared deadline. + Timeout, + /// A non-success HTTP status. + HttpError(http::StatusCode), + /// The request could not be sent or the body exceeded the limit. + TransportFailed, + /// The body could not be parsed at all. + InvalidResponse, + /// The provider was not called. + Skipped(SkipReason), +} + +impl LookupStatus { + /// Metric name for logs (spec §12.2). + #[must_use] + pub fn metric_name(&self) -> &'static str { + match self { + Self::Found { .. } => "found", + Self::NoMatch => "no_match", + Self::Timeout => "timeout", + Self::HttpError(_) => "http_error", + Self::TransportFailed => "transport_error", + Self::InvalidResponse => "invalid_response", + Self::Skipped(SkipReason::Disabled) => "skipped_disabled", + Self::Skipped(SkipReason::Consent) => "skipped_consent", + Self::Skipped(SkipReason::NotDue) => "skipped_not_due", + Self::Skipped(SkipReason::Input) => "skipped_input", + Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::Destination) => "destination_rejected", + } + } +} + +/// One provider's outcome and validated entries. +#[derive(Debug, Clone)] +pub struct ProviderLookup { + /// The provider. + pub provider: ProviderId, + /// What happened. + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `Found`. + pub entries: Vec<(SourceDomain, Vec)>, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Clone, Default)] +pub struct LookupResults { + lookups: Vec, +} + +impl LookupResults { + /// Wraps per-provider results in provider-config order. + #[must_use] + pub fn new(lookups: Vec) -> Self { + Self { lookups } + } + + /// Iterates results in provider-config order. + pub fn iter(&self) -> impl Iterator { + self.lookups.iter() + } + + /// Whether no provider produced a result. + #[must_use] + pub fn is_empty(&self) -> bool { + self.lookups.is_empty() + } +} diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index ce4c831602..eba972a8a6 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1,9 +1,9 @@ //! EC response finalization. //! //! Centralizes post-routing EC behavior so all handlers get consistent cookie -//! and KV semantics. - -use std::collections::HashSet; +//! and KV semantics. Every browser-driven KV write goes through the +//! navigation's single page-load write ([`KvIdentityGraph::commit_page_load`]), +//! so a request without a navigation permit never writes. use edgezero_core::body::Body as EdgeBody; use http::Response; @@ -14,16 +14,16 @@ use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; -use super::generation::{generate_ec_id, is_valid_ec_id}; -use super::kv::{ - CreateIfAbsentOutcome, EidCookieSyncOutcome, KvIdentityGraph, PartnerIdUpdate, - apply_partner_id_updates, -}; +use super::eid_types::UnixSeconds; +use super::generation::generate_ec_id; +use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::page_load::{Generation, PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, EidSyncSource, current_timestamp, log_id}; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -35,12 +35,18 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// Finalizes EC response behavior for all routes. /// -/// Applies withdrawal handling, last-seen updates, cookie reconciliation, -/// Prebid EID ingestion, and cookie writes for new EC generation. +/// Applies withdrawal handling, browser EID ingestion, provider lookup +/// results, orphan recovery and cookie writes for new EC generation. +/// +/// KV is written at most once, and only when the request carries a +/// navigation permit (see [`super::page_load`]) and the final response is not +/// a redirect. Without a permit a newly generated EC is not issued and a +/// withdrawal only strips EC headers: the EC cookie is kept so the next +/// navigation can write the tombstone. /// -/// On consent withdrawal, the browser response clears the EC cookie -/// immediately and the EC identity-graph KV tombstone is the authoritative -/// revocation marker. There is no separate consent KV store to clean up. +/// On consent withdrawal during a navigation, the EC identity-graph KV +/// tombstone is the authoritative revocation marker and the EC cookie is +/// expired once the tombstone is in place (or the row is proven absent). /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted /// from the request *before* routing consumes it. @@ -54,6 +60,9 @@ pub fn ec_finalize_response( response: &mut Response, ) { ec_context.validate_pull_sync_marker(settings, registry); + let permit = ec_context + .take_navigation_permit() + .and_then(|permit| permit.confirm(response.status())); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -69,159 +78,175 @@ pub fn ec_finalize_response( ec_context, kv, registry, + permit, consent_withdrawn, response, ); return; } - // Returning user: consent is granted and EC came from request. - if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { - if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { - let source = ec_context.eid_sync_source(); - let collected = source - .map(|_| { - collect_eid_updates( - eids_cookie, - sharedid_cookie, - ec_context.client_eids(), - registry, - ) - }) - .unwrap_or_default(); - let had_eid_updates = !collected.is_empty(); - let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); - // `sync_eid_cookie_updates` early-returns without refreshing the - // snapshot when `updates` is empty, which is correct when there - // was never anything to write (an unconfigured registry, no sync - // source, or no EID cookies/body this request — the case - // `finalize_not_read_snapshot_does_not_rotate` covers). But when - // consent gating (e.g. TCF Purpose 4 denial) is what emptied - // `updates`, the request *did* have EID data, and orphan recovery - // below still needs an actual `Missing` read to detect an - // orphaned cookie. An unread `NotRead` snapshot proves nothing, so - // consent gating must not also skip that one read on a - // recovery-eligible request. - if updates.is_empty() - && had_eid_updates - && ec_context.recovery_eligible() - && matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) - { - let snapshot = graph.load_snapshot(&ec_id); - ec_context.set_kv_snapshot(snapshot); - } else if let Some(source) = source { - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, source); - } - if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) - && ec_context.recovery_eligible() - { - confirm_then_recover_orphaned_ec( - settings, ec_context, graph, &ec_id, &updates, response, - ); - } - } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); - - // Ordinary returning-user page views no longer refresh the browser - // cookie, emit the EC header, or update KV TTL. - return; - } - - // Newly generated EC in this request. Do not emit a generated EC when - // there is no KV graph: that would mint a browser cookie with no backing - // identity-graph row, producing a phantom ID on later requests. - if ec_context.ec_generated() { - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { - log::info!("Skipping generated EC response write because KV graph is unavailable"); - reconcile_pull_sync_marker(settings, registry, ec_context, response); - return; - }; - - let updates = gate_eid_updates_by_consent( + if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { + let collected = if ec_context.eid_sync_source().is_some() || ec_context.ec_generated() { collect_eid_updates( eids_cookie, sharedid_cookie, ec_context.client_eids(), registry, - ), - ec_context.consent(), - ); - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); - if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { - set_ec_cookie_on_response(settings, ec_context, response); + ) } else { - log::warn!("Skipping generated EC cookie because backing row is not authoritative"); + Vec::new() + }; + let had_eid_updates = !collected.is_empty(); + let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); + let commit = PageLoadCommit { + settings, + graph, + registry, + ec_id: &ec_id, + updates: &updates, + had_eid_updates, + }; + if ec_context.ec_generated() { + commit_generated(&commit, ec_context, permit, response); + } else if ec_context.ec_was_present() { + commit_returning(&commit, ec_context, permit, response); } + } else if ec_context.ec_generated() { + // Do not emit a generated EC when there is no KV graph: that would + // mint a browser cookie with no backing identity-graph row. + log::info!("Skipping generated EC response write because KV graph is unavailable"); } reconcile_pull_sync_marker(settings, registry, ec_context, response); } -fn sync_eid_cookie_updates( - graph: &KvIdentityGraph, - ec_context: &mut EcContext, - ec_id: &str, - updates: &[PartnerIdUpdate], - source: EidSyncSource, -) { - if updates.is_empty() { - return; +/// Shared inputs for the navigation's single write. +struct PageLoadCommit<'a> { + settings: &'a Settings, + graph: &'a KvIdentityGraph, + registry: &'a PartnerRegistry, + ec_id: &'a str, + /// Browser ingest, already consent-gated. + updates: &'a [PartnerIdUpdate], + /// Whether the request carried EID data before consent gating. + had_eid_updates: bool, +} + +impl PageLoadCommit<'_> { + /// Plans the row change for `ec_id` against `snapshot`. + fn plan( + &self, + ec_context: &EcContext, + ec_id: &str, + snapshot: &EcKvSnapshot, + new_entry: Option<&KvEntry>, + ) -> Option { + plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry, + browser_ingest: self.updates, + lookups: ec_context.lookup_results(), + registry: self.registry, + now: UnixSeconds::new(current_timestamp()), + }) } +} - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - ec_id, - updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); - record_eid_sync_terminal(source, outcome); +fn present_snapshot(ec_id: &str, entry: KvEntry) -> EcKvSnapshot { + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: None, + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct EidSyncMeasurement { - source: EidSyncSource, - outcome: EidCookieSyncOutcome, - already_matched: u8, - written: u8, - conflict_duplicate: u8, - deferred: u8, +/// Creates the row of a newly generated EC and issues its cookie on success. +fn commit_generated( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) { + let Some(new_entry) = ec_context.take_pending_entry() else { + log::warn!("Skipping generated EC cookie because no pending row was built"); + return; + }; + let Some(permit) = permit else { + log::info!("Skipping generated EC: not a page-load navigation"); + return; + }; + let Some(change) = commit.plan( + ec_context, + commit.ec_id, + ec_context.kv_snapshot(), + Some(&new_entry), + ) else { + log::warn!("Skipping generated EC: no row change was planned"); + return; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + set_ec_cookie_on_response(commit.settings, ec_context, response); + } + Err(report) => { + log::warn!( + "Page-load write failed for new EC '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + } + } } -impl EidSyncMeasurement { - fn new(source: EidSyncSource, outcome: EidCookieSyncOutcome) -> Self { - Self { - source, - outcome, - already_matched: u8::from(matches!(outcome, EidCookieSyncOutcome::AlreadyMatched)), - written: u8::from(matches!( - outcome, - EidCookieSyncOutcome::Written | EidCookieSyncOutcome::WrittenWithDeferredFreshness - )), - conflict_duplicate: u8::from(matches!(outcome, EidCookieSyncOutcome::ConflictMatched)), - deferred: u8::from(matches!( - outcome, - EidCookieSyncOutcome::WrittenWithDeferredFreshness - | EidCookieSyncOutcome::DeferredConflict - | EidCookieSyncOutcome::DeferredFreshness - | EidCookieSyncOutcome::DeferredStaleRead - )), +/// Applies a returning EC's page-load write, or recovers an orphaned EC. +fn commit_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) { + let Some(permit) = permit else { + return; + }; + // An unread snapshot is read only when this navigation has something to + // write, or when consent gating emptied real EID data on a request that + // may still need orphan recovery. A navigation with nothing to do costs + // no KV operation. + let has_changes = !commit.updates.is_empty() || !ec_context.lookup_results().is_empty(); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) + && (has_changes || (commit.had_eid_updates && ec_context.recovery_eligible())) + { + let snapshot = commit.graph.load_snapshot(commit.ec_id); + ec_context.set_kv_snapshot(snapshot); + } + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) { + if ec_context.recovery_eligible() { + confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); } + return; } + update_returning(commit, ec_context, permit); } -fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome) { - let measurement = EidSyncMeasurement::new(source, outcome); - log::info!( - "EID sync measurement: source={} outcome={} attempted=1 already_matched={} written={} \ - conflict_duplicate={} deferred={}", - measurement.source, - measurement.outcome, - measurement.already_matched, - measurement.written, - measurement.conflict_duplicate, - measurement.deferred, - ); +/// Writes the planned update for a returning EC whose row snapshot is held. +fn update_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: PageLoadWrite, +) { + let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { + return; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)), + Err(report) => log::info!( + "Page-load write skipped for '{}': {}", + log_id(commit.ec_id), + report.current_context() + ), + } } fn reconcile_pull_sync_marker( @@ -252,12 +277,11 @@ fn reconcile_pull_sync_marker( /// the outbound `/auction` bid request, evaluated here through /// [`allows_eid_persistence`]. Without this, a user who denies Purpose 4 /// would have their EIDs correctly stripped from the bid request but still -/// written to the identity graph from the `ts-eids`/`sharedId` cookies or the -/// `/auction` request body. +/// written to the identity graph from the `ts-eids`/`sharedId` cookies. fn gate_eid_updates_by_consent( - updates: Vec, + updates: Vec, consent: &ConsentContext, -) -> Vec { +) -> Vec { if updates.is_empty() || allows_eid_persistence(consent) { return updates; } @@ -268,75 +292,60 @@ fn gate_eid_updates_by_consent( Vec::new() } +/// Replaces a proven-orphaned EC with a new ID whose row is created by this +/// navigation's single write. One attempt: a collision leaves the orphan in +/// place for a later navigation. fn recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, ) { - // Snapshot the orphaned ID once so every fail-closed exit binds the failed - // snapshot to the same key. - let Some(orphan_id) = ec_context.ec_value().map(str::to_owned) else { - return; - }; + let orphan_id = commit.ec_id.to_owned(); let Some(client_ip) = ec_context.client_ip().map(str::to_owned) else { log::warn!("Orphan EC recovery skipped because client IP is unavailable"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); return; }; + let new_id = match generate_ec_id(commit.settings, &client_ip) { + Ok(ec_id) => ec_id, + Err(err) => { + log::warn!("Orphan EC recovery ID generation failed: {err:?}"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); + return; + } + }; + let mut new_entry = KvEntry::new( + ec_context.consent(), + ec_context.geo_info(), + current_timestamp(), + &commit.settings.publisher.domain, + ); + new_entry.device = ec_context + .device_signals() + .map(super::device::DeviceSignals::to_kv_device); - const MAX_RECOVERY_ATTEMPTS: usize = 5; - for _attempt in 0..MAX_RECOVERY_ATTEMPTS { - let ec_id = match generate_ec_id(settings, &client_ip) { - Ok(ec_id) => ec_id, - Err(err) => { - log::warn!("Orphan EC recovery ID generation failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } - }; - let mut entry = KvEntry::new( - ec_context.consent(), - ec_context.geo_info(), - current_timestamp(), - &settings.publisher.domain, - ); - entry.device = ec_context - .device_signals() - .map(super::device::DeviceSignals::to_kv_device); - apply_partner_id_updates(&mut entry, updates); - - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - ec_context.replace_with_generated(ec_id, snapshot); - set_ec_cookie_on_response(settings, ec_context, response); - return; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => continue, - Err(err) => { - log::warn!("Orphan EC recovery failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } + let missing = EcKvSnapshot::Missing { + ec_id: new_id.clone(), + }; + let Some(change) = commit.plan(ec_context, &new_id, &missing, Some(&new_entry)) else { + log::warn!("Orphan EC recovery skipped: no row change was planned"); + return; + }; + match commit.graph.commit_page_load(permit, &new_id, change) { + Ok(committed) => { + let snapshot = present_snapshot(&new_id, committed); + ec_context.replace_with_generated(new_id, snapshot); + set_ec_cookie_on_response(commit.settings, ec_context, response); + } + Err(report) => { + log::warn!( + "Orphan EC recovery write failed: {}", + report.current_context() + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); } } - - log::warn!("Orphan EC recovery exhausted collision retries"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); } /// Proves an orphaned cookie is genuinely absent before rotating it. @@ -348,8 +357,8 @@ fn recover_orphaned_ec( /// POP that has not converged. Two point reads do not fix that — both can be /// stale — so absence has to be *proved*, not observed twice: /// -/// - a row that became visible after the origin round trip is adopted, with any -/// pending updates merged, and is never rotated; +/// - a row that became visible after the origin round trip is adopted, and the +/// navigation's single write is planned against it; it is never rotated; /// - a second miss is escalated to /// [`key_exists_confirmed`](KvIdentityGraph::key_exists_confirmed), which /// reads the primary data source. Only a proven-absent key rotates; @@ -358,23 +367,22 @@ fn recover_orphaned_ec( /// - neither a read failure nor a failed existence check is a miss, and neither /// rotates. fn confirm_then_recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - ec_id: &str, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, ) { - let confirmed = graph.load_snapshot(ec_id); + let confirmed = commit.graph.load_snapshot(commit.ec_id); match confirmed { EcKvSnapshot::Present { .. } => { // The row became visible after the origin round trip: adopt it and - // merge any pending updates rather than rotating a valid identity. + // plan the navigation's write against it rather than rotating a + // valid identity. ec_context.set_kv_snapshot(confirmed); - sync_eid_cookie_updates(graph, ec_context, ec_id, updates, EidSyncSource::Navigation); + update_returning(commit, ec_context, permit); } - EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { - Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), + EcKvSnapshot::Missing { .. } => match commit.graph.key_exists_confirmed(commit.ec_id) { + Ok(false) => recover_orphaned_ec(commit, ec_context, permit, response), Ok(true) => { log::warn!( "Orphan EC recovery skipped: both point reads missed a row the store still \ @@ -439,15 +447,18 @@ pub fn clear_ec_on_response(settings: &Settings, response: &mut Response, registry: &PartnerRegistry, + permit: Option, consent_withdrawn: bool, response: &mut Response, ) { @@ -456,69 +467,111 @@ fn finalize_unusable_consent( if !(consent_withdrawn && ec_context.cookie_was_present()) { return; } - - expire_ec_cookie(settings, response); - - // Compute once for the authoritative identity-graph tombstones. - let ids_to_withdraw = withdrawal_ec_ids(ec_context); - - // The identity-graph tombstone is the authoritative withdrawal marker - // for subsequent EC behavior. - if let Some(graph) = kv { - apply_withdrawal_tombstones(&ids_to_withdraw, |ec_id| { - // The graph hands back the post-withdrawal snapshot rather than - // leaving the caller to rebuild it, so post-send work that reads - // the context — pull sync discloses the raw EC ID to partners — - // sees the tombstone that was just written. Only the active ID has - // a snapshot in the context to correct. - let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { - ec_context.kv_snapshot().clone() - } else { - EcKvSnapshot::NotRead - }; - let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); - // The browser cookie is already cleared, so a failed tombstone - // leaves a live row that server-side consumers still read as - // consented. Report every failure, including the non-active cookie - // ID whose outcome is not retained on the request context. - if matches!(outcome, EcKvSnapshot::Failed { .. }) { - log::warn!( - "EC withdrawal tombstone failed for '{}': the identity-graph row may \ - still be live with consent granted", - log_id(ec_id) - ); - } - if ec_context.ec_value() == Some(ec_id) { - ec_context.set_kv_snapshot(outcome); - } - }); + let Some(permit) = permit else { + // Not a page-load navigation: serve nothing, keep the EC cookie so the + // next navigation can write the tombstone (spec §3.1). + return; + }; + let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { + // No identity graph (or no valid active EC) holds a row to tombstone, + // so the cookie is the only identity state left to clear. + expire_ec_cookie(settings, response); + return; + }; + if withdraw_active_ec(graph, ec_context, &ec_id, permit) { + expire_ec_cookie(settings, response); } } -fn withdrawal_ec_ids(ec_context: &EcContext) -> HashSet { - let mut hashes = HashSet::new(); - - if let Some(cookie_ec_id) = ec_context.existing_cookie_ec_id() - && is_valid_ec_id(cookie_ec_id) - { - hashes.insert(cookie_ec_id.to_owned()); +/// Tombstones the active EC's row with the navigation's single write. +/// +/// Returns whether the withdrawal is complete: the row is now a tombstone or +/// is proven absent. A snapshot that is not a usable live row with a CAS +/// generation is re-read once; a miss is confirmed against the primary data +/// source so a stale read never clears the cookie of a live row. +fn withdraw_active_ec( + graph: &KvIdentityGraph, + ec_context: &mut EcContext, + ec_id: &str, + permit: PageLoadWrite, +) -> bool { + let usable = match ec_context.kv_snapshot().entry_for(ec_id) { + Some(entry) => { + !entry.consent.ok || Generation::of(ec_context.kv_snapshot(), ec_id).is_some() + } + None => false, + }; + if !usable { + let snapshot = graph.load_snapshot(ec_id); + ec_context.set_kv_snapshot(snapshot); } - if let Some(active_ec_id) = ec_context.ec_value() - && is_valid_ec_id(active_ec_id) - { - hashes.insert(active_ec_id.to_owned()); + let snapshot = ec_context.kv_snapshot(); + if let Some(entry) = snapshot.entry_for(ec_id) { + if !entry.consent.ok { + return true; + } + let Some(expected) = Generation::of(snapshot, ec_id) else { + log::warn!( + "EC withdrawal tombstone for '{}' skipped (no_generation); keeping the cookie", + log_id(ec_id) + ); + return false; + }; + return match graph.commit_page_load(permit, ec_id, RowChange::Tombstone { expected }) { + Ok(tombstone) => { + ec_context.set_kv_snapshot(present_snapshot(ec_id, tombstone)); + true + } + Err(report) => { + log::warn!( + "EC withdrawal tombstone failed for '{}': {}; keeping the cookie", + log_id(ec_id), + report.current_context() + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }); + false + } + }; } - - hashes -} - -fn apply_withdrawal_tombstones(ec_ids: &HashSet, mut write_tombstone: F) -where - F: FnMut(&str), -{ - for ec_id in ec_ids { - write_tombstone(ec_id); + if !matches!(snapshot, EcKvSnapshot::Missing { .. }) { + return false; + } + // A still-valid completion marker proves an earlier withdrawal finished, + // so the root existence check can be skipped. + match graph.withdrawal_marker_exists(ec_id, checked_current_timestamp()) { + Ok(true) => return true, + Ok(false) => {} + Err(err) => log::warn!( + "EC withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ), + } + match graph.key_exists_confirmed(ec_id) { + Ok(false) => true, + Ok(true) => { + log::warn!( + "EC withdrawal for '{}': point read missed a row the store still lists; \ + keeping the cookie for a later navigation", + log_id(ec_id) + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }); + false + } + Err(err) => { + log::warn!( + "EC withdrawal for '{}': existence check failed: {err:?}", + log_id(ec_id) + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }); + false + } } } @@ -531,6 +584,8 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::EidSyncSource; + use crate::ec::page_load::NavigationPermit; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -620,90 +675,6 @@ mod tests { } } - #[test] - fn withdrawal_ec_ids_returns_cookie_ec_only_when_active_missing() { - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context(None, Some(&cookie_ec), true, false, Jurisdiction::Unknown); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should include exactly one EC ID"); - assert!( - ids.contains(&cookie_ec), - "should include the cookie EC value" - ); - } - - #[test] - fn withdrawal_ec_ids_deduplicates_matching_cookie_and_active_ec() { - let ec_id = sample_ec_id("same01"); - let ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should deduplicate identical EC IDs"); - assert!(ids.contains(&ec_id), "should retain the shared EC ID"); - } - - #[test] - fn withdrawal_ec_ids_includes_both_cookie_and_active_when_different() { - let active_ec = sample_ec_id("activ1"); - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context( - Some(&active_ec), - Some(&cookie_ec), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 2, "should include both distinct EC IDs"); - assert!(ids.contains(&active_ec), "should include active EC ID"); - assert!(ids.contains(&cookie_ec), "should include cookie EC ID"); - } - - #[test] - fn withdrawal_ec_ids_filters_invalid_values() { - let valid_ec = sample_ec_id("valid1"); - let ec_context = make_context( - Some(&valid_ec), - Some("not-an-ec-id"), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should ignore malformed EC values"); - assert!(ids.contains(&valid_ec), "should keep the valid EC ID"); - } - - #[test] - fn apply_withdrawal_tombstones_invokes_writer_for_each_ec_id() { - let first = sample_ec_id("first1"); - let second = sample_ec_id("second"); - let mut ids = HashSet::new(); - ids.insert(first.clone()); - ids.insert(second.clone()); - - let mut written = Vec::new(); - apply_withdrawal_tombstones(&ids, |ec_id| written.push(ec_id.to_owned())); - written.sort(); - - let mut expected = vec![first, second]; - expected.sort(); - assert_eq!(written, expected, "should write a tombstone for each EC ID"); - } - #[test] fn clear_ec_on_response_removes_headers_and_expires_cookie() { let settings = create_test_settings(); @@ -755,6 +726,7 @@ mod tests { let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); let kv = KvIdentityGraph::in_memory("test-store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -807,6 +779,7 @@ mod tests { ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. } )); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -837,10 +810,10 @@ mod tests { } #[test] - fn withdrawal_still_expires_the_cookie_when_the_store_is_unavailable() { - // Cookie expiry is the primary enforcement, so it has to survive a - // store that cannot answer at all — the case where the identity-graph - // marker is exactly what goes missing. + fn withdrawal_keeps_the_cookie_when_the_tombstone_write_fails() { + // A failed tombstone write is not retried. The EC cookie is kept so the + // next navigation can still write the tombstone; clearing it would + // orphan a live row (spec §9.2). let settings = create_test_settings(); let ec_id = sample_ec_id("dead01"); let consent = ConsentContext { @@ -856,6 +829,7 @@ mod tests { entry: Box::new(live_entry()), generation: Some(1), }); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let kv = KvIdentityGraph::failing("test-store"); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); @@ -871,10 +845,9 @@ mod tests { &mut response, ); - let set_cookie = get_header_str(&response, "set-cookie").unwrap_or_default(); assert!( - set_cookie.contains("Max-Age=0"), - "should expire the EC cookie even when the store is unavailable: {set_cookie}" + get_header(&response, "set-cookie").is_none(), + "should keep the EC cookie when the tombstone write fails" ); assert!( get_header(&response, "x-ts-ec").is_none(), @@ -898,6 +871,7 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); set_header(&mut response, "x-ts-eids", "[]"); @@ -1065,6 +1039,7 @@ mod tests { ec_id: orphaned_ec.clone(), }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1126,6 +1101,7 @@ mod tests { let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1181,6 +1157,7 @@ mod tests { partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1248,10 +1225,22 @@ mod tests { } #[test] - fn finalize_navigation_routes_persist_returning_user_eid_updates() { - for (source, suffix, cookie_id) in [ - (EidSyncSource::Navigation, "naveid", "navigation-cookie-id"), - (EidSyncSource::PageBids, "spaeid", "page-bids-cookie-id"), + fn finalize_persists_returning_user_eid_updates_only_on_navigation() { + // `/_ts/page-bids` belongs to a page load whose navigation already + // made the write, so it never receives a permit (spec D1). + for (source, suffix, cookie_id, permit) in [ + ( + EidSyncSource::Navigation, + "naveid", + "navigation-cookie-id", + true, + ), + ( + EidSyncSource::PageBids, + "spaeid", + "page-bids-cookie-id", + false, + ), ] { let settings = create_test_settings(); let ec_id = sample_ec_id(suffix); @@ -1265,7 +1254,9 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + // A recovery-eligible context is a navigation and carries a permit. + let mut ec_context = + returning_user_context(&ec_id, graph.load_snapshot(&ec_id), permit); ec_context.set_eid_sync_source(source); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1287,8 +1278,8 @@ mod tests { .expect("row should remain"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some(cookie_id), - "{source} should persist the returning-user EID cookie" + permit.then_some(cookie_id), + "{source} should persist the returning-user EID cookie only with a permit" ); } } @@ -1296,20 +1287,17 @@ mod tests { #[test] fn finalize_generated_ec_persists_eid_updates() { let settings = create_test_settings(); - let ec_id = sample_ec_id("geneid"); let graph = KvIdentityGraph::in_memory("test_store"); - let live = KvEntry::new( - &granting_consent(), + let mut ec_context = EcContext::new_for_test_with_ip( None, - current_timestamp(), - &settings.publisher.domain, + granting_consent(), + Some("192.0.2.10".to_owned()), ); - graph - .create(&ec_id, &live) - .expect("should seed generated row"); - let mut ec_context = - make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); - ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1327,114 +1315,18 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should create the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("generated-cookie-id") - ); - } - - #[test] - fn eid_sync_measurement_dimensions_are_bounded_and_identity_free() { - let sources = [ - EidSyncSource::Navigation, - EidSyncSource::Auction, - EidSyncSource::PageBids, - EidSyncSource::NewEc, - ]; - let outcomes = [ - EidCookieSyncOutcome::AlreadyMatched, - EidCookieSyncOutcome::Written, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - EidCookieSyncOutcome::ConflictMatched, - EidCookieSyncOutcome::DeferredConflict, - EidCookieSyncOutcome::DeferredFreshness, - EidCookieSyncOutcome::DeferredStaleRead, - EidCookieSyncOutcome::Missing, - EidCookieSyncOutcome::ConsentWithdrawn, - EidCookieSyncOutcome::Failed, - ]; - - assert_eq!( - sources.map(|source| source.to_string()), - ["navigation", "auction", "page_bids", "new_ec"] - ); - assert_eq!( - outcomes.map(|outcome| outcome.to_string()), - [ - "already_matched", - "written", - "written_with_deferred_freshness", - "conflict_matched", - "deferred_conflict", - "deferred_freshness", - "deferred_stale_read", - "missing", - "consent_withdrawn", - "failed", - ] - ); - - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Navigation, - EidCookieSyncOutcome::AlreadyMatched, - ), - EidSyncMeasurement { - source: EidSyncSource::Navigation, - outcome: EidCookieSyncOutcome::AlreadyMatched, - already_matched: 1, - written: 0, - conflict_duplicate: 0, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Auction, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - ), - EidSyncMeasurement { - source: EidSyncSource::Auction, - outcome: EidCookieSyncOutcome::WrittenWithDeferredFreshness, - already_matched: 0, - written: 1, - conflict_duplicate: 0, - deferred: 1, - } - ); - assert_eq!( - EidSyncMeasurement::new(EidSyncSource::NewEc, EidCookieSyncOutcome::ConflictMatched,), - EidSyncMeasurement { - source: EidSyncSource::NewEc, - outcome: EidCookieSyncOutcome::ConflictMatched, - already_matched: 0, - written: 0, - conflict_duplicate: 1, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::PageBids, - EidCookieSyncOutcome::DeferredStaleRead, - ), - EidSyncMeasurement { - source: EidSyncSource::PageBids, - outcome: EidCookieSyncOutcome::DeferredStaleRead, - already_matched: 0, - written: 0, - conflict_duplicate: 0, - deferred: 1, - } + Some("generated-cookie-id"), + "should ingest the browser EID in the create" ); } #[test] - fn finalize_auction_transient_miss_still_persists_eid_updates() { - // `/auction` saves its first lookup into the context and is never - // recovery eligible, so a stale miss there has no later chance to - // retry. Finalization must revalidate before dropping collected IDs. + fn finalize_auction_transient_miss_does_not_persist_eid_updates() { + // `/auction` never writes KV (spec D1): browser EIDs are persisted only + // by the page load's navigation write, whatever its snapshot says. let settings = create_test_settings(); let ec_id = sample_ec_id("named1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1474,20 +1366,17 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should remain"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "a stale endpoint miss must not suppress EID persistence" + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[test] - fn finalize_persists_every_configured_partner_from_client_eids_over_a_trimmed_cookie() { - // Regression test for #1184: the `ts-eids` cookie only carries what - // the browser could fit under its size cap, but `/auction` also - // hands finalization the full EID set from the request body via - // `EcContext::set_client_eids`. That full set must land in KV even - // when the cookie alone would have dropped a configured partner. + fn finalize_never_persists_auction_body_eids() { + // `/auction` body EIDs go to bidders on that request only and are + // never persisted (spec §5.1); neither is the `ts-eids` cookie on a + // non-navigation request. let settings = create_test_settings(); let ec_id = sample_ec_id("cleids1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1551,16 +1440,10 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should exist after ingestion"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the request body's EID should win over the cookie's stale value" - ); - assert_eq!( - stored.ids.get("liveramp.com").map(|id| id.uid.as_str()), - Some("LR_from_body"), - "a partner the cookie trimmed must still be ingested from the body" + .expect("row should remain"); + assert!( + stored.ids.is_empty(), + "should persist neither the auction body nor the cookie EIDs" ); } @@ -1591,10 +1474,9 @@ mod tests { false, purpose1_only_consent, ); - ec_context.set_kv_snapshot(EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }); - ec_context.set_eid_sync_source(EidSyncSource::Auction); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); ec_context.set_client_eids(vec![Eid::new( "id5-sync.com".to_owned(), vec![Uid { @@ -1870,6 +1752,7 @@ mod tests { ec_id: generated_ec, }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1978,6 +1861,11 @@ mod tests { Some("192.0.2.10".to_owned()), ); ec.set_recovery_eligible(recovery_eligible); + if recovery_eligible { + // Only a top-level navigation is recovery eligible, and only a + // navigation carries the page-load write permit. + ec.set_navigation_permit(NavigationPermit::for_test()); + } ec.set_kv_snapshot(snapshot); ec } @@ -2121,6 +2009,7 @@ mod tests { .create(&active_ec, &live_entry()) .expect("should seed active row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2148,7 +2037,7 @@ mod tests { } #[test] - fn finalize_withdrawal_tombstones_both_present_ids_once() { + fn finalize_withdrawal_tombstones_only_the_active_id_once() { let settings = create_test_settings(); let active_ec = sample_ec_id("activ3"); let cookie_ec = sample_ec_id("cook3e"); @@ -2174,6 +2063,7 @@ mod tests { ) .expect("should seed cookie row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2190,10 +2080,10 @@ mod tests { .get(&active_ec) .expect("should read active row") .expect("should retain active tombstone"); - let (cookie_tombstone, cookie_generation) = graph + let (cookie_row, cookie_generation) = graph .get(&cookie_ec) .expect("should read cookie row") - .expect("should retain cookie tombstone"); + .expect("should retain cookie row"); assert!( !active_tombstone.consent.ok, "active row should be withdrawn" @@ -2203,14 +2093,11 @@ mod tests { "active IDs should be cleared" ); assert!( - !cookie_tombstone.consent.ok, - "cookie row should be withdrawn" - ); - assert!( - cookie_tombstone.ids.is_empty(), - "cookie IDs should be cleared" + cookie_row.consent.ok, + "the navigation's single write should tombstone only the active row" ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut repeated_response = empty_response(); ec_finalize_response( &settings, @@ -2235,15 +2122,15 @@ mod tests { graph .get(&cookie_ec) .expect("should read cookie row") - .expect("should retain cookie tombstone") + .expect("should retain cookie row") .1, cookie_generation, - "repeated finalization should not rewrite cookie tombstone" + "repeated finalization should not rewrite the cookie row" ); } #[test] - fn finalize_withdrawal_keeps_cookie_deletion_on_kv_failure() { + fn finalize_withdrawal_keeps_ec_cookie_but_expires_marker_on_kv_failure() { let settings = create_test_settings(); let ec_id = sample_ec_id("failw1"); let consent = ConsentContext { @@ -2257,6 +2144,7 @@ mod tests { ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2282,10 +2170,8 @@ mod tests { "KV failure should not change response status" ); assert!( - cookies - .iter() - .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), - "KV failure should not prevent EC cookie deletion" + cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), + "KV failure should keep the EC cookie so a later navigation can tombstone the row" ); assert!( cookies.iter().any(|cookie| { @@ -2419,6 +2305,276 @@ mod tests { ); } + fn registry_for(source: &str) -> PartnerRegistry { + PartnerRegistry::from_config(&[make_partner(source)]).expect("should build registry") + } + + fn eids_cookie(source: &str, uid: &str) -> String { + base64::engine::general_purpose::STANDARD.encode( + serde_json::json!([{ "source": source, "uids": [{ "id": uid, "atype": 1 }] }]) + .to_string(), + ) + } + + #[test] + fn auction_request_never_writes_browser_eids() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("auc001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Auction); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!( + entry.ids.is_empty(), + "should not write from a non-navigation request" + ); + } + + #[test] + fn navigation_writes_browser_eids_once() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("nav001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert_eq!( + entry.ids["partner.example.com"].uid, "u-1", + "should persist browser uid" + ); + assert!( + ec_context.take_navigation_permit().is_none(), + "should spend the permit" + ); + } + + #[test] + fn redirect_navigation_does_not_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("red001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!(entry.ids.is_empty(), "should not write on a redirect"); + } + + #[test] + fn generated_ec_without_permit_gets_no_cookie_or_row() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_none(), + "should not create without a permit" + ); + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue the EC" + ); + } + + #[test] + fn generated_ec_with_permit_creates_row_and_sets_cookie() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_some(), + "should create the row" + ); + assert!( + get_header(&response, "set-cookie").is_some(), + "should issue the EC" + ); + } + + #[test] + fn generated_ec_collision_is_not_retried_or_issued() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed a colliding row"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue an EC whose row was not created" + ); + } + + #[test] + fn withdrawal_without_permit_keeps_row_and_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + let mut response = empty_response(); + set_header(&mut response, "x-ts-ec", "stale"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!( + entry.consent.ok, + "should not tombstone outside a navigation" + ); + assert!( + get_header(&response, "set-cookie").is_none(), + "should keep the EC cookie" + ); + assert!( + get_header(&response, "x-ts-ec").is_none(), + "should still strip EC response headers" + ); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 9daaac243c..434c38bbb9 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -22,7 +22,8 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; -use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; +use super::page_load::{KvWriteError, PageLoadWrite, RowChange}; +use super::{EcKvSnapshot, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. const MAX_CAS_RETRIES: u32 = 5; @@ -72,16 +73,16 @@ pub enum CreateIfAbsentOutcome { /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PartnerIdUpdate { - /// Partner namespace key in [`KvEntry::ids`]. - pub(crate) partner_id: String, - /// Partner-scoped user ID value. - pub(crate) uid: String, +pub struct PartnerIdUpdate { + /// Source domain key in [`KvEntry::ids`]. + pub partner_id: String, + /// UID to fill in. + pub uid: String, } impl PartnerIdUpdate { /// Creates a partner UID update. - pub(crate) fn new(partner_id: impl Into, uid: impl Into) -> Self { + pub fn new(partner_id: impl Into, uid: impl Into) -> Self { Self { partner_id: partner_id.into(), uid: uid.into(), @@ -89,102 +90,6 @@ impl PartnerIdUpdate { } } -/// Terminal result of one browser EID-cookie persistence attempt. -#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] -pub(crate) enum EidCookieSyncOutcome { - /// The stored values already matched without a write. - #[display("already_matched")] - AlreadyMatched, - /// The one conditional write succeeded. - #[display("written")] - Written, - /// The write added missing IDs while deferring different values of unknown freshness. - #[display("written_with_deferred_freshness")] - WrittenWithDeferredFreshness, - /// A conflicting writer persisted every desired value. - #[display("conflict_matched")] - ConflictMatched, - /// A conflict left at least one desired value absent or different. - #[display("deferred_conflict")] - DeferredConflict, - /// A different stored value had unknown freshness. - #[display("deferred_freshness")] - DeferredFreshness, - /// An eventually consistent read missed a row already proven to exist. - #[display("deferred_stale_read")] - DeferredStaleRead, - /// The identity graph row was missing. - #[display("missing")] - Missing, - /// Consent had been withdrawn in the identity graph. - #[display("consent_withdrawn")] - ConsentWithdrawn, - /// KV or serialization failed. - #[display("failed")] - Failed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CookieUpdateApplication { - AlreadyMatched, - Changed { deferred: bool }, - DeferredFreshness, -} - -fn apply_cookie_partner_id_updates( - entry: &mut KvEntry, - updates: &[PartnerIdUpdate], -) -> CookieUpdateApplication { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - let mut changed = false; - let mut deferred = false; - for (partner_id, uid) in latest_updates { - match entry.ids.get(partner_id) { - Some(existing) if existing.uid == uid => continue, - // Browser EID cookies carry no value-owned sequence or timestamp. - // A different value therefore has unknown freshness and must not - // replace the stored value. - Some(_) => { - deferred = true; - continue; - } - None => {} - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId::single(uid.to_owned()), - ); - changed = true; - } - - if changed { - CookieUpdateApplication::Changed { deferred } - } else if deferred { - CookieUpdateApplication::DeferredFreshness - } else { - CookieUpdateApplication::AlreadyMatched - } -} - -fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - latest_updates.into_iter().all(|(partner_id, uid)| { - entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.uid == uid) - }) -} - pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -465,6 +370,67 @@ impl KvIdentityGraph { ) } + /// Applies the navigation's single row change in one attempt. + /// + /// The `permit` is consumed, so a navigation cannot write twice. A + /// successful tombstone write also records the existing + /// withdrawal-completion marker: that second insert is the one deliberate + /// exception to the one-write-per-page-load rule. It predates the rule and + /// is kept because later withdrawals rely on it to skip the strongly + /// consistent existence check of an already-withdrawn row; it never changes + /// the identity row itself. + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, [`KvWriteError::AlreadyExists`] when a create finds a row, or + /// [`KvWriteError::Store`] on a serialization or store failure, including + /// a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result> { + permit.spend(); + let is_tombstone = matches!(change, RowChange::Tombstone { .. }); + let (entry, ttl, mode) = match change { + RowChange::Create(entry) => (entry, ENTRY_TTL, EcKvWriteMode::Add), + RowChange::Update { entry, expected } => ( + entry, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ), + RowChange::Tombstone { expected } => ( + KvEntry::tombstone(current_timestamp()), + TOMBSTONE_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ), + }; + let (body, metadata) = + Self::serialize_entry(&entry, self.store_name()).change_context(KvWriteError::Store)?; + let outcome = self + .write_entry(ec_id, &body, &metadata, ttl, mode) + .change_context(KvWriteError::Store)?; + match outcome { + EcKvWriteOutcome::Written => { + if is_tombstone { + // Deliberate exception to the one-write-per-page-load rule: + // the withdrawal-completion marker is a separate key that + // predates the rule. Later withdrawals use it to avoid a + // strongly consistent existence check on a row already + // withdrawn, and it never touches the identity row. + self.record_withdrawal_completion(ec_id, entry.consent.updated); + } + Ok(entry) + } + EcKvWriteOutcome::PreconditionFailed => Err(Report::new(match mode { + EcKvWriteMode::Add => KvWriteError::AlreadyExists, + _ => KvWriteError::Conflict, + })), + } + } + /// Creates a new entry, or overwrites an existing tombstone on re-consent. /// /// Three-way behavior: @@ -560,154 +526,6 @@ impl KvIdentityGraph { ))) } - /// Persists browser EID cookies with one conditional write and one conflict read. - /// - /// Browser cookies do not carry a value-owned version, so a different - /// existing value has unknown freshness and is always deferred. After a CAS - /// conflict this method never writes again. A live follow-up becomes the - /// authoritative snapshot; a missing or failed follow-up retains the live - /// pre-write snapshot as proof and defers the update. - pub(crate) fn sync_eid_cookie_updates_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> (EcKvSnapshot, EidCookieSyncOutcome) { - if updates.is_empty() { - return (snapshot, EidCookieSyncOutcome::AlreadyMatched); - } - - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - let current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return (snapshot, EidCookieSyncOutcome::Failed), - _ => self.load_snapshot(ec_id), - }; - - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - EcKvSnapshot::Missing { .. } => { - let outcome = if proven.is_some() { - EidCookieSyncOutcome::DeferredStaleRead - } else { - EidCookieSyncOutcome::Missing - }; - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, outcome); - } - EcKvSnapshot::Failed { .. } => { - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, EidCookieSyncOutcome::Failed); - } - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - } - }; - - if !entry.consent.ok { - return (current, EidCookieSyncOutcome::ConsentWithdrawn); - } - let deferred_freshness = match apply_cookie_partner_id_updates(&mut entry, updates) { - CookieUpdateApplication::AlreadyMatched => { - return (current, EidCookieSyncOutcome::AlreadyMatched); - } - CookieUpdateApplication::DeferredFreshness => { - return (current, EidCookieSyncOutcome::DeferredFreshness); - } - CookieUpdateApplication::Changed { deferred } => deferred, - }; - - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => ( - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }, - if deferred_freshness { - EidCookieSyncOutcome::WrittenWithDeferredFreshness - } else { - EidCookieSyncOutcome::Written - }, - ), - Ok(EcKvWriteOutcome::PreconditionFailed) => { - let refreshed = self.load_snapshot(ec_id); - let Some(refreshed_entry) = refreshed.entry_for(ec_id) else { - // The failed CAS proved `current`'s generation stale. Keep - // only its existence proof so later writes must reread. - let proof = match current { - EcKvSnapshot::Present { - ec_id: proven_id, - entry, - .. - } => EcKvSnapshot::Present { - ec_id: proven_id, - entry, - generation: None, - }, - other => other, - }; - let kept = Self::keep_proven(ec_id, refreshed, Some(&proof)); - return (kept, EidCookieSyncOutcome::DeferredConflict); - }; - if !refreshed_entry.consent.ok { - return (refreshed, EidCookieSyncOutcome::ConsentWithdrawn); - } - let outcome = if partner_id_updates_match(refreshed_entry, updates) { - EidCookieSyncOutcome::ConflictMatched - } else { - EidCookieSyncOutcome::DeferredConflict - }; - (refreshed, outcome) - } - Err(err) => { - log::warn!("EID cookie sync write failed: {err:?}"); - ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ) - } - } - } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. /// /// A caller-supplied `Present` snapshot is *proof* that the row exists — @@ -1052,7 +870,12 @@ impl KvIdentityGraph { ) } - fn withdrawal_marker_exists( + /// Whether a still-valid withdrawal-completion marker exists for `ec_id`. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] when the marker listing fails. + pub(crate) fn withdrawal_marker_exists( &self, ec_id: &str, now: Option, @@ -1231,6 +1054,7 @@ impl KvIdentityGraph { self.overwrite_withdrawal_tombstone(ec_id).map(Some) } + #[cfg(test)] fn overwrite_withdrawal_tombstone( &self, ec_id: &str, @@ -1254,200 +1078,6 @@ impl KvIdentityGraph { }) } - /// Resolves a tombstone attempt whose point read reported the row absent. - /// - /// A completion marker proves an earlier withdrawal finished and avoids a - /// second root existence check. Otherwise, a proven-absent key is a no-op: - /// there is nothing to withdraw, and a forged cookie must not mint a row. A - /// key that provably exists is tombstoned unconditionally because no CAS - /// generation is available after a missed read. Marker-check failure falls - /// back to that privacy write, as does an unusable clock; root-existence - /// failure leaves withdrawal unresolved rather than silently dropped. - fn tombstone_unproven_missing( - &self, - ec_id: &str, - missing: EcKvSnapshot, - now: Option, - ) -> EcKvSnapshot { - match self.withdrawal_marker_exists(ec_id, now) { - Ok(true) => { - log::debug!( - "withdrawal tombstone for '{}': completion marker already exists", - log_id(ec_id) - ); - return missing; - } - Ok(false) => {} - Err(err) => { - // Marker failure must not weaken withdrawal. Fall back to the - // existing root existence check and unconditional privacy write. - log::warn!( - "withdrawal completion marker lookup failed for '{}': {err:?}", - log_id(ec_id) - ); - } - } - - match self.key_exists_confirmed(ec_id) { - Ok(false) => missing, - Ok(true) => { - log::warn!( - "withdrawal tombstone for '{}': point read missed a row the store still \ - lists; writing an unconditional tombstone", - log_id(ec_id) - ); - match self.overwrite_withdrawal_tombstone(ec_id) { - Ok(tombstone) => { - self.record_withdrawal_completion(ec_id, tombstone.consent.updated); - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - } - } - Err(err) => { - log::warn!( - "unconditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - Err(err) => { - log::warn!( - "withdrawal tombstone for '{}': existence check failed, cannot confirm \ - absence: {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - - /// Writes a tombstone only when an existing row can be confirmed. - /// - /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` - /// cookie must not mint a row. But a *point read* cannot prove absence on - /// an eventually-consistent store, and dropping a withdrawal is worse than - /// a redundant read, so absence is established in two stages: - /// - /// 1. Any snapshot that is not a usable `Present` for this EC ID — a - /// publisher preload that read `Missing`, a read that `Failed`, or one - /// lacking a CAS generation — is re-read. On the publisher path that - /// re-read is separated from the preload by the full origin round trip, - /// which gives replication time to converge. - /// 2. A re-read that still reports the row absent is checked against - /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads - /// the primary data source. - /// - /// Resolving the initial snapshot happens outside the retry counter, so all - /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. - pub(crate) fn tombstone_existing_from_snapshot( - &self, - ec_id: &str, - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - .. - } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - _ => self.load_snapshot(ec_id), - }; - - for _attempt in 0..MAX_CAS_RETRIES { - let generation = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - .. - } if snapshot_id == ec_id && !entry.consent.ok => return current, - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(generation), - .. - } if snapshot_id == ec_id => generation, - // A missing row (including one that disappeared mid-retry) is - // only a no-op once absence is proven against the primary data - // source. - EcKvSnapshot::Missing { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => { - return self.tombstone_unproven_missing( - ec_id, - current, - checked_current_timestamp(), - ); - } - // A refreshed read that failed (or any other unusable state) - // fails closed rather than silently dropping the withdrawal. - _ => { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - }; - - let tombstone = KvEntry::tombstone(current_timestamp()); - let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - TOMBSTONE_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => { - self.record_withdrawal_completion(ec_id, tombstone.consent.updated); - return EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - }; - } - Ok(EcKvWriteOutcome::PreconditionFailed) => { - current = self.load_snapshot(ec_id); - } - Err(err) => { - log::warn!( - "conditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - } - } - - // Withdrawal enforcement lost every CAS race, so the row can still be - // live with consent granted while the browser cookie is cleared. That - // divergence is only visible to operators if it is logged here. - log::warn!( - "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ - identity-graph row may still be live with consent granted", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -1552,7 +1182,7 @@ impl KvIdentityGraph { /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). Consent - /// withdrawal uses [`Self::tombstone_existing_from_snapshot`] instead. + /// withdrawal uses a page-load tombstone via [`Self::commit_page_load`] instead. /// /// # Errors /// @@ -1619,91 +1249,14 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; + use crate::ec::checked_current_timestamp; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_types::{KvEid, KvUid}; + use crate::ec::page_load::{Generation, NavigationPermit}; - /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition - /// and deletes the key, simulating a concurrent withdrawal that removes the - /// row between this writer's read and its write. - struct DisappearOnConflictEcKv { - inner: InMemoryEcKv, - conflicts_remaining: std::sync::Mutex, - } - - impl DisappearOnConflictEcKv { - fn new(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("disappear-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - } - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize seeded entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed live entry"); - } - } - - impl EcKvStore for DisappearOnConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - let mut remaining = self - .conflicts_remaining - .lock() - .expect("should lock conflict counter"); - if *remaining > 0 { - *remaining -= 1; - self.inner.delete(key).expect("should delete on conflict"); - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - - fn snapshot_ec_id() -> String { - format!("{}.ABC123", "a".repeat(64)) + fn snapshot_ec_id() -> String { + format!("{}.ABC123", "a".repeat(64)) } #[test] @@ -1871,15 +1424,6 @@ mod tests { } } - fn with_partner_update_on_conflict(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("partner-conflict-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - revive_on_conflict: true, - partner_update_on_conflict: true, - } - } - fn seed_tombstone(&self, ec_id: &str) { let (body, meta) = KvIdentityGraph::serialize_entry( &KvEntry::tombstone(1000), @@ -1986,131 +1530,6 @@ mod tests { } } - /// Store that replaces the row during the first EID CAS write and records - /// the request's reads and conditional writes. - struct EidConflictEcKv { - inner: InMemoryEcKv, - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - follow_up_miss: bool, - miss_next_lookup: std::sync::atomic::AtomicBool, - } - - impl EidConflictEcKv { - fn new( - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - ) -> Self { - Self { - inner: InMemoryEcKv::new("eid-conflict-store"), - concurrent_entry, - lookups, - conditional_writes, - follow_up_miss: false, - miss_next_lookup: std::sync::atomic::AtomicBool::new(false), - } - } - - fn with_follow_up_miss(mut self) -> Self { - self.follow_up_miss = true; - self - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize initial entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed initial entry"); - } - } - - impl EcKvStore for EidConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.lookups - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if self - .miss_next_lookup - .swap(false, std::sync::atomic::Ordering::Relaxed) - { - return Ok(None); - } - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - self.conditional_writes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let (body, meta) = KvIdentityGraph::serialize_entry( - &self.concurrent_entry, - self.inner.store_name(), - ) - .expect("should serialize concurrent entry"); - self.inner - .insert( - key, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Overwrite, - }, - ) - .expect("should write concurrent entry"); - if self.follow_up_miss { - self.miss_next_lookup - .store(true, std::sync::atomic::Ordering::Relaxed); - } - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn count_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result> { - self.inner.count_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn create_or_revive_retries_cas_conflict_and_succeeds() { let store = ConflictInjectingEcKv::new(2, false); @@ -2247,353 +1666,82 @@ mod tests { } #[test] - fn eid_cookie_sync_conflict_matching_value_stops_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("desired-uid".to_owned()), - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); + fn evaluate_cluster_returns_stored_value_without_store_io() { + let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = live_entry(); + entry.network = Some(KvNetwork { + cluster_size: Some(5), + }); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); + let cluster_size = kv + .evaluate_cluster(&ec_id, &entry, 0) + .expect("should not touch store when cluster_size is already known"); - assert_eq!(outcome, EidCookieSyncOutcome::ConflictMatched); - assert_eq!( - snapshot - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("desired-uid"), - "the authoritative follow-up snapshot should replace stale request state" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" + cluster_size, + Some(5), + "should return stored cluster_size without re-listing keys" ); } #[test] - fn eid_cookie_sync_keeps_live_proof_when_conflict_follow_up_misses() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - live_entry(), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ) - .with_follow_up_miss(); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); + fn evaluate_cluster_skips_tombstone_without_store_io() { + let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = KvEntry::tombstone(1000); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); + let cluster_size = kv + .evaluate_cluster(&ec_id, &entry, 0) + .expect("should not touch store for tombstone entries"); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the live pre-write row should prevent conflict deferral from entering orphan recovery" - ); - assert_eq!( - snapshot.generation_for(&ec_id), - None, - "a failed CAS must not return its rejected generation" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "the initial refresh and conflict follow-up should be the only reads" - ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "the conflict must remain the request's only conditional write" + cluster_size, None, + "should not evaluate or write cluster_size for tombstones" ); } #[test] - fn eid_cookie_sync_reports_proven_refresh_miss_as_deferred() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("empty-store"); - let proven = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; + fn create_then_get_roundtrips_entry() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = live_entry(); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - proven, - ); + kv.create(&ec_id, &entry).expect("should create new entry"); + let (loaded, generation) = kv + .get(&ec_id) + .expect("should read entry back") + .expect("should find created entry"); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the add-confirmed row should survive an eventually consistent miss" - ); - assert_eq!( - outcome, - EidCookieSyncOutcome::DeferredStaleRead, - "a row already proven to exist should report a deferred stale read" - ); + assert!(loaded.consent.ok, "should preserve consent state"); + assert!(generation > 0, "should expose a generation marker"); } #[test] - fn partner_id_conflict_match_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("latest-uid".to_owned()), - ); - let updates = [ - PartnerIdUpdate::new("ssp_x", "older-uid"), - PartnerIdUpdate::new("ssp_x", "latest-uid"), - ]; + fn create_rejects_existing_key() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = live_entry(); + kv.create(&ec_id, &entry).expect("should create new entry"); + let err = kv + .create(&ec_id, &entry) + .expect_err("should reject duplicate create"); assert!( - partner_id_updates_match(&entry, &updates), - "conflict matching should use the same last-value-wins rule as writes" + format!("{err}").contains("already exists"), + "should report duplicate key" ); } #[test] - fn eid_cookie_sync_conflicting_value_defers_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "stale-uid")], - snapshot, - ); + fn create_if_absent_reports_written_and_collision() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read concurrent row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "concurrent-uid"); - } - - #[test] - fn eid_cookie_sync_preserves_concurrent_withdrawal_after_conflict() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - KvEntry::tombstone(2_000), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::ConsentWithdrawn); - assert!( - snapshot - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the refreshed withdrawal tombstone must remain authoritative" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - } - - #[test] - fn eid_cookie_sync_defers_different_values_without_value_owned_freshness() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "incoming-uid")], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read protected row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - } - - #[test] - fn eid_cookie_sync_adds_missing_ids_while_deferring_different_values() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("mixed-freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("protected-uid".to_owned()), - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[ - PartnerIdUpdate::new("ssp_x", "different-uid"), - PartnerIdUpdate::new("ssp_y", "new-uid"), - ], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::WrittenWithDeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read updated row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - assert_eq!(stored.ids["ssp_y"].uid, "new-uid"); - } - - #[test] - fn evaluate_cluster_returns_stored_value_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut entry = live_entry(); - entry.network = Some(KvNetwork { - cluster_size: Some(5), - }); - - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store when cluster_size is already known"); - - assert_eq!( - cluster_size, - Some(5), - "should return stored cluster_size without re-listing keys" - ); - } - - #[test] - fn evaluate_cluster_skips_tombstone_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = KvEntry::tombstone(1000); - - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store for tombstone entries"); - - assert_eq!( - cluster_size, None, - "should not evaluate or write cluster_size for tombstones" - ); - } - - #[test] - fn create_then_get_roundtrips_entry() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = live_entry(); - - kv.create(&ec_id, &entry).expect("should create new entry"); - let (loaded, generation) = kv - .get(&ec_id) - .expect("should read entry back") - .expect("should find created entry"); - - assert!(loaded.consent.ok, "should preserve consent state"); - assert!(generation > 0, "should expose a generation marker"); - } - - #[test] - fn create_rejects_existing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = live_entry(); - - kv.create(&ec_id, &entry).expect("should create new entry"); - let err = kv - .create(&ec_id, &entry) - .expect_err("should reject duplicate create"); - assert!( - format!("{err}").contains("already exists"), - "should report duplicate key" - ); - } - - #[test] - fn create_if_absent_reports_written_and_collision() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should create absent entry"), - CreateIfAbsentOutcome::Written + kv.create_if_absent(&ec_id, &live_entry()) + .expect("should create absent entry"), + CreateIfAbsentOutcome::Written ); assert_eq!( kv.create_if_absent(&ec_id, &live_entry()) @@ -2661,270 +1809,91 @@ mod tests { } #[test] - fn create_or_revive_clears_withdrawal_marker() { - let kv = KvIdentityGraph::in_memory("test_store"); + fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { + let graph = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + graph + .write_withdrawal_marker(&ec_id, 1000) + .expect("should seed marker"); + let valid_until = 1000 + TOMBSTONE_TTL.as_secs(); + + for (now, expected) in [ + (Some(valid_until - 1), true), + (Some(valid_until), false), + (Some(valid_until + 1), false), + (None, false), + ] { + assert_eq!( + graph + .withdrawal_marker_exists(&ec_id, now) + .expect("should check marker validity"), + expected, + "marker validity should honor clock availability and exclusive expiry at {now:?}" + ); + } + } + + #[test] + fn withdrawal_marker_existence_rejects_malformed_expiry_suffix() { let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" + let marker_key = KvIdentityGraph::withdrawal_marker_key( + &ec_id, + current_timestamp() + TOMBSTONE_TTL.as_secs(), ); + let longer_key = format!("{marker_key}-longer"); + let store = InMemoryEcKv::new("test_store"); + store + .insert( + &longer_key, + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed longer marker key"); + let kv = KvIdentityGraph::new(store); - kv.create_or_revive(&ec_id, &live_entry()) - .expect("should revive tombstone"); - - let (loaded, _) = kv - .get(&ec_id) - .expect("should read revived entry") - .expect("should find revived entry"); - assert!(loaded.consent.ok, "should be live after revive"); assert!( !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "revival should clear stale withdrawal completion" + .expect("should validate withdrawal marker expiry"), + "a malformed expiry suffix must not prove withdrawal completion" ); } #[test] - fn stale_completion_marker_does_not_suppress_withdrawal_after_root_recreation() { + fn clear_absent_withdrawal_marker_lists_once_without_delete() { let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); let ec_id = snapshot_ec_id(); - let expired_updated = current_timestamp().saturating_sub(TOMBSTONE_TTL.as_secs() + 1); - let expired_tombstone = KvEntry::tombstone(expired_updated); - graph - .create(&ec_id, &expired_tombstone) - .expect("should seed old tombstone"); - graph - .write_withdrawal_marker(&ec_id, expired_updated) - .expect("should seed old completion marker"); - // Model the root expiring just before its later-written completion - // marker, followed by recreation of the same key. - graph.store.delete(&ec_id).expect("should expire root row"); - assert_eq!( - graph - .create_if_absent(&ec_id, &live_entry()) - .expect("should recreate expired root"), - CreateIfAbsentOutcome::Written, - "should make the same key live again after root expiry" - ); + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept an absent marker"); - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, + assert_eq!( + operations.list_count(), + 1, + "absent marker guard should list once" ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "an expired completion marker must not suppress withdrawal of a recreated live row" + assert_eq!( + operations.delete_count(), + 0, + "absent marker guard should avoid a delete" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read recreated row") - .expect("should retain recreated row"); - assert!(!stored.consent.ok, "recreated row should be tombstoned"); } #[test] - fn withdrawal_failed_clock_stale_miss_tombstones_live_root() { + fn writing_an_existing_withdrawal_marker_is_idempotent() { let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &concurrent_live_entry()) - .expect("should seed a live row with partner IDs"); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a completion marker left after root recreation"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - assert!( - matches!(missing, EcKvSnapshot::Missing { .. }), - "should reproduce a stale point-read miss for the live row" - ); + let updated = current_timestamp(); - // None is the checked clock's failure result, not Unix epoch zero. - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); - - let (stored, generation) = graph - .get(&ec_id) - .expect("should read back persisted withdrawal") - .expect("should retain the root"); - assert!( - !stored.consent.ok, - "an unusable clock must not let a marker suppress withdrawal" - ); - assert!( - stored.ids.is_empty(), - "withdrawal should clear stored partner IDs" - ); - assert_eq!( - generation, 2, - "withdrawal should write the existing root once" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "clock failure should not fail a strongly confirmed withdrawal" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should strongly check the root" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL - }, - ], - "should write only the root tombstone and its completion marker" - ); - } - - #[test] - fn withdrawal_failed_clock_does_not_create_absent_root() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a marker without a root"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "absent root should remain missing" - ); - assert!( - graph - .get(&ec_id) - .expect("should read absent root") - .is_none(), - "clock failure must not mint an unknown root" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should prove root absence despite the marker" - ); - assert!( - operations.inserts().is_empty(), - "absent identity should cause no writes" - ); - } - - #[test] - fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { - let graph = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed marker"); - let valid_until = 1000 + TOMBSTONE_TTL.as_secs(); - - for (now, expected) in [ - (Some(valid_until - 1), true), - (Some(valid_until), false), - (Some(valid_until + 1), false), - (None, false), - ] { - assert_eq!( - graph - .withdrawal_marker_exists(&ec_id, now) - .expect("should check marker validity"), - expected, - "marker validity should honor clock availability and exclusive expiry at {now:?}" - ); - } - } - - #[test] - fn withdrawal_marker_existence_rejects_malformed_expiry_suffix() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let marker_key = KvIdentityGraph::withdrawal_marker_key( - &ec_id, - current_timestamp() + TOMBSTONE_TTL.as_secs(), - ); - let longer_key = format!("{marker_key}-longer"); - let store = InMemoryEcKv::new("test_store"); - store - .insert( - &longer_key, - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed longer marker key"); - let kv = KvIdentityGraph::new(store); - - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should validate withdrawal marker expiry"), - "a malformed expiry suffix must not prove withdrawal completion" - ); - } - - #[test] - fn clear_absent_withdrawal_marker_lists_once_without_delete() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept an absent marker"); - - assert_eq!( - operations.list_count(), - 1, - "absent marker guard should list once" - ); - assert_eq!( - operations.delete_count(), - 0, - "absent marker guard should avoid a delete" - ); - } - - #[test] - fn writing_an_existing_withdrawal_marker_is_idempotent() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let updated = current_timestamp(); - - kv.write_withdrawal_marker(&ec_id, updated) - .expect("should write first completion marker"); - kv.write_withdrawal_marker(&ec_id, updated) - .expect("should accept completion marker add collision"); + kv.write_withdrawal_marker(&ec_id, updated) + .expect("should write first completion marker"); + kv.write_withdrawal_marker(&ec_id, updated) + .expect("should accept completion marker add collision"); assert_eq!( operations.inserts(), @@ -3015,99 +1984,6 @@ mod tests { ); } - #[test] - fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept a marker removed by another request"); - - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker removal"), - "completion marker should remain absent" - ); - } - - #[test] - fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.clear_withdrawal_marker(&ec_id).is_err(), - "a marker that remains after delete failure should block revival" - ); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker remains"), - "completion marker should remain present" - ); - } - - #[test] - fn withdrawal_marker_is_excluded_from_hash_prefix_count() { - let hash = "a".repeat(64); - let ec_id = format!("{hash}.ABC123"); - let kv = KvIdentityGraph::in_memory("test_store"); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count live root"), - 1, - "the root should be the only hash-prefix key" - ); - - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" - ); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count tombstoned root"), - 1, - "the marker namespace must not affect cluster counts" - ); - } - - #[test] - fn delete_removes_withdrawal_marker() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - kv.delete(&ec_id).expect("should delete entry and marker"); - - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "hard delete should remove the root" - ); - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "hard delete should remove withdrawal completion" - ); - } - #[test] fn upsert_partner_id_if_exists_reports_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -3234,48 +2110,6 @@ mod tests { assert!(kv.get(&ec_id).expect("should read store").is_none()); } - #[test] - fn tombstone_existing_from_snapshot_never_creates_missing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let snapshot = EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }; - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "withdrawal should preserve the missing snapshot for an absent identity" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "withdrawal must not create a tombstone for an absent key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_uses_existing_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should return the persisted tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "should persist withdrawal state"); - } - #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -3344,14 +2178,6 @@ mod tests { self.deletes.load(std::sync::atomic::Ordering::Relaxed) } - fn operation_count(&self) -> usize { - self.lookup_count() - + self.exact_check_count() - + self.inserts().len() - + self.list_count() - + self.delete_count() - } - fn inserts(&self) -> Vec { self.inserts .lock() @@ -3405,55 +2231,12 @@ mod tests { } } - fn completed_with_root_check_failure( + fn with_marker_delete_failure( operations: Arc, - ec_id: &str, + remove_before_error: bool, ) -> Self { - let store = Self { - root_check_fails: true, - stale_lookups_remaining: std::sync::Mutex::new(u32::MAX), - ..Self::new(operations) - }; - let tombstone = KvEntry::tombstone(current_timestamp()); - let (body, metadata) = - KvIdentityGraph::serialize_entry(&tombstone, store.inner.store_name()) - .expect("should serialize tombstone"); - store - .inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &metadata, - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed tombstone"); - store - .inner - .insert( - &KvIdentityGraph::withdrawal_marker_key( - ec_id, - tombstone.consent.updated + TOMBSTONE_TTL.as_secs(), - ), - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed completion marker"); - store - } - - fn with_marker_delete_failure( - operations: Arc, - remove_before_error: bool, - ) -> Self { - Self { - marker_delete_failure_removes_key: Some(remove_before_error), + Self { + marker_delete_failure_removes_key: Some(remove_before_error), ..Self::new(operations) } } @@ -3794,562 +2577,87 @@ mod tests { let ec_id = snapshot_ec_id(); let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a confirmed miss must stay missing" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "must not create a root entry for a missing key" - ); - } - - #[test] - fn snapshot_upsert_failed_snapshot_is_not_revalidated() { - // A lookup that already errored is not retried on the hot path, even - // though the row exists. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed lookup must not be retried by partner enrichment" - ); - } - - #[test] - fn snapshot_upsert_rejects_tombstone() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should seed tombstone"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| entry.ids.is_empty()), - "a tombstone must reject partner enrichment" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("tombstone should remain"); - assert!(stored.ids.is_empty(), "no update should reach the store"); - } - - #[test] - fn snapshot_upsert_store_failure_returns_failed_not_request_local() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a store write failure must not claim request-local IDs were persisted" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { - for generation in [Some(7), None] { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation, - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - - assert_eq!( - outcome, snapshot, - "should preserve authoritative tombstone state" - ); - assert_eq!( - operations.operation_count(), - 0, - "an authoritative tombstone should not access the backend" - ); - } - } - - #[test] - fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let live_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); - - assert_eq!( - operations.lookup_count(), - 0, - "usable generation should avoid a read" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "first withdrawal should write the root and its completion marker" - ); - let first_snapshot = graph.load_snapshot(&ec_id); - let (first_entry, first_generation) = match &first_snapshot { - EcKvSnapshot::Present { - entry, generation, .. - } => (entry.as_ref().clone(), *generation), - other => panic!("should load first tombstone, got {other:?}"), - }; - operations.reset(); - - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); - - assert_eq!( - operations.operation_count(), - 0, - "repeated withdrawal should not access the backend" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - first_generation, - "repeated withdrawal should preserve the stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - Some(first_entry.consent.updated), - "repeated withdrawal should preserve the first tombstone timestamp" - ); - } - - #[test] - fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 2, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - let first_updated = first_outcome - .entry_for(&ec_id) - .expect("should return first tombstone") - .consent - .updated; - assert_eq!( - operations.lookup_count(), - 1, - "first stale miss should retry its point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "first stale miss should list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "first stale miss should confirm root existence once" - ); - assert_eq!( - operations.inserts().len(), - 2, - "first stale miss should write the root and completion marker" - ); - operations.reset(); - - graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!( - operations.lookup_count(), - 1, - "a repeated stale miss should retry its point read once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "a valid completion marker should avoid a root existence check" - ); - assert_eq!( - operations.list_count(), - 1, - "a repeated stale miss should list completion markers once" - ); - assert_eq!( - operations.delete_count(), - 0, - "a repeated withdrawal should not delete marker state" - ); - assert!( - operations.inserts().is_empty(), - "a repeated stale miss should not rewrite the completed tombstone" - ); - let (stored, generation) = graph - .get(&ec_id) - .expect("should read stored tombstone") - .expect("should preserve tombstone"); - assert_eq!( - generation, 2, - "only the first withdrawal should advance the root generation" - ); - assert_eq!( - stored.consent.updated, first_updated, - "repeated withdrawal should preserve the first tombstone timestamp" - ); - } - - #[test] - fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let stale_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); - - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - ], - "parallel loser should attempt stale CAS once and never replace the winner" - ); - assert_eq!( - operations.lookup_count(), - 1, - "parallel loser should reread exactly once after its conflict" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - Some(2), - "parallel loser should return the winner's stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - first_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - "parallel loser should preserve the winner's tombstone" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { - let graph = KvIdentityGraph::failing("unavailable-store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(3), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - - assert_eq!( - outcome, snapshot, - "authoritative tombstone should not touch unavailable backend" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { - let ec_id = snapshot_ec_id(); - let states = [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }, - EcKvSnapshot::Present { - ec_id: "different-ec-id".to_owned(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(9), - }, - ]; - - for state in states { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); - - assert_eq!( - operations.lookup_count(), - 1, - "state should force one reread" - ); - assert_eq!( - operations.inserts().len(), - 2, - "live reread should write the root and completion marker" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "reread live row should be tombstoned" - ); - } - } - - #[test] - fn tombstone_stale_miss_valid_marker_skips_root_check_and_writes() { - let ec_id = snapshot_ec_id(); - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::completed_with_root_check_failure( - Arc::clone(&operations), - &ec_id, - )); - - let outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a completion marker should resolve a repeated stale miss" - ); - assert_eq!( - operations.lookup_count(), - 1, - "should retry the stale point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "should strongly list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "valid marker should bypass the failing root check" - ); - assert!( - operations.inserts().is_empty(), - "valid marker should prevent redundant writes" - ); - } - - #[test] - fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_marker_failures( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let outcome = graph.tombstone_existing_from_snapshot( + let outcome = kv.upsert_partner_ids_from_snapshot( &ec_id, + &updates, EcKvSnapshot::Missing { ec_id: ec_id.clone(), }, ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "marker failures must not suppress the withdrawal write" + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a confirmed miss must stay missing" ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "failed completion recording should still be attempted after the root write" + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "must not create a root entry for a missing key" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read stored row") - .expect("should preserve the root"); - assert!(!stored.consent.ok, "root should remain tombstoned"); } #[test] - fn tombstone_existing_from_snapshot_retries_cas_conflict() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + fn snapshot_upsert_failed_snapshot_is_not_revalidated() { + // A lookup that already errored is not retried on the hot path, even + // though the row exists. + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should retry the conflict and persist the tombstone" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a failed lookup must not be retried by partner enrichment" ); } #[test] - fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot refreshes once before its CAS. That - // refresh must not spend a CAS attempt, so a withdrawal tombstone still - // persists after four conflicts and a successful fifth write. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + fn snapshot_upsert_rejects_tombstone() { + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let snapshot = kv.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); assert!( outcome .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { - // Every CAS attempt loses its race, so the row stays live with consent - // granted while the browser cookie is already cleared. The caller must - // see a failure it can report rather than a silent no-op. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "CAS exhaustion must report a failed withdrawal" + .is_some_and(|entry| entry.ids.is_empty()), + "a tombstone must reject partner enrichment" ); - let (stored, _) = graph + let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("row should remain"); - assert!( - stored.consent.ok, - "the row is still live, which is exactly why the failure must be reported" - ); + .expect("tombstone should remain"); + assert!(stored.ids.is_empty(), "no update should reach the store"); } #[test] - fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::with_partner_update_on_conflict(1)); + fn snapshot_upsert_store_failure_returns_failed_not_request_local() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let snapshot = graph.load_snapshot(&ec_id); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - let entry = outcome - .entry_for(&ec_id) - .expect("should return persisted tombstone"); - assert!(!entry.consent.ok, "withdrawal should win after retry"); assert!( - entry.ids.is_empty(), - "withdrawal should clear concurrent partner IDs" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a store write failure must not claim request-local IDs were persisted" ); } @@ -4375,73 +2683,6 @@ mod tests { ); } - #[test] - fn tombstone_existing_from_snapshot_store_failure_returns_failed() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed tombstone write should return failed snapshot state" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { - let store = DisappearOnConflictEcKv::new(1); - store.seed_live(&snapshot_ec_id()); - let graph = KvIdentityGraph::new(store); - let ec_id = snapshot_ec_id(); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a row that disappears during retry becomes a no-op" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "must not recreate the disappeared key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { - // A prior request-scoped read failed, so the snapshot is `Failed`. A - // withdrawal must not silently drop consent removal: re-read the store - // and tombstone the row if it is authoritatively present. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "a failed snapshot must re-read and persist the tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "withdrawal must reach the store"); - } - #[test] fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { let graph = KvIdentityGraph::stale_lookup("stale-store", 1); @@ -4883,4 +3124,131 @@ mod tests { "a different suffix under the same hash is a different identity" ); } + + fn tombstone_via_page_load(kv: &KvIdentityGraph, ec_id: &str) { + let snapshot = kv.load_snapshot(ec_id); + let expected = Generation::of(&snapshot, ec_id).expect("should read generation"); + let permit = NavigationPermit::for_test() + .confirm(http::StatusCode::OK) + .expect("should confirm permit"); + kv.commit_page_load(permit, ec_id, RowChange::Tombstone { expected }) + .expect("should write tombstone"); + } + + #[test] + fn create_or_revive_clears_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); + + let (loaded, _) = kv + .get(&ec_id) + .expect("should read revived entry") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "revival should clear stale withdrawal completion" + ); + } + + #[test] + fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept a marker removed by another request"); + + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker removal"), + "completion marker should remain absent" + ); + } + + #[test] + fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.clear_withdrawal_marker(&ec_id).is_err(), + "a marker that remains after delete failure should block revival" + ); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker remains"), + "completion marker should remain present" + ); + } + + #[test] + fn withdrawal_marker_is_excluded_from_hash_prefix_count() { + let hash = "a".repeat(64); + let ec_id = format!("{hash}.ABC123"); + let kv = KvIdentityGraph::in_memory("test_store"); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count live root"), + 1, + "the root should be the only hash-prefix key" + ); + + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count tombstoned root"), + 1, + "the marker namespace must not affect cluster counts" + ); + } + + #[test] + fn delete_removes_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + + kv.delete(&ec_id).expect("should delete entry and marker"); + + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "hard delete should remove the root" + ); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "hard delete should remove withdrawal completion" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 2a638dc219..45ac8b6df6 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -186,6 +186,21 @@ impl KvPartnerId { .unwrap_or_default(); *self != before } + + /// Removes the last UID of the last entry and resyncs `uid`; used by the + /// write planner's size cap. + pub(crate) fn drop_last_uid(&mut self) { + if let Some(eid) = self.eids.last_mut() { + eid.uids.pop(); + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + } } /// One stored EID entry with its EID-level fields. diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index e815c43a19..33981b49a5 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -7,9 +7,10 @@ //! context. This is called pre-routing on every request. //! //! 2. **Generate** — [`EcContext::generate_if_needed`] creates a new EC ID -//! when none exists and consent allows it. This is called only in organic -//! handlers (publisher proxy, integration proxy) — never in read-only -//! endpoints like `/_ts/api/v1/identify`. +//! when none exists and consent allows it. This is called only for +//! publisher navigations — never in integration proxies or read-only +//! endpoints like `/_ts/api/v1/identify`. The new row is written by the +//! navigation's single page-load write in finalize ([`page_load`]). //! //! # Module structure //! @@ -29,6 +30,9 @@ //! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes +//! - [`page_load`] — Single-use page-load write permit and row changes +//! - [`write_plan`] — Plans a navigation's single row change +//! - [`coordinator`] — Server-side identity lookup coordination mod auth; @@ -36,6 +40,7 @@ pub mod admin; pub mod batch_sync; pub mod consent; pub mod cookies; +pub mod coordinator; pub mod device; pub mod eid_types; pub mod eids; @@ -45,12 +50,14 @@ pub mod identify; pub mod kv; pub mod kv_backend; pub mod kv_types; +pub mod page_load; pub mod partner; pub mod prebid_eids; pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; +pub mod write_plan; /// Characters of an identifier kept when redacting it for a log. const LOG_ID_PREFIX_CHARS: usize = 8; @@ -85,8 +92,10 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; -use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; +use self::coordinator::results::LookupResults; +use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; +use self::page_load::NavigationPermit; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. @@ -242,6 +251,17 @@ pub fn get_ec_id(req: &Request) -> Result, Report); + +impl Clone for PermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during @@ -284,6 +304,12 @@ pub struct EcContext { pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, + /// Navigation permit issued by the router for page-load navigations. + navigation_permit: PermitSlot, + /// Validated provider lookup results collected for this navigation. + lookup_results: LookupResults, + /// New row built by [`Self::generate_if_needed`], written by finalize. + pending_entry: Option>, } impl EcContext { @@ -366,14 +392,20 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, }) } /// Generates a new EC ID if none exists and consent allows it. /// - /// This is the second phase of the EC lifecycle. Call this only in - /// organic handlers (publisher proxy, integration proxy, auction) — - /// never in read-only endpoints. + /// This is the second phase of the EC lifecycle. Call this only for + /// publisher navigations — never in read-only endpoints. + /// + /// Never writes KV. When `kv` is available, the new row is held as the + /// pending entry and created by the navigation's single page-load write + /// in finalize; the EC is issued only if that write succeeds. /// /// If an EC ID already exists (from the request), this is a no-op. /// If consent does not permit EC creation, this is a no-op. @@ -405,62 +437,32 @@ impl EcContext { }) })?; - const MAX_CREATE_ATTEMPTS: usize = 5; - for attempt in 0..MAX_CREATE_ATTEMPTS { - let ec_id = generation::generate_ec_id(settings, client_ip)?; - log::info!("Generated new EC ID: {}", log_id(&ec_id)); - let now = current_timestamp(); - let mut entry = KvEntry::new( - &self.consent, - self.geo_info.as_ref(), - now, - &settings.publisher.domain, - ); - entry.device = self - .device_signals - .as_ref() - .map(DeviceSignals::to_kv_device); - - if let Some(graph) = kv { - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - self.kv_snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => { - log::warn!( - "Generated EC ID collision on attempt {}/{MAX_CREATE_ATTEMPTS}", - attempt + 1 - ); - continue; - } - Err(err) => { - log::error!( - "Failed to create EC entry for id '{}' after generation: {err:?}", - log_id(&ec_id), - ); - return Err(err.change_context(TrustedServerError::EdgeCookie { - message: "Failed to persist generated EC ID to KV identity graph" - .to_string(), - })); - } - } - } - - self.ec_value = Some(ec_id); - self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); - return Ok(()); + let ec_id = generation::generate_ec_id(settings, client_ip)?; + log::info!("Generated new EC ID: {}", log_id(&ec_id)); + let now = current_timestamp(); + let mut entry = KvEntry::new( + &self.consent, + self.geo_info.as_ref(), + now, + &settings.publisher.domain, + ); + entry.device = self + .device_signals + .as_ref() + .map(DeviceSignals::to_kv_device); + + if kv.is_some() { + // The row is created by the navigation's single page-load write in + // finalize; a collision there leaves this EC unissued. + self.pending_entry = Some(Box::new(entry)); + self.kv_snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; } - - Err(Report::new(TrustedServerError::EdgeCookie { - message: format!( - "Failed to allocate a unique EC ID after {MAX_CREATE_ATTEMPTS} attempts" - ), - })) + self.ec_value = Some(ec_id); + self.ec_generated = true; + self.pull_sync_marker.invalidate_for_replaced_ec(); + Ok(()) } /// Returns the EC ID value, if present (either from request or generated). @@ -609,6 +611,37 @@ impl EcContext { self.pull_sync_marker = state; } + /// Stores the navigation permit issued by the router. + pub fn set_navigation_permit(&mut self, permit: NavigationPermit) { + self.navigation_permit = PermitSlot(Some(permit)); + } + + /// Takes the navigation permit; returns `None` after the first call. + pub(crate) fn take_navigation_permit(&mut self) -> Option { + self.navigation_permit.0.take() + } + + /// Takes the navigation permit, for adapter tests. + #[cfg(any(test, feature = "test-utils"))] + pub fn take_navigation_permit_for_test(&mut self) -> Option { + self.take_navigation_permit() + } + + /// Stores validated provider lookup results for finalize. + pub fn set_lookup_results(&mut self, results: LookupResults) { + self.lookup_results = results; + } + + /// Provider lookup results collected for this navigation. + pub(crate) fn lookup_results(&self) -> &LookupResults { + &self.lookup_results + } + + /// Takes the row built by [`Self::generate_if_needed`]. + pub(crate) fn take_pending_entry(&mut self) -> Option { + self.pending_entry.take().map(|entry| *entry) + } + /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); @@ -667,6 +700,9 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -692,6 +728,9 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -720,6 +759,9 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } } @@ -768,69 +810,9 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - use crate::ec::kv_backend::{ - EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, - }; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; - /// [`EcKvStore`] wrapper whose first `collisions` `Add` writes report a - /// precondition failure, forcing generation to retry with a fresh suffix. - struct AddCollidingEcKv { - inner: InMemoryEcKv, - collisions_remaining: std::sync::Mutex, - } - - impl AddCollidingEcKv { - fn new(collisions: u32) -> Self { - Self { - inner: InMemoryEcKv::new("add-colliding-store"), - collisions_remaining: std::sync::Mutex::new(collisions), - } - } - } - - impl EcKvStore for AddCollidingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::Add) { - let mut remaining = self - .collisions_remaining - .lock() - .expect("should lock collision counter"); - if *remaining > 0 { - *remaining -= 1; - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - fn granting_consent() -> ConsentContext { ConsentContext { jurisdiction: Jurisdiction::NonRegulated, @@ -840,37 +822,48 @@ mod tests { } #[test] - fn generate_if_needed_retries_id_collision_then_persists() { + fn generate_if_needed_defers_row_creation_to_the_page_load_write() { let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.5".to_owned())); - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(2)); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + let graph = KvIdentityGraph::in_memory("gen"); - ec.generate_if_needed(&settings, Some(&graph)) - .expect("should generate after bounded collisions"); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint EC"); - assert!(ec.ec_value().is_some(), "should allocate a fresh EC ID"); - assert!(ec.ec_generated(), "should mark the EC as generated"); + let ec_id = ec_context.ec_value().expect("should set EC").to_owned(); + assert!( + graph.get(&ec_id).expect("should read").is_none(), + "should not write during generation" + ); + assert!( + ec_context.take_pending_entry().is_some(), + "should hold the new row for finalize" + ); assert!( - matches!(ec.kv_snapshot(), EcKvSnapshot::Present { .. }), - "generation should seed a present snapshot" + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }), + "should record missing row" ); } #[test] - fn generate_if_needed_errors_after_collision_exhaustion() { - let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.6".to_owned())); - // Collide on every attempt so the bounded retry is exhausted. - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(u32::MAX)); + fn cloned_context_never_carries_the_navigation_permit() { + let mut ec_context = EcContext::default(); + ec_context.set_navigation_permit(NavigationPermit::for_test()); - let result = ec.generate_if_needed(&settings, Some(&graph)); + let mut clone = ec_context.clone(); - assert!(result.is_err(), "should fail after exhausting attempts"); assert!( - ec.ec_value().is_none() && !ec.ec_generated(), - "must not activate an EC ID it could not persist" + clone.take_navigation_permit().is_none(), + "should drop permit on clone" + ); + assert!( + ec_context.take_navigation_permit().is_some(), + "should keep permit on original" ); } diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs new file mode 100644 index 0000000000..212034c3f4 --- /dev/null +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -0,0 +1,320 @@ +//! The one-KV-write-per-page-load rule, enforced by types. +//! +//! A request classified as a top-level document navigation receives a +//! [`NavigationPermit`]. Once the final response status is known it becomes a +//! [`PageLoadWrite`], which [`KvIdentityGraph::commit_page_load`] consumes. +//! Neither type is `Clone`, so one navigation makes at most one write. +//! +//! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load + +use edgezero_core::body::Body as EdgeBody; +use http::{Method, Request, StatusCode}; + +use super::EcKvSnapshot; +use super::kv_types::KvEntry; + +/// Issued to a request classified as a page-load navigation. +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation: + /// `GET`, `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate`, and no + /// `prefetch` token in `Sec-Purpose`. + /// + /// Call once per request, from the publisher-proxy route only. + #[must_use] + pub fn classify(request: &Request) -> Option { + let headers = request.headers(); + let header_is = |name: &str, expected: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(expected)) + }; + let prefetch = headers + .get("sec-purpose") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split([';', ',']) + .any(|token| token.trim().eq_ignore_ascii_case("prefetch")) + }); + let navigation = request.method() == Method::GET + && header_is("sec-fetch-dest", "document") + && header_is("sec-fetch-mode", "navigate") + && !prefetch; + navigation.then_some(Self(())) + } + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect (301, 302, 303, 307, 308): the redirect + /// target is the page load. Other 3xx statuses, such as 304 Not Modified, + /// are the page load itself and keep the permit. + #[must_use] + pub fn confirm(self, status: StatusCode) -> Option { + let redirect = matches!( + status, + StatusCode::MOVED_PERMANENTLY + | StatusCode::FOUND + | StatusCode::SEE_OTHER + | StatusCode::TEMPORARY_REDIRECT + | StatusCode::PERMANENT_REDIRECT + ); + (!redirect).then_some(PageLoadWrite(())) + } + + /// Builds a permit without a request, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +impl PageLoadWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} +} + +/// KV generation a conditional write expects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Generation(u64); + +impl Generation { + /// Reads the generation captured in `snapshot` for `ec_id`. + /// + /// Returns `None` when the snapshot has no generation, so no blind + /// overwrite can be planned. + #[must_use] + pub fn of(snapshot: &EcKvSnapshot, ec_id: &str) -> Option { + snapshot.generation_for(ec_id).map(Self) + } + + /// The raw generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { + /// The full new row. + entry: KvEntry, + /// Generation the row must still have. + expected: Generation, + }, + /// Replace a live row with a withdrawal tombstone. + Tombstone { + /// Generation the row must still have. + expected: Generation, + }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + /// The row changed since it was read. + #[display("row changed since it was read")] + Conflict, + /// A create found an existing row. + #[display("row already exists")] + AlreadyExists, + /// The KV store rejected the write. + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::KvEntry; + + fn request(headers: &[(&str, &str)]) -> http::Request { + let mut builder = http::Request::builder() + .method(http::Method::GET) + .uri("https://www.example.com/"); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + const DOCUMENT: [(&str, &str); 2] = [ + ("sec-fetch-dest", "document"), + ("sec-fetch-mode", "navigate"), + ]; + + #[test] + fn classify_accepts_top_level_document_navigation() { + assert!( + NavigationPermit::classify(&request(&DOCUMENT)).is_some(), + "should permit navigation" + ); + } + + #[test] + fn classify_rejects_missing_fetch_metadata_iframes_and_prefetch() { + assert!( + NavigationPermit::classify(&request(&[])).is_none(), + "should reject without fetch metadata" + ); + assert!( + NavigationPermit::classify(&request(&[ + ("sec-fetch-dest", "iframe"), + ("sec-fetch-mode", "navigate") + ])) + .is_none(), + "should reject iframe" + ); + let mut prefetch = DOCUMENT.to_vec(); + prefetch.push(("sec-purpose", "prefetch;prerender")); + assert!( + NavigationPermit::classify(&request(&prefetch)).is_none(), + "should reject prefetch" + ); + } + + #[test] + fn confirm_rejects_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::FOUND) + .is_none(), + "should not write on redirect" + ); + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_FOUND) + .is_some(), + "should write on error page" + ); + } + + #[test] + fn confirm_keeps_not_modified_and_drops_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_MODIFIED) + .is_some(), + "should write on 304 Not Modified" + ); + for status in [ + http::StatusCode::FOUND, + http::StatusCode::TEMPORARY_REDIRECT, + ] { + assert!( + NavigationPermit::for_test().confirm(status).is_none(), + "should not write on redirect {status}" + ); + } + } + + fn permit() -> PageLoadWrite { + NavigationPermit::for_test() + .confirm(http::StatusCode::OK) + .expect("should confirm") + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + #[test] + fn commit_create_reports_existing_row() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect("should create row"); + + let error = graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect_err("should refuse second create"); + + assert!( + matches!(error.current_context(), KvWriteError::AlreadyExists), + "should report existing row" + ); + } + + #[test] + fn commit_update_with_stale_generation_conflicts_without_retry() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let snapshot = graph.load_snapshot(&ec_id); + let expected = Generation::of(&snapshot, &ec_id).expect("should read generation"); + graph + .create_or_revive(&ec_id, &live_entry()) + .expect("should no-op on live row"); + graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect("should write at current generation"); + + let error = graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect_err("should reject stale generation"); + + assert!( + matches!(error.current_context(), KvWriteError::Conflict), + "should report conflict" + ); + } + + #[test] + fn commit_tombstone_marks_consent_withdrawn() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let expected = + Generation::of(&graph.load_snapshot(&ec_id), &ec_id).expect("should read generation"); + + let committed = graph + .commit_page_load(permit(), &ec_id, RowChange::Tombstone { expected }) + .expect("should write tombstone"); + + assert!(!committed.consent.ok, "should return tombstone"); + let (stored, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!(!stored.consent.ok, "should persist tombstone"); + } +} diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 3e2fbee219..7be48e1e6d 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,7 +157,10 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } - dedupe_partner_updates(updates) + let mut updates = dedupe_partner_updates(updates); + // Provider-owned sources are written only by their identity provider. + updates.retain(|update| !registry.is_provider_owned(&update.partner_id)); + updates } pub(crate) fn collect_prebid_eid_updates( diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 784c217e51..4aa8617f62 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -560,6 +560,7 @@ mod tests { pull_sync_ttl_sec: ttl_sec, pull_sync_rate_limit: 20, ts_pull_token: Some(Redacted::new("token".to_owned())), + provider_owned: false, } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index c4637b4310..b138ecbe16 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -47,6 +47,9 @@ pub struct PartnerConfig { pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, + /// Whether an enabled identity provider claims this source. Browser + /// ingest skips it and batch sync rejects it. + pub provider_owned: bool, } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -234,6 +237,29 @@ impl PartnerRegistry { self.get(domain) } + /// Marks `source` as owned by an enabled identity provider. + /// + /// Returns `false` when the source has no `[[ec.partners]]` entry. + pub fn mark_provider_owned(&mut self, source: &str) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.provider_owned = true; + true + } + None => false, + } + } + + /// Whether an enabled identity provider owns `source`. + #[must_use] + pub fn is_provider_owned(&self, source: &str) -> bool { + self.find_by_source_domain(source) + .is_some_and(|partner| partner.provider_owned) + } + /// Returns all partners with `pull_sync_enabled = true`. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { @@ -305,6 +331,7 @@ fn build_partner_config( pull_sync_ttl_sec: partner.pull_sync_ttl_sec, pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), + provider_owned: false, } } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs new file mode 100644 index 0000000000..075b14b59e --- /dev/null +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -0,0 +1,354 @@ +//! Plans a navigation's single row change. +//! +//! Combines browser ingest (fill missing, never provider-owned sources) and +//! validated provider results (replace the provider's sources), prunes +//! expired UIDs and enforces [`MAX_ENTRY_BYTES`]. Pure: no I/O. + +use super::EcKvSnapshot; +use super::coordinator::results::LookupResults; +use super::eid_types::{SourceDomain, UnixSeconds}; +use super::kv::PartnerIdUpdate; +use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; +use super::page_load::{Generation, RowChange}; +use super::registry::PartnerRegistry; + +/// Inputs to [`plan_write`]. +#[derive(Debug, Clone, Copy)] +pub struct WritePlanInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read for this request. + pub snapshot: &'a EcKvSnapshot, + /// The new row when this navigation creates one (new EC or recovery). + pub new_entry: Option<&'a KvEntry>, + /// Browser updates from `ts-eids` and `sharedId`, already reduced by the + /// ingest rules and consent-gated. + pub browser_ingest: &'a [PartnerIdUpdate], + /// Validated provider results. + pub lookups: &'a LookupResults, + /// Partner registry, for provider ownership. + pub registry: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Plans the navigation's single row change. +/// +/// Returns `None` when nothing would change, when the row is a tombstone, or +/// when an update has no generation to guard it. +#[must_use] +pub fn plan_write(input: &WritePlanInput<'_>) -> Option { + let (base, expected) = match input.new_entry { + Some(entry) => (entry.clone(), None), + None => { + let entry = input.snapshot.entry_for(input.ec_id)?; + if !entry.consent.ok { + return None; + } + let Some(expected) = Generation::of(input.snapshot, input.ec_id) else { + log::info!( + "EC write planner: skipping update for '{}' (no_generation)", + super::log_id(input.ec_id) + ); + return None; + }; + (entry.clone(), Some(expected)) + } + }; + + let mut entry = base.clone(); + apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); + let written = apply_lookups(&mut entry, input.lookups, input.now); + for record in entry.ids.values_mut() { + record.prune_expired(input.now); + } + entry.ids.retain(|_, record| !record.uid.is_empty()); + enforce_size_cap(&mut entry, &written); + + match expected { + None => Some(RowChange::Create(entry)), + Some(_) if entry == base => None, + Some(expected) => Some(RowChange::Update { entry, expected }), + } +} + +fn apply_browser_ingest( + entry: &mut KvEntry, + updates: &[PartnerIdUpdate], + registry: &PartnerRegistry, +) { + for update in updates { + if registry.is_provider_owned(&update.partner_id) { + continue; + } + entry + .ids + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + } +} + +fn apply_lookups( + entry: &mut KvEntry, + lookups: &LookupResults, + now: UnixSeconds, +) -> Vec { + let mut written = Vec::new(); + for lookup in lookups.iter() { + for (source, eids) in &lookup.entries { + if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { + entry.ids.insert(source.as_str().to_owned(), record); + written.push(source.clone()); + } + } + } + written +} + +fn serialized_len(entry: &KvEntry) -> usize { + serde_json::to_vec(entry).map_or(usize::MAX, |bytes| bytes.len()) +} + +fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { + while serialized_len(entry) > MAX_ENTRY_BYTES { + let Some(source) = written.iter().rev().find(|source| { + entry + .ids + .get(source.as_str()) + .is_some_and(KvPartnerId::is_provider_written) + }) else { + log::warn!("EC write planner: row exceeds size cap with no provider UIDs left to drop"); + return; + }; + let record = entry + .ids + .get_mut(source.as_str()) + .expect("should hold a written source"); + record.drop_last_uid(); + if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn partner(source: &str) -> EcPartner { + serde_json::from_value(serde_json::json!({ "name": source, "source_domain": source, "bidstream_enabled": true })) + .expect("should build partner") + } + + fn registry(owned: &[&str]) -> PartnerRegistry { + let mut registry = PartnerRegistry::from_config(&[ + partner("browser.example.com"), + partner("ids.example.com"), + ]) + .expect("should build registry"); + for source in owned { + assert!(registry.mark_provider_owned(source), "should mark {source}"); + } + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn present(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("plan"); + graph.create(EC_ID, entry).expect("should seed row"); + graph.load_snapshot(EC_ID) + } + + fn found(source: &str, ids: &[&str]) -> LookupResults { + let uids = ids + .iter() + .map(|id| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + LookupResults::new(vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), uids)], + )], + }]) + } + + fn input<'a>( + snapshot: &'a EcKvSnapshot, + new_entry: Option<&'a KvEntry>, + ingest: &'a [PartnerIdUpdate], + lookups: &'a LookupResults, + registry: &'a PartnerRegistry, + ) -> WritePlanInput<'a> { + WritePlanInput { + ec_id: EC_ID, + snapshot, + new_entry, + browser_ingest: ingest, + lookups, + registry, + now: UnixSeconds::new(2000), + } + } + + #[test] + fn new_ec_creates_row_with_ingest_and_lookups() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = [ + PartnerIdUpdate::new("browser.example.com", "b-1"), + PartnerIdUpdate::new("ids.example.com", "client"), + ]; + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert_eq!( + entry.ids["browser.example.com"].uid, "b-1", + "should ingest browser uid" + ); + assert_eq!( + entry.ids["ids.example.com"].uid, "p-1", + "should store provider uid, not client uid" + ); + } + + #[test] + fn unchanged_returning_row_plans_no_write() { + let registry = registry(&[]); + let mut entry = live_entry(); + entry + .ids + .insert("browser.example.com".to_owned(), KvPartnerId::single("b-1")); + let snapshot = present(&entry); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-2")]; + let lookups = LookupResults::default(); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn lookup_replaces_provider_source_with_cas_update() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1", "p-2"]); + + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert_eq!( + entry.ids["ids.example.com"].eids()[0].uids().len(), + 2, + "should keep both provider uids" + ); + } + + #[test] + fn tombstoned_row_plans_nothing() { + let registry = registry(&[]); + let snapshot = present(&KvEntry::tombstone(1000)); + let lookups = found("ids.example.com", &["p-1"]); + assert!( + plan_write(&input(&snapshot, None, &[], &lookups, ®istry)).is_none(), + "should never write over a tombstone" + ); + } + + #[test] + fn snapshot_without_generation_plans_nothing() { + let registry = registry(&[]); + let snapshot = EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(live_entry()), + generation: None, + }; + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let lookups = LookupResults::default(); + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite blindly" + ); + } + + #[test] + fn oversized_row_drops_provider_uids_until_it_fits() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let big: Vec = (0..40) + .map(|i| format!("{i:03}{}", "x".repeat(500))) + .collect(); + let ids: Vec<&str> = big.iter().map(String::as_str).collect(); + let lookups = found("ids.example.com", &ids); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + let kept = entry.ids["ids.example.com"].eids()[0].uids(); + assert!(kept.len() < 40, "should drop some uids"); + assert_eq!(kept[0].id().as_str(), ids[0], "should drop from the end"); + } +} diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 15a6486fd8..fc40711240 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -14,7 +14,6 @@ use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; -use crate::http_util::is_navigation_request; use crate::platform::RuntimeServices; use crate::settings::Settings; use crate::streaming_processor::StreamProcessor; @@ -1018,7 +1017,8 @@ impl IntegrationRegistry { /// Dispatch a proxy request when an integration handles the path. /// /// This method removes any caller-supplied `x-ts-ec` before proxying. - /// Response-side cookie mutation is centralized in EC finalize. + /// Response-side cookie mutation is centralized in EC finalize. It never + /// mints an EC: only a publisher navigation may create the EC row. #[must_use] pub async fn handle_proxy( &self, @@ -1028,24 +1028,14 @@ impl IntegrationRegistry { method, path, settings, - kv, - ec_context, + kv: _, + ec_context: _, services, mut req, } = input; if let Some((proxy, _)) = self.find_route(method, path) { - // Organic proxy handler: generate if needed (best effort). - // Only generate for document navigations — subresource requests - // may lack consent signals such as the Sec-GPC header. - if is_navigation_request(&req) { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { - log::warn!("EC generation failed for integration proxy: {err:?}"); - } - } else { - log::debug!( - "EC generation skipped for integration proxy: non-document request (path={path})", - ); - } + // Integration proxies never mint an EC: only the publisher + // navigation may create the row (one write per page load). // Remove any caller-supplied EC header rather than forwarding it. req.headers_mut().remove(HEADER_X_TS_EC.clone()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a670d31416..b3e2e6e20d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -16350,6 +16350,9 @@ mod tests { scheduling_consent() }; let mut ec_context = EcContext::new_for_test(Some(identity.clone()), consent); + // A publisher page load: only a navigation may write the tombstone. + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert_eq!( ec_context.ec_allowed(), !withdrawn, From 3f36d00d409205734b24515bf160c6e84b5eb139 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:31:32 +0530 Subject: [PATCH 03/58] Collect identity provider lookups on publisher navigations Add the IdentityProvider/ProviderKind interface and [[ec.identity_providers]] config, response validation, a deadline-bounded LookupTransport over select_until, the IdentityCoordinator with consent, allowlist and budget checks, the trusted-server-identity-providers crate (no kinds yet), and Fastly wiring that dispatches before the origin fetch and collects before the single page-load write. Signed-off-by: dhruv8sh --- .cargo/config.toml | 4 +- Cargo.lock | 10 + Cargo.toml | 2 + crates/trusted-server-adapter-axum/src/app.rs | 3 +- .../src/app.rs | 2 +- .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/app.rs | 369 +++++- .../trusted-server-adapter-fastly/src/main.rs | 38 +- crates/trusted-server-adapter-spin/src/app.rs | 2 +- crates/trusted-server-core/src/config.rs | 160 ++- .../trusted-server-core/src/config_payload.rs | 41 + crates/trusted-server-core/src/consent/mod.rs | 37 + .../src/ec/coordinator/mod.rs | 1064 ++++++++++++++++- .../src/ec/coordinator/transport.rs | 459 +++++++ .../src/ec/coordinator/validate.rs | 354 ++++++ crates/trusted-server-core/src/ec/mod.rs | 2 + crates/trusted-server-core/src/ec/provider.rs | 681 +++++++++++ .../src/ec/pull_sync_marker.rs | 4 +- crates/trusted-server-core/src/ec/registry.rs | 51 +- crates/trusted-server-core/src/publisher.rs | 23 +- crates/trusted-server-core/src/settings.rs | 117 ++ .../Cargo.toml | 21 + .../src/lib.rs | 146 +++ docs/guide/configuration.md | 49 + trusted-server.example.toml | 23 + 25 files changed, 3635 insertions(+), 28 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/coordinator/transport.rs create mode 100644 crates/trusted-server-core/src/ec/coordinator/validate.rs create mode 100644 crates/trusted-server-core/src/ec/provider.rs create mode 100644 crates/trusted-server-identity-providers/Cargo.toml create mode 100644 crates/trusted-server-identity-providers/src/lib.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index aea0df2d59..0c81704c8b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -28,8 +28,8 @@ test_details = "test --target aarch64-apple-darwin" # (native) are simply not listed. build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. diff --git a/Cargo.lock b/Cargo.lock index 70316712ac..d9cf35f9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6604,6 +6604,7 @@ dependencies = [ "serde_json", "toml", "trusted-server-core", + "trusted-server-identity-providers", "url", "urlencoding", "web-time", @@ -6737,6 +6738,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "trusted-server-identity-providers" +version = "0.1.0" +dependencies = [ + "error-stack", + "toml", + "trusted-server-core", +] + [[package]] name = "trusted-server-integration-tests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 139cf4f610..f431012308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/trusted-server-adapter-spin", "crates/trusted-server-cli", "crates/trusted-server-core", + "crates/trusted-server-identity-providers", "crates/trusted-server-integration-tests", "crates/trusted-server-js", "crates/trusted-server-openrtb", @@ -121,6 +122,7 @@ toml_edit = "0.23.10" tower = "0.4" tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } +trusted-server-identity-providers = { path = "crates/trusted-server-identity-providers" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } url = "2.5.8" diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 0b1858e208..314580e79a 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -499,8 +499,7 @@ fn named_route_handler( Ok(admin_ec_lookup_not_supported()) } NamedRouteHandler::AdminEidsLookup => { - let partner_registry = - PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 668b025ccb..54029b36c7 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -655,7 +655,7 @@ fn build_router(state: &Arc) -> RouterService { .get( "/_ts/admin/eids", make_handler(Arc::clone(&state), |s, _services, req| async move { - let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&s.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) }), ) diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 6c7121d0d6..0e5d6aaceb 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -37,6 +37,7 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-identity-providers = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } web-time = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 7f75267188..3b454d9725 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -86,6 +86,8 @@ //! [`load_settings_from_config_store`]; if settings loading itself fails, they //! are returned without geo or TS headers. +use std::cell::RefCell; +use std::net::IpAddr; use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; @@ -111,12 +113,18 @@ use trusted_server_core::ec::admin::{ }; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, PendingLookups, ProviderCoordinator, +}; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::ec::{EcContext, EidSyncSource}; +use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ @@ -124,7 +132,8 @@ use trusted_server_core::integrations::{ RequestFilterRegistryOutcome, }; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, PlatformPendingRequest, + RuntimeServices, StoreName, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, @@ -143,6 +152,7 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; +use trusted_server_identity_providers::build_coordinator; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -335,6 +345,95 @@ pub(crate) struct EcFinalizeState { pub(crate) services: RuntimeServices, } +/// Identity lookups dispatched for this navigation, collected before finalize. +/// +/// In-flight platform requests are not `Send`, so they cannot ride in +/// response extensions with [`EcFinalizeState`]. They travel through a +/// request-local slot instead: the entry point clears it before routing and +/// takes it after the router returns. +pub(crate) struct IdentityLookups { + pub(crate) coordinator: ProviderCoordinator, + pub(crate) pending: PendingLookups, +} + +thread_local! { + static PENDING_IDENTITY_LOOKUPS: RefCell> = const { RefCell::new(None) }; +} + +/// Stores this request's dispatched identity lookups for the entry point. +fn stash_identity_lookups(lookups: IdentityLookups) { + PENDING_IDENTITY_LOOKUPS.with(|slot| *slot.borrow_mut() = Some(lookups)); +} + +/// Takes the identity lookups dispatched for the current request, if any. +/// +/// The entry point calls this before routing (discarding anything a prior +/// request left behind) and after the router returns. +pub(crate) fn take_identity_lookups() -> Option { + PENDING_IDENTITY_LOOKUPS.with(|slot| slot.borrow_mut().take()) +} + +/// Starts identity-provider lookups for a permitted publisher navigation. +/// +/// Returns `None` when no provider is enabled, the request has no EC, or the +/// identity graph is unavailable. A provider configuration that cannot be +/// built is logged and disables lookups for this request. +async fn dispatch_identity_lookups( + state: &AppState, + services: &RuntimeServices, + req: &Request, + ec: &mut EcRequestState, +) -> Option { + let coordinator = match build_coordinator(&state.settings) { + Ok(Some(coordinator)) => coordinator, + Ok(None) => return None, + Err(report) => { + log::error!("Identity providers disabled for this request: {report:?}"); + return None; + } + }; + let graph = ec.kv_graph.as_ref()?; + let ec_id = ec.ec_context.ec_value()?.to_owned(); + if matches!(ec.ec_context.kv_snapshot(), EcKvSnapshot::NotRead) { + let snapshot = graph.load_snapshot(&ec_id); + ec.ec_context.set_kv_snapshot(snapshot); + } + let transport = PlatformLookupTransport::new(services, state.settings.proxy.certificate_check); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let client_ip = ec + .ec_context + .client_ip() + .and_then(|ip| ip.parse::().ok()); + let user_agent = req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()); + let pending = coordinator + .dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: ec.ec_context.kv_snapshot(), + consent: ec.ec_context.consent(), + client: ClientSignals { + ip: client_ip, + user_agent, + }, + publisher_domain: &state.settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + ) + .await; + Some(IdentityLookups { + coordinator, + pending, + }) +} + /// Per-request EC identity state built before dispatch, mirroring the /// pre-routing prelude of the legacy `route_request` (device signals, bot /// gate, cookie capture, consent/geo-aware [`EcContext`], and KV graph @@ -580,7 +679,7 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = PartnerRegistry::from_config(&state.settings.ec.partners) + let response = PartnerRegistry::from_ec(&state.settings.ec) .and_then(|registry| match handler { NamedRouteHandler::AdminEcLookup => { // Deliberately do not use an EC request-state graph: that @@ -661,7 +760,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; handle_identify( &state.settings, &kv, @@ -675,7 +774,7 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -699,7 +798,7 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -746,7 +845,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -849,12 +948,14 @@ async fn dispatch_fallback( // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. let is_navigation = is_navigation_request(&req); + let mut has_permit = false; if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); // The page-load write permit uses the stricter Fetch Metadata test // (spec §3.1); requests without it never write KV. if let Some(permit) = NavigationPermit::classify(&req) { ec.ec_context.set_navigation_permit(permit); + has_permit = true; } } let is_publisher_navigation = ec.is_real_browser && is_navigation; @@ -866,6 +967,15 @@ async fn dispatch_fallback( log::warn!("EC generation failed for publisher proxy: {err:?}"); } + // Identity lookups start before the origin fetch so they overlap it; + // the entry point collects them before EC finalization. + if is_publisher_navigation + && has_permit + && let Some(lookups) = dispatch_identity_lookups(state, services, &req, &mut ec).await + { + stash_identity_lookups(lookups); + } + // Run the server-side auction with the configured creative- // opportunity slots and collect dispatched bids from the lazy // publisher body stream. `handle_publisher_request` matches the @@ -873,7 +983,7 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_config(&state.settings.ec.partners) { + match PartnerRegistry::from_ec(&state.settings.ec) { Ok(partner_registry) => { let auction = AuctionDispatch { integration_registry: &state.registry, @@ -1427,8 +1537,24 @@ mod tests { use error_stack::Report; use futures::StreamExt as _; use futures::executor::block_on; + use std::borrow::Cow; + use std::num::NonZeroUsize; + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::consent::jurisdiction::Jurisdiction; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; + use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, ProviderCoordinator, RegisteredProvider, + }; use trusted_server_core::ec::device::DeviceSignals; + use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; + use trusted_server_core::ec::provider::{ + ClientSignals, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, + ProviderResponse, ProviderUid, SourceCapability, SourceInput, + }; + use trusted_server_core::ec::rate_limiter::RateLimiter; + use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, @@ -1441,7 +1567,7 @@ mod tests { RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; - use trusted_server_core::settings::Settings; + use trusted_server_core::settings::{EcPartner, IdentityProviderConfig, Settings}; use trusted_server_core::test_support::nextjs_auction; #[test] @@ -2612,6 +2738,231 @@ mod tests { ); } + #[test] + fn publisher_navigation_without_providers_dispatches_nothing() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let _state = take_finalize_state(&mut response); + assert!( + super::take_identity_lookups().is_none(), + "should dispatch no identity lookups without enabled providers" + ); + } + + /// Returns `p-1` for the lookup it was sent, then reports nothing left. + struct LookupHttpClient; + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for LookupHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + Ok(PlatformPendingRequest::new(()).with_backend_name(request.backend_name)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn select_until( + &self, + mut pending_requests: Vec, + _deadline: web_time::Instant, + ) -> Result, Report> { + let Some(ready) = pending_requests.pop() else { + return Ok(None); + }; + let backend_name = ready + .backend_name() + .expect("should carry the backend name") + .to_owned(); + let response = edgezero_core::http::response_builder() + .status(StatusCode::OK) + .body(Body::from("p-1")) + .map_err(|_| Report::new(PlatformError::HttpClient))?; + Ok(Some(PlatformSelectResult { + ready: Ok(PlatformResponse::new(response).with_backend_name(backend_name)), + remaining: pending_requests, + failed_backend_name: None, + })) + } + } + + /// Looks up `ids.example.com` and returns the response body as its UID. + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: Method::GET, + url: url::Url::parse("https://ids.example.com/lookup").expect("should parse"), + headers: edgezero_core::http::HeaderMap::new(), + body: Bytes::new(), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + }], + }])) + } + } + + struct AllowAll; + + impl RateLimiter for AllowAll { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(false) + } + } + + #[test] + fn collected_results_reach_the_page_load_write() { + let provider_config: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider config"); + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + let mut settings = test_settings(); + settings.ec.ec_store = Some("ec_identity_store".to_owned()); + settings.ec.partners = vec![partner]; + settings.ec.identity_providers = vec![provider_config.clone()]; + let router = TrustedServerApp::routes_for_state(&app_state_for_settings(settings.clone())); + let mut request = navigation_request("/article"); + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + let mut response = route(&router, request); + let mut state = take_finalize_state(&mut response); + let ec_id = state + .ec_context + .ec_value() + .expect("should generate an EC") + .to_owned(); + state.services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(LookupHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let provider: Box = Box::new(EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse source"), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + })); + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config, provider)], + Arc::new(PartnerRegistry::from_ec(&settings.ec).expect("should build registry")), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = PlatformLookupTransport::new(&state.services, false); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }; + let pending = block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: state.ec_context.kv_snapshot(), + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "test-publisher.com", + rate_limiter: &AllowAll, + now: UnixSeconds::new(1_700_000_000), + }, + )); + + let result = crate::apply_edgezero_ec_finalize( + &settings, + &mut state, + Some(super::IdentityLookups { + coordinator, + pending, + }), + &mut response, + ); + + result.expect("should finalize"); + assert!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| value.to_str().is_ok_and(|cookie| cookie.contains(&ec_id))), + "should set the EC cookie after the page-load write" + ); + let graph = crate::require_identity_graph(&settings).expect("should open identity graph"); + let snapshot = graph.load_snapshot(&ec_id); + let entry = snapshot.entry_for(&ec_id).expect("should store the row"); + let row = serde_json::to_string(entry).expect("should serialize row"); + assert!( + row.contains("ids.example.com") && row.contains("p-1"), + "should write the provider EID into the row: {row}" + ); + } + #[test] fn publisher_request_without_fetch_metadata_navigation_gets_no_permit() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 3cc13fdfa8..a6ac48f369 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,6 +13,8 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::coordinator::IdentityCoordinator as _; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -44,8 +46,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, + AppState, EcFinalizeState, IdentityLookups, RuntimeStoreConfig, TrustedServerApp, + load_settings_from_config_store, take_identity_lookups, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -491,6 +493,8 @@ fn edgezero_main( core_req.extensions_mut().insert(device_signals); } core_req.extensions_mut().insert(client_info); + // Drop lookups a previous request on this sandbox left behind. + drop(take_identity_lookups()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -517,6 +521,9 @@ fn edgezero_main( // Pop response extensions before the Fastly conversion, which drops them. let ec_state = response.extensions_mut().remove::(); + // Identity lookups dispatched on a publisher navigation are collected in + // EC finalization, after the router's executor has returned. + let identity = take_identity_lookups(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); @@ -544,7 +551,7 @@ fn edgezero_main( if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize(settings, &mut ec_state, identity, &mut response) { Ok(partner_registry) => { send_edgezero_response( response, @@ -563,7 +570,12 @@ fn edgezero_main( } else { match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -636,12 +648,28 @@ fn apply_entry_point_finalize_headers( apply_finalize_headers(settings, geo_info.as_ref(), response); } +/// Runs EC finalization for a routed response. +/// +/// Collects `identity` lookups first. This must run outside the router's +/// `block_on`: collection drives its own executor, and nesting executors +/// panics. fn apply_edgezero_ec_finalize( settings: &Settings, ec_state: &mut EcFinalizeState, + identity: Option, response: &mut HttpResponse, ) -> Result> { - let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; + if let Some(IdentityLookups { + coordinator, + pending, + }) = identity + { + let transport = + PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + ec_state.ec_context.set_lookup_results(results); + } + let partner_registry = PartnerRegistry::from_ec(&settings.ec)?; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9edce69fa3..f5fec2efc7 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -653,7 +653,7 @@ fn build_router(state: &Arc) -> RouterService { let s = Arc::clone(&s); async move { let req = ctx.into_request(); - let result = PartnerRegistry::from_config(&s.settings.ec.partners) + let result = PartnerRegistry::from_ec(&s.settings.ec) .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); Ok::(result.unwrap_or_else(|e| http_error(&e))) } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 059d500092..85bfd5cf0f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -13,6 +13,7 @@ use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; +use crate::ec::provider::validate_enabled_configs; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ @@ -32,7 +33,7 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; +use crate::settings::{AssetOriginAuth, Ec, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] @@ -167,6 +168,16 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { ], true, ), + field( + vec![ + object("ec"), + optional_object("identity_providers"), + SecretPathSegment::ArrayEach, + optional_object("secrets"), + object("token"), + ], + true, + ), field( vec![ object("handlers"), @@ -268,6 +279,7 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Result<(), Report> { + if ec.identity_providers.is_empty() { + return Ok(()); + } + let partners = PartnerRegistry::from_ec(ec)?; + validate_enabled_configs(&ec.identity_providers, &partners).map_err(|report| { + let detail = report + .frames() + .filter_map(|frame| frame.downcast_ref::()) + .map(String::as_str) + .collect::>() + .join("; "); + report.change_context(TrustedServerError::Configuration { + message: format!("ec.identity_providers: {detail}"), + }) + }) +} + fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { let Some(raw_config) = settings.integrations.get(JS_ASSET_PROXY_INTEGRATION_ID) else { return Ok(()); @@ -447,6 +482,15 @@ fn validate_secret_key_references(settings: &Settings) -> Result<(), Report Settings { + let mut settings = valid_settings(); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ]; + let mut config = serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + }); + if let (Some(base), serde_json::Value::Object(overrides)) = + (config.as_object_mut(), provider) + { + base.extend(overrides); + } + settings.ec.identity_providers = + vec![serde_json::from_value(config).expect("should build provider config")]; + settings + } + + #[test] + fn runtime_validation_accepts_valid_identity_provider() { + let settings = settings_with_identity_provider(serde_json::json!({})); + + validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); + } + + #[test] + fn runtime_validation_rejects_provider_endpoint_outside_allowlist() { + let settings = settings_with_identity_provider(serde_json::json!({ + "endpoint": "https://other.example.com/v1/lookup" + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject endpoint outside allowlist"); + + assert!( + err.to_string().contains("allowed_domains"), + "should explain the allowlist rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_provider_source_without_partner() { + let settings = settings_with_identity_provider(serde_json::json!({ + "sources": ["unknown.example.com"] + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject source without partner"); + + assert!( + err.to_string().contains("[[ec.partners]]"), + "should explain the partner rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_source_claimed_by_two_providers() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + let mut second = settings.ec.identity_providers[0].clone(); + second.id = "second-provider".to_owned(); + settings.ec.identity_providers.push(second); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject double-claimed source"); + + assert!( + err.to_string() + .contains("claimed by another enabled provider"), + "should explain the claim rule: {err:?}" + ); + } + + #[test] + fn deploy_validation_rejects_empty_provider_token_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": " " } + })); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject empty provider token reference"); + + assert!( + err.to_string() + .contains("ec.identity_providers[0].secrets.token"), + "should identify the provider token reference: {err:?}" + ); + } + + #[test] + fn app_config_keeps_provider_token_key_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": "example_provider_token" } + })); + + let app_config = + TrustedServerAppConfig::new(settings).expect("should validate provider token key name"); + + let serialized = serde_json::to_string(&app_config).expect("should serialize app config"); + assert!( + serialized.contains("example_provider_token"), + "should keep the provider token key reference" + ); + } + #[test] fn trace_config_requires_enabled_on_both_validation_paths() { for raw in [ diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 34a4143cf2..dd7545198b 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -205,6 +205,7 @@ mod tests { "s3-session-key" => "resolved-session-token", "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", + "provider-token-key" => "resolved-provider-token", "trusted-client-ip-key" => "resolved-trusted-client-ip-secret-32-bytes", "trusted-forwarder-key" => "resolved-trusted-forwarder-secret-32-bytes", _ => key, @@ -332,6 +333,46 @@ mod tests { ); } + #[test] + fn resolves_identity_provider_token_from_the_mapped_default_store() { + let mut original = test_settings(); + original.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ); + original.ec.identity_providers.push( + serde_json::from_value(serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "secrets": { "token": "provider-token-key" }, + })) + .expect("should build provider config"), + ); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve the provider token"); + + assert_eq!( + reconstructed.ec.identity_providers[0] + .secrets + .token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-provider-token"), + "should resolve the provider token key" + ); + } + #[test] fn resolves_all_static_credentials_from_the_mapped_default_store() { let mut original = test_settings(); diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 76aa6ad859..83bf3f33db 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -41,6 +41,7 @@ pub use types::{ ConsentContext, ConsentSource, PrivacyFlag, RawConsentSignals, TcfConsent, UsPrivacy, }; +use std::num::NonZeroU16; use std::sync::atomic::{AtomicBool, Ordering}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -287,6 +288,14 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { }) } +/// Whether the effective TCF string grants vendor consent to `vendor_id`. +/// +/// Returns `false` when no TCF string applies. +#[must_use] +pub fn has_tcf_vendor_consent(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + effective_tcf(ctx).is_some_and(|tcf| tcf.has_vendor_consent(vendor_id.get())) +} + /// Returns whether a server-side auction may be dispatched for this request. /// /// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present @@ -685,6 +694,7 @@ mod tests { ConsentPipelineInput, allows_ec_creation, allows_eid_persistence, apply_expiration_check, apply_tcf_conflict_resolution, build_consent_context, build_context_from_signals, consent_allows_server_side_auction, gate_eids_by_consent, has_explicit_ec_withdrawal, + has_tcf_vendor_consent, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -692,6 +702,7 @@ mod tests { }; use crate::consent_config::{ConflictMode, ConsentConfig, ConsentMode}; use crate::cookies::parse_cookies_to_jar; + use std::num::NonZeroU16; fn build_request() -> Request { Request::builder() @@ -755,6 +766,32 @@ mod tests { } } + #[test] + fn tcf_vendor_consent_reads_the_effective_tcf_string() { + let mut tcf = TcfBuilder::new().with_storage(true).build(); + tcf.vendor_consents = vec![9999]; + let context = ConsentContext { + gdpr_applies: true, + tcf: Some(tcf), + ..ConsentContext::default() + }; + let granted = NonZeroU16::new(9999).expect("should be non-zero"); + let other = NonZeroU16::new(1234).expect("should be non-zero"); + + assert!( + has_tcf_vendor_consent(&context, granted), + "should grant consented vendor" + ); + assert!( + !has_tcf_vendor_consent(&context, other), + "should deny other vendor" + ); + assert!( + !has_tcf_vendor_consent(&ConsentContext::default(), granted), + "should deny when no TCF string applies" + ); + } + fn make_tcf(last_updated_ds: u64, allows_eids: bool) -> TcfConsent { TcfBuilder::new() .with_storage(true) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 7fe8da9015..28d9c98caa 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -1,3 +1,1065 @@ -//! Server-side identity lookup coordination. +//! Server-side identity lookup coordination (spec §8). +//! +//! [`ProviderCoordinator`] owns eligibility, consent, destination checks, a +//! deadline-bounded fan-out through a [`LookupTransport`], validation and +//! planning of the navigation's single row change. Providers stay pure. pub mod results; +pub mod transport; +mod validate; + +use std::collections::HashSet; +use std::num::{NonZeroU16, NonZeroU32}; +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; + +use self::results::{LookupResults, LookupStatus, ProviderLookup, SkipReason}; +use self::transport::{ + Deadline, LookupTransport, ProviderIndex, Tagged, TransportError, TransportResponse, +}; +use self::validate::{ValidatedEntries, ValidationPolicy, validate}; +use crate::consent::ConsentContext; +use crate::consent::{allows_eid_persistence, has_tcf_vendor_consent}; +use crate::ec::EcKvSnapshot; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::kv_types::KvEntry; +use crate::ec::page_load::RowChange; +use crate::ec::provider::{ + ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, + ProviderResponse, SourceInput, destination_allowed, parse_sources, validate_enabled_configs, +}; +use crate::ec::rate_limiter::RateLimiter; +use crate::ec::registry::PartnerRegistry; +use crate::ec::write_plan::{WritePlanInput, plan_write}; +use crate::settings::IdentityProviderConfig; + +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget and validation. Providers never see KV, cookies, HTTP or the +/// clock. Implementations never write KV: they return the change and the +/// caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when no provider is enabled; skipped + /// providers are recorded and reported by [`Self::collect`]. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; + + /// Merges browser ingest and lookup results into the navigation's single + /// row change. + /// + /// Returns `None` when the row would not change, so no write is made. + fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; +} + +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read before dispatch. + pub snapshot: &'a EcKvSnapshot, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals lent to providers. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + /// Request time. + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. +/// +/// Only the coordinator module can create a non-empty value. `Default` +/// (empty) and `Debug` (counts only) are implemented by hand so they do not +/// require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// Request time, for validating `exp` and resolving relative expiry. + now: UnixSeconds, + /// Providers that finished at dispatch (skipped or failed to send). + early: Vec>, +} + +impl Default for PendingLookups { + fn default() -> Self { + Self { + requests: Vec::new(), + deadline: None, + now: UnixSeconds::new(0), + early: Vec::new(), + } + } +} + +impl std::fmt::Debug for PendingLookups { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingLookups") + .field("requests", &self.requests.len()) + .field("early", &self.early.len()) + .finish_non_exhaustive() + } +} + +impl PendingLookups { + /// Whether nothing was dispatched or skipped. + #[must_use] + pub fn is_empty(&self) -> bool { + self.requests.is_empty() && self.early.is_empty() + } +} + +/// A configured provider handed to [`ProviderCoordinator::new`]. +pub struct RegisteredProvider

{ + /// The provider's configuration entry. + pub config: IdentityProviderConfig, + /// The built provider. + pub provider: P, + rate_limit_per_hour: Option, +} + +impl

RegisteredProvider

{ + /// Pairs a configuration entry with its built provider. + #[must_use] + pub fn new(config: IdentityProviderConfig, provider: P) -> Self { + Self { + config, + provider, + rate_limit_per_hour: None, + } + } + + /// Applies an hourly per-EC lookup limit (carried over from pull sync). + #[must_use] + pub fn with_rate_limit(mut self, per_hour: NonZeroU32) -> Self { + self.rate_limit_per_hour = Some(per_hour); + self + } +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated when the coordinator is built. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +/// The production coordinator, built per request from settings. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation. + partners: Arc, + budget: Duration, +} + +impl ProviderCoordinator

{ + /// Validates and registers the enabled providers. + /// + /// Disabled providers are dropped: they make no requests. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when an enabled provider's + /// endpoint is not `https` with an allowlisted host, a source has no + /// partner entry, a source is outside the provider's capabilities, a + /// source is claimed by two enabled providers, or more than `u16::MAX` + /// providers are enabled. + pub fn new( + registered: Vec>, + partners: Arc, + budget: Duration, + ) -> Result> { + let enabled: Vec> = registered + .into_iter() + .filter(|registered| registered.config.enabled) + .collect(); + if u16::try_from(enabled.len()).is_err() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("too many identity providers are enabled")); + } + validate_enabled_configs( + enabled.iter().map(|registered| ®istered.config), + &partners, + )?; + + let providers = enabled + .into_iter() + .map(Self::register) + .collect::, Report>>()?; + + Ok(Self { + providers, + partners, + budget, + }) + } + + fn register( + registered: RegisteredProvider

, + ) -> Result, Report> { + let RegisteredProvider { + config, + provider, + rate_limit_per_hour, + } = registered; + let sources = parse_sources(&config)?; + let capable: HashSet<&SourceDomain> = provider + .capabilities() + .sources + .iter() + .map(|capability| &capability.source) + .collect(); + if let Some(source) = sources.iter().find(|source| !capable.contains(source)) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is outside its kind's capabilities", + config.id + ))); + } + Ok(Registered { + id: ProviderId::new(&config.id), + policy: ProviderPolicy { + sources, + allowed_domains: config + .allowed_domains + .iter() + .map(|domain| Box::from(domain.as_str())) + .collect(), + tcf_vendor_id: config.tcf_vendor_id, + rate_limit_per_hour, + }, + provider, + }) + } + + /// Whether no provider is enabled. + #[must_use] + pub fn is_empty(&self) -> bool { + self.providers.is_empty() + } + + /// Claimed sources lacking a live UID, minus sources needing input V1 + /// never has; the flag reports whether any source was removed for input. + fn due_sources( + registered: &Registered

, + entry: Option<&KvEntry>, + now: UnixSeconds, + ) -> (Vec, bool) { + let capabilities = registered.provider.capabilities(); + let mut input_blocked = false; + let mut due = Vec::new(); + for source in ®istered.policy.sources { + let held = entry + .and_then(|entry| entry.ids.get(source.as_str())) + .is_some_and(|record| record.has_live_uid(now)); + if held { + continue; + } + let needs_input = capabilities + .sources + .iter() + .find(|capability| capability.source == *source) + .is_some_and(|capability| capability.input != SourceInput::Anonymous); + if needs_input { + input_blocked = true; + continue; + } + due.push(source.clone()); + } + (due, input_blocked) + } + + /// Whether the provider's hourly per-EC limit is reached. Limiter errors + /// fail open, matching pull sync. + fn rate_limited(registered: &Registered

, lookup: &LookupContext<'_>) -> bool { + let Some(limit) = registered.policy.rate_limit_per_hour else { + return false; + }; + let hash = lookup.ec_id.split('.').next().unwrap_or(lookup.ec_id); + let key = format!("identity:{}:{hash}", registered.id); + match lookup.rate_limiter.exceeded(&key, limit.get()) { + Ok(exceeded) => exceeded, + Err(report) => { + log::warn!( + "Identity lookup: provider={} rate limiter failed, allowing lookup: {report:?}", + registered.id + ); + false + } + } + } + + fn lookup( + &self, + provider: ProviderIndex, + status: LookupStatus, + entries: ValidatedEntries, + ) -> ProviderLookup { + ProviderLookup { + provider: self.providers[provider.get()].id.clone(), + status, + entries, + } + } + + fn interpret( + &self, + provider: ProviderIndex, + response: Result>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let response = match response { + Ok(response) => response, + Err(report) => { + let status = match report.current_context() { + TransportError::Timeout => LookupStatus::Timeout, + TransportError::Send | TransportError::BodyTooLarge => { + log::debug!( + "Identity lookup: provider={} transport failed: {report:?}", + registered.id + ); + LookupStatus::TransportFailed + } + }; + return self.lookup(provider, status, Vec::new()); + } + }; + if matches!( + response.status, + http::StatusCode::NOT_FOUND | http::StatusCode::NO_CONTENT + ) { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + if !response.status.is_success() { + return self.lookup( + provider, + LookupStatus::HttpError(response.status), + Vec::new(), + ); + } + let parsed = registered.provider.parse_response(ProviderResponse { + status: response.status, + headers: &response.headers, + body: &response.body, + }); + let eids = match parsed { + Ok(LookupOutcome::Found(eids)) => eids, + Ok(LookupOutcome::NoMatch) => { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + Err(report) => { + log::debug!( + "Identity lookup: provider={} response not parsed: {}", + registered.id, + report.current_context() + ); + return self.lookup(provider, LookupStatus::InvalidResponse, Vec::new()); + } + }; + let policy = ValidationPolicy { + provider: ®istered.id, + claimed: ®istered.policy.sources, + shape: registered.provider.capabilities().shape, + partners: &self.partners, + now, + }; + let (entries, dropped) = validate(eids, &policy); + let kept = entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + if kept == 0 { + log::info!( + "Identity lookup: provider={} dropped every UID ({} drops)", + registered.id, + dropped.total() + ); + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + let kept = u16::try_from(kept).unwrap_or(u16::MAX); + self.lookup(provider, LookupStatus::Found { kept, dropped }, entries) + } +} + +impl IdentityCoordinator for ProviderCoordinator

{ + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups { + let mut pending = PendingLookups { + now: lookup.now, + ..PendingLookups::default() + }; + if self.providers.is_empty() { + return pending; + } + let entry = lookup.snapshot.entry_for(lookup.ec_id); + // A withdrawal tombstone or missing EID consent skips every provider. + let consented = + allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); + let deadline = Deadline::after(self.budget); + pending.deadline = Some(deadline); + for (index, registered) in self.providers.iter().enumerate() { + let provider = ProviderIndex::new(index); + let skip = |reason| Tagged { + provider, + value: LookupStatus::Skipped(reason), + }; + let vendor_denied = registered + .policy + .tcf_vendor_id + .is_some_and(|vendor| !has_tcf_vendor_consent(lookup.consent, vendor)); + if !consented || vendor_denied { + pending.early.push(skip(SkipReason::Consent)); + continue; + } + let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + if due.is_empty() { + pending.early.push(skip(if input_blocked { + SkipReason::Input + } else { + SkipReason::NotDue + })); + continue; + } + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } + let input = LookupInput { + ec_id: lookup.ec_id, + consent: lookup.consent, + client: lookup.client, + publisher_domain: lookup.publisher_domain, + due_sources: &due, + current: OwnedEntries::new(entry, ®istered.policy.sources), + hashed_email: None, + }; + let request = match registered.provider.build_request(&input) { + Ok(request) => request, + Err(report) => { + log::debug!( + "Identity lookup: provider={} request not built: {}", + registered.id, + report.current_context() + ); + pending.early.push(skip(SkipReason::Input)); + continue; + } + }; + if !destination_allowed(&request.url, ®istered.policy.allowed_domains) { + pending.early.push(skip(SkipReason::Destination)); + continue; + } + match transport.start(request, deadline).await { + Ok(value) => pending.requests.push(Tagged { provider, value }), + Err(report) => { + log::warn!( + "Identity lookup: provider={} send failed: {}", + registered.id, + report.current_context() + ); + pending.early.push(Tagged { + provider, + value: LookupStatus::TransportFailed, + }); + } + } + } + pending + } + + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults { + let PendingLookups { + requests, + deadline, + now, + early, + } = pending; + let responses = match deadline { + Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, + _ => Vec::new(), + }; + let mut slots: Vec> = + (0..self.providers.len()).map(|_| None).collect(); + for Tagged { provider, value } in early { + slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); + } + for Tagged { provider, value } in responses { + slots[provider.get()] = Some(self.interpret(provider, value, now)); + } + let lookups: Vec = slots.into_iter().flatten().collect(); + for lookup in &lookups { + log_outcome(lookup); + } + LookupResults::new(lookups) + } + + fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { + plan_write(input) + } +} + +fn log_outcome(lookup: &ProviderLookup) { + let (kept, dropped) = match lookup.status { + LookupStatus::Found { kept, dropped } => (kept, dropped.total()), + _ => (0, 0), + }; + log::info!( + "Identity lookup: provider={} outcome={} kept={kept} dropped={dropped}", + lookup.provider, + lookup.status.metric_name() + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::jurisdiction::Jurisdiction; + use crate::consent::types::TcfConsent; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::KvPartnerId; + use crate::ec::provider::{ + EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, + ProviderRequest, ProviderUid, SourceCapability, + }; + use crate::error::TrustedServerError; + use crate::settings::EcPartner; + use std::borrow::Cow; + use std::cell::RefCell; + use std::num::NonZeroUsize; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + type Reply = fn() -> Result>; + + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!( + "https://ids.example.com/lookup?due={}", + lookup.due_sources.len() + )) + .expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + }], + }])) + } + } + + /// Answers every request with a canned result, recording the URLs sent. + struct CannedTransport { + reply: Reply, + sent: RefCell>, + } + + impl CannedTransport { + fn new(reply: Reply) -> Self { + Self { + reply, + sent: RefCell::default(), + } + } + } + + impl LookupTransport for CannedTransport { + type Pending = (); + + async fn start( + &self, + request: ProviderRequest, + _deadline: Deadline, + ) -> Result<(), Report> { + self.sent.borrow_mut().push(request.url.to_string()); + Ok(()) + } + + async fn gather( + &self, + pending: Vec>, + _deadline: Deadline, + ) -> Vec>>> { + pending + .into_iter() + .map(|tagged| Tagged { + provider: tagged.provider, + value: (self.reply)(), + }) + .collect() + } + } + + struct Limiter(bool); + + impl RateLimiter for Limiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(self.0) + } + } + + fn reply(status: http::StatusCode, body: &'static [u8]) -> TransportResponse { + TransportResponse { + status, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from_static(body), + } + } + + fn ok_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"p-1")) + } + + fn timeout_reply() -> Result> { + Err(Report::new(TransportError::Timeout)) + } + + fn not_found_reply() -> Result> { + Ok(reply(http::StatusCode::NOT_FOUND, b"")) + } + + fn server_error_reply() -> Result> { + Ok(reply(http::StatusCode::BAD_GATEWAY, b"")) + } + + fn invalid_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"\xff")) + } + + fn partners() -> Arc { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + Arc::new(PartnerRegistry::from_config(&[partner]).expect("should build registry")) + } + + fn provider_config(extra: &str) -> IdentityProviderConfig { + toml::from_str(&format!( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]\n{extra}" + )) + .expect("should parse config") + } + + fn echo_provider(input: SourceInput) -> EchoProvider { + EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + }) + } + + fn coordinator_with( + registered: RegisteredProvider, + ) -> ProviderCoordinator { + ProviderCoordinator::new(vec![registered], partners(), Duration::from_millis(200)) + .expect("should build coordinator") + } + + fn coordinator(extra: &str) -> ProviderCoordinator { + coordinator_with(RegisteredProvider::new( + provider_config(extra), + echo_provider(SourceInput::Anonymous), + )) + } + + fn granting() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + } + } + + fn missing() -> EcKvSnapshot { + EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + } + } + + fn run_with( + coordinator: &ProviderCoordinator, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + limiter: &dyn RateLimiter, + ) -> LookupResults { + let context = LookupContext { + ec_id: EC_ID, + snapshot, + consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + rate_limiter: limiter, + now: UnixSeconds::new(1000), + }; + futures::executor::block_on(async { + let pending = coordinator.dispatch(transport, &context).await; + coordinator.collect(transport, pending).await + }) + } + + fn run( + coordinator: &ProviderCoordinator, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + ) -> LookupResults { + run_with(coordinator, transport, snapshot, consent, &Limiter(false)) + } + + fn only_status(results: &LookupResults) -> LookupStatus { + results + .iter() + .next() + .expect("should report the provider") + .status + } + + fn seeded_snapshot(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("coord"); + graph.create(EC_ID, entry).expect("should seed"); + graph.load_snapshot(EC_ID) + } + + #[test] + fn due_source_is_looked_up_and_validated() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep one UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0] + .owner() + .map(ProviderId::as_str), + Some("example-provider"), + "should record the owner" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + #[test] + fn source_with_live_uid_is_not_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn tombstoned_row_skips_for_consent() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&KvEntry::tombstone(1000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip a tombstoned row" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn missing_tcf_vendor_consent_skips_before_building_a_request() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn granted_tcf_vendor_consent_allows_the_lookup() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + let consent = ConsentContext { + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: vec![9999], + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + }; + + let results = run(&coordinator, &transport, &missing(), &consent); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up with vendor consent" + ); + } + + #[test] + fn hashed_email_sources_are_skipped_for_input() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + echo_provider(SourceInput::HashedEmail), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Input), + "should skip sources needing hashed email" + ); + } + + #[test] + fn rate_limited_provider_is_skipped() { + let coordinator = coordinator_with( + RegisteredProvider::new(provider_config(""), echo_provider(SourceInput::Anonymous)) + .with_rate_limit(NonZeroU32::new(1).expect("should be non-zero")), + ); + let transport = CannedTransport::new(ok_reply); + + let results = run_with( + &coordinator, + &transport, + &missing(), + &granting(), + &Limiter(true), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip a rate-limited provider" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn built_url_outside_allowlist_is_rejected() { + let coordinator = coordinator(""); + // The echo provider builds its URL on the configured host; narrow the + // policy afterwards so the built URL falls outside the allowlist. + let coordinator = ProviderCoordinator { + providers: coordinator + .providers + .into_vec() + .into_iter() + .map(|mut registered| { + registered.policy.allowed_domains = Box::new([Box::from("other.example.com")]); + registered + }) + .collect(), + ..coordinator + }; + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Destination), + "should reject the destination" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn transport_and_http_outcomes_map_to_statuses() { + let cases: [(Reply, LookupStatus); 4] = [ + (timeout_reply, LookupStatus::Timeout), + (not_found_reply, LookupStatus::NoMatch), + ( + server_error_reply, + LookupStatus::HttpError(http::StatusCode::BAD_GATEWAY), + ), + (invalid_reply, LookupStatus::InvalidResponse), + ]; + for (reply, expected) in cases { + let coordinator = coordinator(""); + let transport = CannedTransport::new(reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report"); + assert_eq!(lookup.status, expected, "should map the outcome"); + assert!(lookup.entries.is_empty(), "should carry no entries"); + } + } + + #[test] + fn disabled_provider_is_not_registered() { + let coordinator = coordinator("enabled = false"); + + assert!(coordinator.is_empty(), "should drop disabled providers"); + } + + #[test] + fn endpoint_outside_allowlist_is_rejected_at_build() { + let mut config = provider_config(""); + config.endpoint = "https://other.example.com/".to_owned(); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject endpoint"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn source_outside_capabilities_is_rejected_at_build() { + let provider = EchoProvider(ProviderCapabilities { + sources: Box::new([]), + ..echo_provider(SourceInput::Anonymous).0 + }); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config(""), provider)], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject source outside capabilities"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs new file mode 100644 index 0000000000..e01f2c6a13 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -0,0 +1,459 @@ +//! Sends provider lookups and gathers responses before a shared deadline. + +use std::cell::Cell; +use std::time::Duration; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; + +use crate::ec::provider::ProviderRequest; +use crate::platform::{ + PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, + RuntimeServices, +}; + +/// Largest provider response body accepted. +pub const MAX_PROVIDER_RESPONSE_BYTES: usize = 64 * 1024; + +/// Smallest transport timeout registered for a lookup backend, in +/// milliseconds; the shared deadline still bounds collection. +const MIN_TRANSPORT_TIMEOUT_MS: u32 = 50; + +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + /// The provider's position in the coordinator. + pub provider: ProviderIndex, + /// The labelled value. + pub value: T, +} + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ProviderIndex(u16); + +impl ProviderIndex { + /// Wraps a position; the coordinator rejects more than `u16::MAX` providers. + /// + /// # Panics + /// + /// Panics when `index` does not fit in a `u16`. + pub(crate) fn new(index: usize) -> Self { + Self(u16::try_from(index).expect("should hold fewer than 65536 providers")) + } + + /// The position. + #[must_use] + pub fn get(self) -> usize { + usize::from(self.0) + } +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: http::HeaderMap, + /// Response body, at most [`MAX_PROVIDER_RESPONSE_BYTES`]. + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +impl Deadline { + /// A deadline `budget` from now. + #[must_use] + pub fn after(budget: Duration) -> Self { + Self(web_time::Instant::now() + budget) + } + + /// The deadline instant. + #[must_use] + pub fn instant(self) -> web_time::Instant { + self.0 + } + + /// Time left before the deadline; zero once it has passed. + #[must_use] + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(web_time::Instant::now()) + } +} + +/// Why a lookup produced no response. +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + /// The request could not be sent. + #[display("lookup request could not be sent")] + Send, + /// No response before the deadline. + #[display("lookup did not complete before the deadline")] + Timeout, + /// The body exceeded [`MAX_PROVIDER_RESPONSE_BYTES`]. + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} + +/// [`LookupTransport`] over the platform HTTP client. +/// +/// Each request gets its own backend name (a per-transport sequence number is +/// the backend discriminator), so a ready response maps back to its provider +/// by backend name. Names repeat across requests and the transport timeout is +/// canonicalized, keeping the number of registered dynamic backends bounded. +pub struct PlatformLookupTransport<'s> { + services: &'s RuntimeServices, + certificate_check: bool, + next: Cell, +} + +impl<'s> PlatformLookupTransport<'s> { + /// Borrows the request's runtime services. + #[must_use] + pub fn new(services: &'s RuntimeServices, certificate_check: bool) -> Self { + Self { + services, + certificate_check, + next: Cell::new(0), + } + } + + fn transport_timeout(&self, deadline: Deadline) -> Duration { + let remaining_ms = u32::try_from(deadline.remaining().as_millis()).unwrap_or(u32::MAX); + let canonical = self + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, u32::MAX); + Duration::from_millis(u64::from(canonical.max(MIN_TRANSPORT_TIMEOUT_MS))) + } +} + +impl LookupTransport for PlatformLookupTransport<'_> { + type Pending = PlatformPendingRequest; + + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result> { + let ProviderRequest { + method, + url, + headers, + body, + } = request; + let host = url + .host_str() + .ok_or_else(|| Report::new(TransportError::Send))? + .to_owned(); + let timeout = self.transport_timeout(deadline); + let sequence = self.next.get(); + self.next.set(sequence.wrapping_add(1)); + let backend = self + .services + .backend() + .ensure(&PlatformBackendSpec { + scheme: url.scheme().to_owned(), + host, + port: url.port(), + host_header_override: None, + certificate_check: self.certificate_check, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(format!("ts-identity-{sequence}")), + }) + .change_context(TransportError::Send)?; + let mut builder = http::Request::builder().method(method).uri(url.as_str()); + if let Some(target) = builder.headers_mut() { + *target = headers; + } + let edge_request = builder + .body(EdgeBody::from(body.to_vec())) + .change_context(TransportError::Send)?; + self.services + .http_client() + .send_async(PlatformHttpRequest::new(edge_request, backend).with_cache_bypass()) + .await + .change_context(TransportError::Send) + } + + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>> { + // Names are captured before the first select: some adapters drop + // them from the requests that remain in flight. + let mut outstanding: Vec<(String, ProviderIndex)> = Vec::with_capacity(pending.len()); + let mut remaining = Vec::with_capacity(pending.len()); + let mut results = Vec::with_capacity(pending.len()); + for Tagged { provider, value } in pending { + match value.backend_name() { + Some(name) => { + outstanding.push((name.to_owned(), provider)); + remaining.push(value); + } + None => results.push(Tagged { + provider, + value: Err(Report::new(TransportError::Send)), + }), + } + } + while !remaining.is_empty() { + match self + .services + .http_client() + .select_until(remaining, deadline.instant()) + .await + { + Ok(Some(selected)) => { + remaining = selected.remaining; + let (name, value) = match selected.ready { + Ok(response) => { + (response.backend_name.clone(), read_response(response).await) + } + Err(report) => ( + selected.failed_backend_name, + Err(report.change_context(TransportError::Send)), + ), + }; + match name.and_then(|name| take_provider(&mut outstanding, &name)) { + Some(provider) => results.push(Tagged { provider, value }), + None => { + log::warn!("Identity lookup: dropping response with unknown backend"); + } + } + } + Ok(None) => break, + Err(report) => { + log::warn!("Identity lookup: select failed: {report:?}"); + break; + } + } + } + // Every request not yet answered timed out or was abandoned. + results.extend(outstanding.into_iter().map(|(_, provider)| Tagged { + provider, + value: Err(Report::new(TransportError::Timeout)), + })); + results + } +} + +/// Removes the first outstanding request with backend `name`. +fn take_provider( + outstanding: &mut Vec<(String, ProviderIndex)>, + name: &str, +) -> Option { + let index = outstanding + .iter() + .position(|(candidate, _)| candidate == name)?; + Some(outstanding.remove(index).1) +} + +async fn read_response( + response: PlatformResponse, +) -> Result> { + let (parts, body) = response.response.into_parts(); + let body = match body { + EdgeBody::Once(bytes) => bytes, + stream => stream + .into_bytes_bounded(MAX_PROVIDER_RESPONSE_BYTES) + .await + .change_context(TransportError::BodyTooLarge)?, + }; + if body.len() > MAX_PROVIDER_RESPONSE_BYTES { + return Err(Report::new(TransportError::BodyTooLarge)); + } + Ok(TransportResponse { + status: parts.status, + headers: parts.headers, + body, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use std::sync::Arc; + + fn request(path: &str) -> ProviderRequest { + ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!("https://ids.example.com/{path}")).expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } + } + + fn gather_one( + transport: &PlatformLookupTransport<'_>, + deadline: Deadline, + ) -> Vec>>> { + futures::executor::block_on(async { + let pending = transport + .start(request("a"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![Tagged { + provider: ProviderIndex::new(0), + value: pending, + }], + deadline, + ) + .await + }) + } + + #[test] + fn gather_returns_tagged_responses() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, b"{\"a\":1}".to_vec()); + client.push_response(204, Vec::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + let deadline = Deadline::after(Duration::from_millis(200)); + + let results = futures::executor::block_on(async { + let first = transport + .start(request("a"), deadline) + .await + .expect("should start"); + let second = transport + .start(request("b"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![ + Tagged { + provider: ProviderIndex::new(0), + value: first, + }, + Tagged { + provider: ProviderIndex::new(1), + value: second, + }, + ], + deadline, + ) + .await + }); + + assert_eq!(results.len(), 2, "should return one result per request"); + let first = results + .iter() + .find(|result| result.provider == ProviderIndex::new(0)) + .expect("should tag provider 0"); + let first = first.value.as_ref().expect("should succeed"); + assert_eq!(first.status, http::StatusCode::OK, "should map provider 0"); + assert_eq!(first.body.as_ref(), b"{\"a\":1}", "should carry the body"); + let second = results + .iter() + .find(|result| result.provider == ProviderIndex::new(1)) + .expect("should tag provider 1"); + assert_eq!( + second.value.as_ref().expect("should succeed").status, + http::StatusCode::NO_CONTENT, + "should map provider 1" + ); + assert_eq!( + client.recorded_request_uris(), + [ + "https://ids.example.com/a".to_owned(), + "https://ids.example.com/b".to_owned() + ], + "should send both requests" + ); + } + + #[test] + fn gather_reports_timeout_for_requests_past_the_deadline() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, Vec::new()); + client.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(1))); + + let error = results[0].value.as_ref().expect_err("should time out"); + assert!( + matches!(error.current_context(), TransportError::Timeout), + "should report timeout" + ); + } + + #[test] + fn gather_rejects_oversized_bodies() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, vec![b'x'; MAX_PROVIDER_RESPONSE_BYTES + 1]); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(200))); + + let error = results[0] + .value + .as_ref() + .expect_err("should reject the body"); + assert!( + matches!(error.current_context(), TransportError::BodyTooLarge), + "should report an oversized body" + ); + } + + #[test] + fn gather_with_no_requests_returns_nothing() { + let client = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services, true); + + let results = futures::executor::block_on( + transport.gather(Vec::new(), Deadline::after(Duration::from_millis(1))), + ); + + assert!(results.is_empty(), "should return nothing"); + assert_eq!( + client.recorded_select_count(), + 0, + "should not select an empty list" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs new file mode 100644 index 0000000000..25cd7332f1 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -0,0 +1,354 @@ +//! Turns a provider's parsed response into stored entries (spec §8.2). + +use std::borrow::Cow; + +use crate::ec::coordinator::results::DropCounts; +use crate::ec::eid_types::{ + AgentType, EidFields, EidValidationError, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, + RawJson, SourceDomain, UidValue, UnixSeconds, +}; +use crate::ec::kv_types::{KvEid, KvUid}; +use crate::ec::provider::{EidLevelFields, EidShape, ProviderEid, ProviderExpiry, ProviderUid}; +use crate::ec::registry::PartnerRegistry; + +/// Policy one response is validated against. +pub(crate) struct ValidationPolicy<'a> { + /// Owner recorded on every kept UID. + pub provider: &'a ProviderId, + /// Sources the provider claims in config. + pub claimed: &'a [SourceDomain], + /// The provider's declared shape limits. + pub shape: EidShape, + /// Partner config, for the default `atype`. + pub partners: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Validated entries grouped by source, in response order. +pub(crate) type ValidatedEntries = Vec<(SourceDomain, Vec)>; + +/// Validates parsed entries, keeping valid ones grouped by source in +/// response order. +pub(crate) fn validate( + eids: Vec>, + policy: &ValidationPolicy<'_>, +) -> (ValidatedEntries, DropCounts) { + let mut dropped = DropCounts::default(); + let mut grouped: ValidatedEntries = Vec::new(); + let mut kept_eids = 0_usize; + + for eid in eids { + let Some(source) = claimed_source(&eid.source, policy.claimed) else { + dropped.record(EidValidationError::UnclaimedSource); + continue; + }; + if kept_eids == policy.shape.max_eids.get() { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + let fields = match eid_fields(&eid, policy.shape.eid_level_fields) { + Ok(fields) => fields, + Err(reason) => { + dropped.record(reason); + continue; + } + }; + let default_atype = policy + .partners + .find_by_source_domain(source.as_str()) + .map_or(1, |partner| partner.openrtb_atype); + let slot = match grouped.iter().position(|(existing, _)| existing == source) { + Some(index) => index, + None => { + grouped.push((source.clone(), Vec::new())); + grouped.len() - 1 + } + }; + let max_uids = policy.shape.max_uids_per_eid.get(); + let mut uids = Vec::with_capacity(eid.uids.len().min(max_uids)); + for uid in eid.uids { + if uids.len() == max_uids { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + match stored_uid(uid, default_atype, policy) { + Ok(uid) if is_duplicate(&grouped[slot].1, &uids, &uid) => { + dropped.duplicate = dropped.duplicate.saturating_add(1); + } + Ok(uid) => uids.push(uid), + Err(reason) => dropped.record(reason), + } + } + if !uids.is_empty() { + grouped[slot].1.push(KvEid::new(fields, uids)); + kept_eids += 1; + } + } + + grouped.retain(|(_, eids)| !eids.is_empty()); + (grouped, dropped) +} + +fn claimed_source<'p>(raw: &str, claimed: &'p [SourceDomain]) -> Option<&'p SourceDomain> { + let source = SourceDomain::try_from(raw).ok()?; + claimed.iter().find(|candidate| **candidate == source) +} + +fn bounded_field(value: Option<&Cow<'_, str>>) -> Result>, EidValidationError> { + match value { + Some(text) if text.len() > MAX_EID_FIELD_BYTES => Err(EidValidationError::FieldTooLong), + Some(text) => Ok(Some(Box::from(text.as_ref()))), + None => Ok(None), + } +} + +fn eid_fields( + eid: &ProviderEid<'_>, + level: EidLevelFields, +) -> Result { + if level == EidLevelFields::Ignored { + return Ok(EidFields::default()); + } + Ok(EidFields { + inserter: bounded_field(eid.inserter.as_ref())?, + matcher: bounded_field(eid.matcher.as_ref())?, + mm: eid.mm.map(MatchMethod::try_from).transpose()?, + ext: eid.ext.map(RawJson::try_from).transpose()?, + }) +} + +fn stored_uid( + uid: ProviderUid<'_>, + default_atype: i32, + policy: &ValidationPolicy<'_>, +) -> Result { + let id = UidValue::try_from(uid.id.into_owned())?; + let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; + let ext = uid.ext.map(RawJson::try_from).transpose()?; + let expiry = match uid.expiry { + ProviderExpiry::Unspecified => Expiry::Never, + ProviderExpiry::At(at) => Expiry::At(at), + ProviderExpiry::After(after) => Expiry::At(policy.now.saturating_add_secs(after.as_secs())), + }; + if !expiry.is_live(policy.now) { + return Err(EidValidationError::Expired); + } + Ok(KvUid::new( + id, + atype, + ext, + Some(policy.provider.clone()), + expiry, + )) +} + +fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { + earlier + .iter() + .flat_map(KvEid::uids) + .chain(current) + .any(|existing| existing.id() == uid.id()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::settings::EcPartner; + use serde_json::value::RawValue; + use std::num::NonZeroUsize; + use std::time::Duration; + + fn registry() -> PartnerRegistry { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "openrtb_atype": 3, "bidstream_enabled": true + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn uid(id: &str) -> ProviderUid<'_> { + ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + } + } + + fn eid<'a>(source: &'a str, uids: Vec>) -> ProviderEid<'a> { + ProviderEid { + source: Cow::Borrowed(source), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } + + fn policy<'a>( + provider: &'a ProviderId, + claimed: &'a [SourceDomain], + partners: &'a PartnerRegistry, + ) -> ValidationPolicy<'a> { + ValidationPolicy { + provider, + claimed, + shape: EidShape { + max_eids: NonZeroUsize::new(2).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(2).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + partners, + now: UnixSeconds::new(1000), + } + } + + fn claimed() -> [SourceDomain; 1] { + [SourceDomain::try_from("ids.example.com").expect("should parse")] + } + + #[test] + fn keeps_valid_uids_and_fills_owner_atype_and_expiry() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut timed = uid("u-2"); + timed.expiry = ProviderExpiry::After(Duration::from_secs(60)); + + let (entries, dropped) = validate( + vec![eid("IDS.example.com", vec![uid("u-1"), timed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + let uids = entries[0].1[0].uids(); + assert_eq!(uids[0].atype().get(), 3, "should fill atype from partner"); + assert_eq!(uids[0].owner(), Some(&provider), "should record owner"); + assert_eq!( + uids[1].expiry(), + Expiry::At(UnixSeconds::new(1060)), + "should resolve relative expiry" + ); + } + + #[test] + fn drops_invalid_values_by_reason() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let long = "x".repeat(600); + let mut expired = uid("u-old"); + expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); + let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); + let mut bad_ext = uid("u-ext"); + bad_ext.ext = Some(&array_ext); + + let (entries, dropped) = validate( + vec![ + eid( + "ids.example.com", + vec![uid(" "), uid(&long), expired, bad_ext, uid("ok"), uid("ok")], + ), + eid("unclaimed.example.com", vec![uid("u-3")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped.invalid_uid, 2, "should drop blank and long UIDs"); + assert_eq!(dropped.expired, 1, "should drop expired UID"); + assert_eq!( + dropped.invalid_field, 1, + "should drop UID with non-object ext" + ); + assert_eq!(dropped.duplicate, 1, "should drop duplicate UID"); + assert_eq!(dropped.unclaimed_source, 1, "should drop unclaimed source"); + assert_eq!(entries.len(), 1, "should keep the claimed source only"); + assert_eq!(entries[0].1[0].uids().len(), 1, "should keep one valid UID"); + } + + #[test] + fn enforces_shape_limits_from_the_end() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + + let (entries, dropped) = validate( + vec![ + eid("ids.example.com", vec![uid("a"), uid("b"), uid("c")]), + eid("ids.example.com", vec![uid("d")]), + eid("ids.example.com", vec![uid("e")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!( + dropped.over_limit, 2, + "should drop third entry and third UID" + ); + let ids: Vec<&str> = entries[0] + .1 + .iter() + .flat_map(KvEid::uids) + .map(|uid| uid.id().as_str()) + .collect(); + assert_eq!( + ids, + ["a", "b", "d"], + "should keep the first entries and UIDs" + ); + } + + #[test] + fn ignored_eid_level_fields_are_stripped_not_rejected() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut with_fields = eid("ids.example.com", vec![uid("a")]); + with_fields.inserter = Some(Cow::Borrowed("inserter.example.com")); + with_fields.mm = Some(99); + let mut policy = policy(&provider, &claimed, &partners); + policy.shape.eid_level_fields = EidLevelFields::Ignored; + + let (entries, dropped) = validate(vec![with_fields], &policy); + + assert_eq!( + dropped, + DropCounts::default(), + "should not count ignored fields" + ); + assert_eq!( + entries[0].1[0].fields(), + &EidFields::default(), + "should strip EID-level fields" + ); + } + + #[test] + fn preserved_eid_level_fields_are_validated() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut bad_mm = eid("ids.example.com", vec![uid("a")]); + bad_mm.mm = Some(99); + let mut kept = eid("ids.example.com", vec![uid("b")]); + kept.inserter = Some(Cow::Borrowed("inserter.example.com")); + kept.mm = Some(3); + + let (entries, dropped) = + validate(vec![bad_mm, kept], &policy(&provider, &claimed, &partners)); + + assert_eq!(dropped.invalid_field, 1, "should drop unknown mm entry"); + assert_eq!( + entries[0].1[0].fields().inserter.as_deref(), + Some("inserter.example.com"), + "should keep valid inserter" + ); + assert_eq!( + entries[0].1[0].fields().mm, + Some(MatchMethod::Authenticated), + "should keep valid mm" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 33981b49a5..1663cd5d6b 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -32,6 +32,7 @@ //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes //! - [`page_load`] — Single-use page-load write permit and row changes //! - [`write_plan`] — Plans a navigation's single row change +//! - [`provider`] — Identity provider interface and configuration //! - [`coordinator`] — Server-side identity lookup coordination mod auth; @@ -53,6 +54,7 @@ pub mod kv_types; pub mod page_load; pub mod partner; pub mod prebid_eids; +pub mod provider; pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs new file mode 100644 index 0000000000..d2be63fb7a --- /dev/null +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -0,0 +1,681 @@ +//! Identity provider interface (spec §7). +//! +//! Providers are pure: they build a lookup request and parse its response. +//! They never perform HTTP, KV, cookie or clock work; the coordinator in +//! [`crate::ec::coordinator`] executes requests and applies consent, +//! transport, validation, expiry and persistence policy. +//! +//! Provider implementations live in the `trusted-server-identity-providers` +//! crate, which depends on core; core never depends on it. + +use core::fmt; +use std::borrow::Cow; +use std::collections::HashSet; +use std::net::IpAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize as _; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use url::Url; + +use crate::consent::ConsentContext; +use crate::ec::eid_types::{SourceDomain, UnixSeconds}; +use crate::ec::kv_types::{KvEid, KvEntry}; +use crate::ec::registry::PartnerRegistry; +use crate::settings::{IdentityProviderConfig, ProviderSecrets}; + +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. +pub trait IdentityProvider { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be + /// parsed at all. Individually malformed entries are returned and dropped + /// by the coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; +} + +impl IdentityProvider for Box

{ + fn capabilities(&self) -> &ProviderCapabilities { + (**self).capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + (**self).build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + (**self).parse_response(response) + } +} + +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +/// One source a provider can return. +#[derive(Debug, Clone)] +pub struct SourceCapability { + /// The source domain. + pub source: SourceDomain, + /// Visitor input the source needs. + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +/// Limits a provider declares for one response. +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + /// Maximum EID entries kept from one response. + pub max_eids: NonZeroUsize, + /// Maximum UIDs kept per EID entry. + pub max_uids_per_eid: NonZeroUsize, + /// Whether EID-level fields are kept. + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + /// The fields are validated and stored. + Preserved, + /// The fields are dropped without rejecting the entry. + Ignored, +} + +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +/// Client signals lent to providers. +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + /// Client IP address. + pub ip: Option, + /// Client `User-Agent`. + pub user_agent: Option<&'a str>, +} + +/// This provider's stored entries, borrowed from the row. +#[derive(Debug, Clone, Copy)] +pub struct OwnedEntries<'a> { + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], +} + +impl<'a> OwnedEntries<'a> { + /// Views the entries of `entry` for `sources`. + #[must_use] + pub fn new(entry: Option<&'a KvEntry>, sources: &'a [SourceDomain]) -> Self { + Self { entry, sources } + } + + /// Iterates `(source, entry)` pairs in source order. + pub fn iter(&self) -> impl Iterator + 'a { + let entry = self.entry; + self.sources.iter().flat_map(move |source| { + entry + .and_then(|entry| entry.ids.get_key_value(source.as_str())) + .into_iter() + .flat_map(|(key, record)| record.eids().iter().map(move |eid| (key.as_str(), eid))) + }) + } +} + +/// A validated lowercase hex SHA-256 email digest. V1 never constructs one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HashedEmail(Box); + +impl HashedEmail { + /// Returns the digest. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + /// HTTP method. + pub method: http::Method, + /// Request URL; must be `https` with an allowlisted host. + pub url: Url, + /// Request headers. + pub headers: http::HeaderMap, + /// Request body. + pub body: bytes::Bytes, +} + +impl fmt::Debug for ProviderRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRequest") + .field("method", &self.method) + .field("host", &self.url.host_str()) + .finish_non_exhaustive() + } +} + +/// A provider response, borrowed from the transport. +#[derive(Debug, Clone, Copy)] +pub struct ProviderResponse<'body> { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: &'body http::HeaderMap, + /// Response body. + pub body: &'body [u8], +} + +/// A parsed provider response. +#[derive(Debug)] +pub enum LookupOutcome<'body> { + /// Entries were returned. + Found(Vec>), + /// The provider knows no ID for this visitor. + NoMatch, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + /// EID source domain. + pub source: Cow<'body, str>, + /// `OpenRTB` `inserter`. + pub inserter: Option>, + /// `OpenRTB` `matcher`. + pub matcher: Option>, + /// `OpenRTB` `mm`. + pub mm: Option, + /// EID-level `ext`. + pub ext: Option<&'body RawValue>, + /// UIDs of the entry. + pub uids: Vec>, +} + +/// One UID as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderUid<'body> { + /// UID value. + pub id: Cow<'body, str>, + /// Agent type; filled from partner config when missing. + pub atype: Option, + /// UID-level `ext`. + pub ext: Option<&'body RawValue>, + /// Expiry as the provider states it. + pub expiry: ProviderExpiry, +} + +/// Expiry as a provider states it. The coordinator turns it into +/// [`crate::ec::eid_types::Expiry`] using its own clock, so providers never +/// read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + /// No expiry was stated. + Unspecified, + /// Expires at this time. + At(UnixSeconds), + /// Expires this long after the request. + After(Duration), +} + +/// Why a provider could not build a request or parse a response. +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + /// The input cannot form a request. + #[display("provider request could not be built")] + Request, + /// The body cannot be parsed at all. + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} + +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +#[derive(Debug, Clone, Copy)] +pub struct ProviderCommon<'a> { + /// Configured lookup endpoint. + pub endpoint: &'a Url, + /// Configured sources. + pub sources: &'a [SourceDomain], + /// Resolved secrets. + pub secrets: &'a ProviderSecrets, +} + +/// Why a provider configuration was rejected. +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + /// No implementation is registered for the configured `kind`. + #[display("provider kind is not registered")] + UnknownKind, + /// The configuration is unusable. + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} + +/// Parses `config.settings` for `K` and builds a boxed provider. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when the settings do not +/// deserialize into `K::Settings`, a source or the endpoint is invalid, or +/// [`ProviderKind::build`] rejects them. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + let settings = K::Settings::deserialize(serde_json::Value::Object(config.settings.clone())) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("settings for provider `{}` are invalid", config.id))?; + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + let sources = parse_sources(config)?; + let common = ProviderCommon { + endpoint: &endpoint, + sources: &sources, + secrets, + }; + let provider = K::build(&common, settings)?; + Ok(Box::new(provider)) +} + +/// Parses and normalizes a provider's configured sources. +pub(crate) fn parse_sources( + config: &IdentityProviderConfig, +) -> Result, Report> { + config + .sources + .iter() + .map(|source| SourceDomain::try_from(source.as_str())) + .collect::, _>>() + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("provider `{}` has an invalid source", config.id)) +} + +/// Whether `url` is `https` and its host is one of `allowed_domains`, +/// compared ASCII case-insensitively. +#[must_use] +pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S]) -> bool { + url.scheme() == "https" + && url.host_str().is_some_and(|host| { + allowed_domains + .iter() + .any(|allowed| allowed.as_ref().eq_ignore_ascii_case(host)) + }) +} + +/// Checks the configuration rules core owns for every enabled provider +/// (spec §12.1): the endpoint is an `https` URL whose host is in +/// `allowed_domains`, every source has an `[[ec.partners]]` entry, and no +/// source is claimed by two enabled providers. +/// +/// Unknown kinds and capability checks need the provider crate and are +/// enforced when the coordinator is built. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] naming the first offending +/// provider. +pub fn validate_enabled_configs<'a>( + configs: impl IntoIterator, + partners: &PartnerRegistry, +) -> Result<(), Report> { + let mut claimed: HashSet = HashSet::new(); + for config in configs.into_iter().filter(|config| config.enabled) { + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + if !destination_allowed(&endpoint, &config.allowed_domains) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "endpoint for provider `{}` must be https with a host in allowed_domains", + config.id + ))); + } + for source in parse_sources(config)? { + if partners.find_by_source_domain(source.as_str()).is_none() { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` has no [[ec.partners]] entry", + config.id + ))); + } + if !claimed.insert(source.clone()) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is claimed by another enabled provider", + config.id + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + use serde_json::json; + + struct Fixed(ProviderCapabilities); + + impl IdentityProvider for Fixed { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Ok(LookupOutcome::NoMatch) + } + } + + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FixedSettings { + max_eids: usize, + } + + struct FixedKind; + + impl ProviderKind for FixedKind { + const KIND: &'static str = "fixed"; + type Settings = FixedSettings; + type Provider = Fixed; + + fn build( + common: &ProviderCommon<'_>, + settings: FixedSettings, + ) -> Result> { + let max_eids = NonZeroUsize::new(settings.max_eids) + .ok_or_else(|| Report::new(ProviderConfigError::Invalid))?; + Ok(Fixed(ProviderCapabilities { + sources: common + .sources + .iter() + .map(|source| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids, + max_uids_per_eid: max_eids, + eid_level_fields: EidLevelFields::Preserved, + }, + })) + } + } + + fn config(settings: &serde_json::Value) -> IdentityProviderConfig { + serde_json::from_value(json!({ + "id": "example-provider", + "kind": FixedKind::KIND, + "endpoint": "https://ids.example.com/v1", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "settings": settings, + })) + .expect("should parse provider config") + } + + fn partners() -> PartnerRegistry { + let partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn assert_invalid(error: &Report) { + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn build_kind_parses_typed_settings_and_boxes_provider() { + let provider = build_kind::( + &config(&json!({ "max_eids": 2 })), + &ProviderSecrets::default(), + ) + .expect("should build"); + + assert_eq!( + provider.capabilities().sources.len(), + 1, + "should claim configured source" + ); + assert_eq!( + provider.capabilities().shape.max_eids.get(), + 2, + "should apply typed settings" + ); + } + + #[test] + fn build_kind_rejects_unknown_settings() { + let error = build_kind::( + &config(&json!({ "unexpected": true })), + &ProviderSecrets::default(), + ) + .err() + .expect("should reject"); + + assert_invalid(&error); + } + + #[test] + fn config_parses_from_toml_with_a_settings_table() { + let parsed: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"fixed\"\nendpoint = \"https://ids.example.com/\"\n[settings]\nmax_eids = 3", + ) + .expect("should parse TOML config"); + + assert_eq!(parsed.kind(), "fixed", "should keep kind"); + assert!(parsed.enabled, "should default to enabled"); + assert_eq!( + parsed.settings.get("max_eids"), + Some(&json!(3)), + "should keep provider settings" + ); + } + + #[test] + fn request_debug_hides_query_and_headers() { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("Bearer example-secret"), + ); + let request = ProviderRequest { + method: http::Method::GET, + url: Url::parse("https://ids.example.com/v1?token=example-secret") + .expect("should parse"), + headers, + body: bytes::Bytes::new(), + }; + + let printed = format!("{request:?}"); + + assert!( + !printed.contains("example-secret"), + "should not print secrets" + ); + assert!(printed.contains("ids.example.com"), "should print host"); + } + + #[test] + fn validate_enabled_configs_accepts_valid_config() { + validate_enabled_configs([&config(&json!({}))], &partners()) + .expect("should accept valid config"); + } + + #[test] + fn validate_enabled_configs_rejects_endpoint_outside_allowlist() { + let mut outside = config(&json!({})); + outside.endpoint = "https://other.example.com/".to_owned(); + let mut plain_http = config(&json!({})); + plain_http.endpoint = "http://ids.example.com/".to_owned(); + + for rejected in [outside, plain_http] { + let error = validate_enabled_configs([&rejected], &partners()) + .expect_err("should reject endpoint"); + assert_invalid(&error); + } + } + + #[test] + fn validate_enabled_configs_rejects_source_without_partner() { + let mut unknown = config(&json!({})); + unknown.sources = vec!["unknown.example.com".to_owned()]; + + let error = validate_enabled_configs([&unknown], &partners()) + .expect_err("should reject source without partner"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_source_claimed_twice() { + let first = config(&json!({})); + let mut second = config(&json!({})); + second.id = "second-provider".to_owned(); + + let error = validate_enabled_configs([&first, &second], &partners()) + .expect_err("should reject double claim"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_ignores_disabled_providers() { + let first = config(&json!({})); + let mut disabled = config(&json!({})); + disabled.enabled = false; + disabled.endpoint = "https://other.example.com/".to_owned(); + + validate_enabled_configs([&first, &disabled], &partners()) + .expect("should ignore disabled provider"); + } + + #[test] + fn owned_entries_yields_only_configured_sources() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3 }] }] }, + "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3 }] }] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + + let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources) + .iter() + .map(|(source, _)| source) + .collect(); + + assert_eq!( + owned, + ["ids.example.com"], + "should yield owned sources only" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 7b3c853074..7cc86898b2 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -357,8 +357,8 @@ mod tests { fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { let mut settings = create_test_settings(); settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); (settings, registry) } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index b138ecbe16..ed0d2fbdb7 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -10,7 +10,7 @@ use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::redacted::Redacted; -use crate::settings::EcPartner; +use crate::settings::{Ec, EcPartner}; use super::partner::{hash_api_key, normalize_partner_source_domain}; @@ -208,6 +208,25 @@ impl PartnerRegistry { }) } + /// Builds the registry from `[ec]` settings and marks every source claimed + /// by an enabled identity provider as provider-owned. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`]. + pub fn from_ec(ec: &Ec) -> Result> { + let mut registry = Self::from_config(&ec.partners)?; + for source in ec + .identity_providers + .iter() + .filter(|provider| provider.enabled) + .flat_map(|provider| &provider.sources) + { + registry.mark_provider_owned(source); + } + Ok(registry) + } + /// Returns an empty registry (no partners configured). #[must_use] pub fn empty() -> Self { @@ -462,6 +481,36 @@ mod tests { } } + #[test] + fn from_ec_marks_enabled_provider_sources_only() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![ + make_partner("ids.example.com", &valid_api_token("ids-partner")), + make_partner("other.example.com", &valid_api_token("other-partner")), + ]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"k\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + toml::from_str( + "id = \"b\"\nkind = \"k\"\nenabled = false\nendpoint = \"https://ids.example.com/\"\nsources = [\"other.example.com\"]", + ) + .expect("should parse disabled provider"), + ]; + + let registry = PartnerRegistry::from_ec(&ec).expect("should build registry"); + + assert!( + registry.is_provider_owned("ids.example.com"), + "should mark enabled provider source" + ); + assert!( + !registry.is_provider_owned("other.example.com"), + "should not mark disabled provider source" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b3e2e6e20d..b70fad1355 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3454,9 +3454,15 @@ struct EcSnapshotPreloadInput { auction_needs_row: bool, eid_cookie_may_need_persistence: bool, privacy_needs_row: bool, + snapshot_already_read: bool, } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { + // An earlier step of this request (identity-provider dispatch) already + // read the row for an existing EC, so a second read adds nothing. + if input.snapshot_already_read { + return false; + } let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; let marker_can_skip = input.marker_valid && !input.auction_needs_row @@ -5307,6 +5313,10 @@ pub async fn handle_publisher_request( .is_some_and(|registry| !registry.is_empty()), eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), + // A freshly generated EC keeps its refresh read: the Add-confirmed + // snapshot may still upgrade to the stored row. + snapshot_already_read: !ec_context.ec_generated() + && !matches!(ec_context.kv_snapshot(), crate::ec::EcKvSnapshot::NotRead), }); let mut pending_origin = None; // A shared-template hit skips the origin entirely, and an in-flight request @@ -8389,6 +8399,7 @@ mod tests { auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(should_preload_ec_snapshot(&baseline)); assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8407,6 +8418,13 @@ mod tests { has_kv: false, ..baseline })); + assert!( + !should_preload_ec_snapshot(&EcSnapshotPreloadInput { + snapshot_already_read: true, + ..baseline + }), + "should skip the preload when the snapshot was already read" + ); } #[test] @@ -8420,6 +8438,7 @@ mod tests { auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(!should_preload_ec_snapshot(&marker_only)); assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8844,8 +8863,8 @@ mod tests { create_test_settings() }; settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); let http = Arc::new(StubHttpClient::new()); http.set_concurrent_fanout(true); http.push_response(200, b"ok".to_vec()); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 754acfc262..af233cee06 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::num::NonZeroU16; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -579,6 +580,66 @@ impl EcPartner { } } +/// One `[[ec.identity_providers]]` entry. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IdentityProviderConfig { + /// Unique provider ID; recorded as the owner of the UIDs it writes. + pub id: String, + /// Implementation kind, matched by the provider crate's `build`. + kind: String, + /// Whether lookups run. A disabled provider owns no sources. + #[serde( + default = "IdentityProviderConfig::default_enabled", + deserialize_with = "from_value_or_str" + )] + pub enabled: bool, + /// Lookup endpoint; must be `https` with a host in `allowed_domains`. + pub endpoint: String, + /// Exact hosts the coordinator may call for this provider. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub allowed_domains: Vec, + /// Source domains this provider owns; each needs an `[[ec.partners]]` entry. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub sources: Vec, + /// TCF vendor ID that must have vendor consent, when set. + #[serde(default)] + pub tcf_vendor_id: Option, + /// Secret Store key references, resolved at startup. + #[serde(default)] + pub secrets: ProviderSecrets, + /// Provider-specific settings, parsed by the provider kind. + /// + /// Held as a JSON object because settings travel through the JSON app + /// config pipeline. + #[serde(default)] + pub settings: serde_json::Map, +} + +impl IdentityProviderConfig { + const fn default_enabled() -> bool { + true + } + + /// The configured implementation kind. + #[must_use] + pub fn kind(&self) -> &str { + &self.kind + } +} + +/// Secret values for one identity provider. +/// +/// `EdgeZero` secret paths address fixed field names, so V1 supports one +/// optional bearer token per provider. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderSecrets { + /// Secret Store key whose value is the provider's token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token: Option>, +} + /// Edge Cookie (EC) configuration. /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, @@ -616,6 +677,18 @@ pub struct Ec { #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub partners: Vec, + + /// Shared time budget, in milliseconds, for all provider lookups on one + /// navigation. + #[serde( + default = "Ec::default_identity_budget_ms", + deserialize_with = "from_value_or_str" + )] + pub identity_budget_ms: u64, + + /// Server-side identity providers. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub identity_providers: Vec, } impl Ec { @@ -646,6 +719,12 @@ impl Ec { 3600 } + /// Default shared identity lookup budget (200 ms). + #[must_use] + pub const fn default_identity_budget_ms() -> u64 { + 200 + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -3510,6 +3589,19 @@ impl Settings { )); } } + for provider in &self.ec.identity_providers { + if provider + .secrets + .token + .as_ref() + .is_some_and(|token| EcPartner::is_placeholder_api_token(token.expose())) + { + insecure_fields.push(format!( + "ec.identity_providers[{}].secrets.token", + provider.id + )); + } + } for handler in &self.handlers { if Handler::is_placeholder_password(handler.password.expose()) { insecure_fields.push(format!("handlers[{}].password", handler.path)); @@ -6591,6 +6683,31 @@ source_domain = "partner.example.com" .expect("should accept a realistic partner pull token"); } + #[test] + fn reject_placeholder_secrets_includes_identity_provider_tokens() { + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); + settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); + let mut provider: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse provider config"); + provider.secrets.token = Some(Redacted::new( + "partner-api-token-32-bytes-minimum".to_owned(), + )); + settings.ec.identity_providers = vec![provider]; + + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder provider token"); + + assert!( + format!("{err:?}").contains("ec.identity_providers[example-provider].secrets.token"), + "should mention the provider token field" + ); + } + #[test] fn reject_placeholder_secrets_reports_both_partner_tokens() { let mut settings = diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml new file mode 100644 index 0000000000..1ff50dedf8 --- /dev/null +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "trusted-server-identity-providers" +description = "Identity provider implementations for Trusted Server." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +error-stack = { workspace = true } +trusted-server-core = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs new file mode 100644 index 0000000000..2b1154104a --- /dev/null +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -0,0 +1,146 @@ +//! Identity provider implementations for Trusted Server. +//! +//! Each provider kind is a module implementing +//! [`trusted_server_core::ec::provider::ProviderKind`]. [`build`] maps a +//! configured `kind` to its implementation; [`build_coordinator`] builds the +//! coordinator for every enabled provider. +//! +//! Dependencies point one way: this crate depends on core, and core never +//! depends on it. + +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; +use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; +use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets, Settings}; + +/// Builds the provider for a configured `kind`. +/// +/// No kinds are registered yet, so every kind is rejected. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`trusted_server_core::ec::provider::build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + _secrets: &ProviderSecrets, +) -> Result, Report> { + let kind = config.kind(); + Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))) +} + +/// Builds the coordinator for every enabled provider in `settings`. +/// +/// Returns `None` when no provider is enabled. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. +pub fn build_coordinator( + settings: &Settings, +) -> Result, Report> { + let enabled: Vec<&IdentityProviderConfig> = settings + .ec + .identity_providers + .iter() + .filter(|config| config.enabled) + .collect(); + if enabled.is_empty() { + return Ok(None); + } + let registered = enabled + .into_iter() + .map(|config| { + build(config, &config.secrets) + .map(|provider| RegisteredProvider::new(config.clone(), provider)) + }) + .collect::, Report>>()?; + let partners = PartnerRegistry::from_ec(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + let budget = Duration::from_millis(settings.ec.identity_budget_ms); + ProviderCoordinator::new(registered, Arc::new(partners), budget).map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "example-admin-password" + + [publisher] + domain = "www.example.com" + cookie_domain = ".example.com" + origin_url = "https://origin.example.com" + proxy_secret = "example-proxy-secret" + + [ec] + passphrase = "example-passphrase-32-bytes-minimum" + "#, + ) + .expect("should parse test settings") + } + + #[test] + fn unknown_kind_is_rejected() { + let config: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nallowed_domains = [\"ids.example.com\"]", + ) + .expect("should parse"); + + let error = build(&config, &ProviderSecrets::default()) + .err() + .expect("should reject unknown kind"); + + assert!( + matches!(error.current_context(), ProviderConfigError::UnknownKind), + "should report unknown kind" + ); + } + + #[test] + fn no_enabled_provider_builds_no_coordinator() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nenabled = false\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse"), + ]; + + let coordinator = build_coordinator(&settings).expect("should skip disabled providers"); + + assert!(coordinator.is_none(), "should build no coordinator"); + } + + #[test] + fn enabled_unknown_kind_fails_the_build() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse"), + ]; + + let error = build_coordinator(&settings) + .err() + .expect("should reject unknown kind"); + + assert!( + matches!(error.current_context(), ProviderConfigError::UnknownKind), + "should report unknown kind" + ); + } +} diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 8e87e6fa86..60f47a9be6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -59,6 +59,7 @@ publisher, trusted-client-IP, EC, handler, Tinybird, DataDome, and S3 fields: - `ec.passphrase` - `ec.partners[*].api_token`, when inbound identify or batch sync is used - `ec.partners[*].ts_pull_token`, when pull sync is enabled +- `ec.identity_providers[*].secrets.token`, when a provider needs an API token - `handlers[*].password` - `tinybird.auction_token_secret`, when Tinybird auction telemetry is enabled - `integrations.datadome.server_side_key_secret_name`, when protection is enabled @@ -756,6 +757,8 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. | `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | | `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | | `partners` | Array | No | Static partner registry entries | +| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | +| `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. @@ -788,6 +791,52 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` +### Identity providers + +`[[ec.identity_providers]]` entries look up a visitor's IDs from a provider's +server on publisher page navigations. Lookups start before the origin fetch, +share the `identity_budget_ms` budget, and are collected before the response is +finalized. Results go into the navigation's single EC row write. A lookup that +times out or fails leaves the row unchanged. + +Lookups run only for an existing or newly issued EC, only when consent allows +EID persistence, and, when `tcf_vendor_id` is set, only with TCF vendor consent +for that vendor. + +| Field | Type | Required | Description | +| ----------------- | ------- | -------- | ----------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation built into this release | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | +| `allowed_domains` | Array | Yes | Hosts the provider may call | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | + +**Example**: + +```toml +[ec] +identity_budget_ms = 200 + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +sources = ["ids.example.com"] + +[ec.identity_providers.secrets] +token = "example_provider_token" +``` + +**Validation**: configuration validation fails when an enabled provider's +`endpoint` is not HTTPS or its host is not in `allowed_domains`, when a source +has no `[[ec.partners]]` entry, or when two enabled providers claim the same +source. Deploy validation also rejects an empty `secrets.token` key name. + ### Field Details #### `passphrase` diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 99afa869dd..e0cec612cf 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -107,6 +107,29 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false +# Optional server-side identity providers, looked up on publisher navigations +# before the origin fetch. Every enabled provider shares one time budget: +# identity_budget_ms = 200 # default 200 +# +# [[ec.identity_providers]] +# id = "example-provider" +# Must name a provider kind built into this release. +# kind = "example-kind" +# enabled = false +# endpoint = "https://ids.example.com/v1/lookup" +# The endpoint host must be listed here. +# allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry and must be claimed by only one +# enabled provider. +# sources = ["ids.example.com"] +# Optional: require TCF vendor consent for this provider. +# tcf_vendor_id = 9999 +# Secret Store key name, resolved like other secret references. +# [ec.identity_providers.secrets] +# token = "example_provider_token" +# Provider-specific settings, parsed by the provider kind. +# [ec.identity_providers.settings] + # ============================================================================= # OPTIONAL — Core features (disabled/omitted by default) From b64bd1501f2e7d4413504dfb08c89ffe3135195a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 12:53:06 +0530 Subject: [PATCH 04/58] Fix provider ownership, orphan attribution and size cap issues Signed-off-by: dhruv8sh --- Cargo.lock | 1 + .../trusted-server-adapter-fastly/src/app.rs | 20 +-- .../trusted-server-adapter-fastly/src/main.rs | 12 +- .../src/auction/endpoints.rs | 56 +++++++- .../src/ec/coordinator/mod.rs | 7 +- .../src/ec/coordinator/results.rs | 21 ++- .../trusted-server-core/src/ec/page_load.rs | 37 ++++- crates/trusted-server-core/src/ec/registry.rs | 73 ++++++++-- .../trusted-server-core/src/ec/write_plan.rs | 130 ++++++++++++++---- .../Cargo.toml | 1 + .../src/lib.rs | 49 +++++-- 11 files changed, 342 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9cf35f9f0..116d45a14c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6743,6 +6743,7 @@ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ "error-stack", + "log", "toml", "trusted-server-core", ] diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 3b454d9725..175898ae53 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -123,7 +123,6 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::provider::ClientSignals; -use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; @@ -152,7 +151,7 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use trusted_server_identity_providers::build_coordinator; +use trusted_server_identity_providers::{build_coordinator, partner_registry}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -679,7 +678,7 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = PartnerRegistry::from_ec(&state.settings.ec) + let response = partner_registry(&state.settings.ec) .and_then(|registry| match handler { NamedRouteHandler::AdminEcLookup => { // Deliberately do not use an EC request-state graph: that @@ -760,7 +759,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; handle_identify( &state.settings, &kv, @@ -774,7 +773,7 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -798,7 +797,7 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; let registry_ref = if partner_registry.is_empty() { None } else { @@ -845,7 +844,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; + let partner_registry = partner_registry(&state.settings.ec)?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -983,7 +982,7 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_ec(&state.settings.ec) { + match partner_registry(&state.settings.ec) { Ok(partner_registry) => { let auction = AuctionDispatch { integration_registry: &state.registry, @@ -2909,7 +2908,10 @@ mod tests { })); let coordinator = ProviderCoordinator::new( vec![RegisteredProvider::new(provider_config, provider)], - Arc::new(PartnerRegistry::from_ec(&settings.ec).expect("should build registry")), + Arc::new( + PartnerRegistry::from_ec_with_kinds(&settings.ec, |_| true) + .expect("should build registry"), + ), Duration::from_millis(200), ) .expect("should build coordinator"); diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index a6ac48f369..efff0d4952 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -18,6 +18,7 @@ use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -30,6 +31,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; +use trusted_server_identity_providers::partner_registry; mod app; mod backend; @@ -659,6 +661,14 @@ fn apply_edgezero_ec_finalize( identity: Option, response: &mut HttpResponse, ) -> Result> { + // A redirect skips the write, so do not wait for lookups it would discard. + let identity = identity.filter(|_| { + let redirect = NavigationPermit::is_redirect(response.status()); + if redirect { + log::debug!("Identity lookups dropped: redirect response skips the write"); + } + !redirect + }); if let Some(IdentityLookups { coordinator, pending, @@ -669,7 +679,7 @@ fn apply_edgezero_ec_finalize( let results = futures::executor::block_on(coordinator.collect(&transport, pending)); ec_state.ec_context.set_lookup_results(results); } - let partner_registry = PartnerRegistry::from_ec(&settings.ec)?; + let partner_registry = partner_registry(&settings.ec)?; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index b261add7b3..50fdd263f8 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -660,14 +660,24 @@ pub(crate) fn merge_auction_eids( if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { continue; } + // A duplicate keeps the earlier UID's fields but fills its + // missing `atype` and `ext` from the later one. let duplicate = merged - .iter() + .iter_mut() .filter(|existing| existing.source == source) - .flat_map(|existing| existing.uids.iter()) - .chain(kept.iter()) - .any(|existing: &Uid| existing.id == uid.id); - if !duplicate { - kept.push(uid); + .flat_map(|existing| existing.uids.iter_mut()) + .chain(kept.iter_mut()) + .find(|existing: &&mut Uid| existing.id == uid.id); + match duplicate { + Some(existing) => { + if existing.atype.is_none() { + existing.atype = uid.atype; + } + if existing.ext.is_none() { + existing.ext = uid.ext; + } + } + None => kept.push(uid), } } if kept.is_empty() { @@ -2258,6 +2268,40 @@ mod tests { ); } + #[test] + fn merge_auction_eids_fills_missing_atype_and_ext_from_duplicate() { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: Some(1), + ext: Some(json!({ "provider": "client" })), + }], + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: None, + ext: None, + }], + )]); + + let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); + + assert_eq!(merged[0].uids.len(), 1, "should deduplicate duplicate uid"); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should fill missing atype from the duplicate" + ); + assert_eq!( + merged[0].uids[0].ext, + Some(json!({ "provider": "client" })), + "should fill missing ext from the duplicate" + ); + } + #[test] fn auction_rejects_oversized_body() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 28d9c98caa..cd95bf667a 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -96,6 +96,8 @@ pub struct LookupContext<'a> { pub struct PendingLookups { requests: Vec>, deadline: Option, + /// EC ID the lookups were dispatched for. + ec_id: String, /// Request time, for validating `exp` and resolving relative expiry. now: UnixSeconds, /// Providers that finished at dispatch (skipped or failed to send). @@ -107,6 +109,7 @@ impl Default for PendingLookups { Self { requests: Vec::new(), deadline: None, + ec_id: String::new(), now: UnixSeconds::new(0), early: Vec::new(), } @@ -421,6 +424,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ lookup: &LookupContext<'_>, ) -> PendingLookups { let mut pending = PendingLookups { + ec_id: lookup.ec_id.to_owned(), now: lookup.now, ..PendingLookups::default() }; @@ -511,6 +515,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ let PendingLookups { requests, deadline, + ec_id, now, early, } = pending; @@ -530,7 +535,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ for lookup in &lookups { log_outcome(lookup); } - LookupResults::new(lookups) + LookupResults::new(&ec_id, lookups) } fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index e9bc32a2d1..53e675ad57 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -124,14 +124,29 @@ pub struct ProviderLookup { /// Validated results of one navigation's lookups. #[derive(Debug, Clone, Default)] pub struct LookupResults { + /// EC ID the lookups were made for; `None` when nothing was looked up. + ec_id: Option, lookups: Vec, } impl LookupResults { - /// Wraps per-provider results in provider-config order. + /// Wraps per-provider results, made for `ec_id`, in provider-config + /// order. #[must_use] - pub fn new(lookups: Vec) -> Self { - Self { lookups } + pub fn new(ec_id: &str, lookups: Vec) -> Self { + Self { + ec_id: Some(ec_id.to_owned()), + lookups, + } + } + + /// Whether these results were looked up for `ec_id`. + /// + /// A navigation can switch EC IDs after dispatch (orphan recovery mints a + /// new one), so results for another ID must not be attributed to it. + #[must_use] + pub fn is_for(&self, ec_id: &str) -> bool { + self.ec_id.as_deref() == Some(ec_id) } /// Iterates results in provider-config order. diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs index 212034c3f4..7453d93e7f 100644 --- a/crates/trusted-server-core/src/ec/page_load.rs +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -54,15 +54,24 @@ impl NavigationPermit { /// are the page load itself and keep the permit. #[must_use] pub fn confirm(self, status: StatusCode) -> Option { - let redirect = matches!( + (!Self::is_redirect(status)).then_some(PageLoadWrite(())) + } + + /// Whether `status` is a redirect (301, 302, 303, 307, 308) on which the + /// page load's write is skipped. + /// + /// Callers use this to drop work, such as pending identity lookups, whose + /// only purpose is that write. + #[must_use] + pub fn is_redirect(status: StatusCode) -> bool { + matches!( status, StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | StatusCode::SEE_OTHER | StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT - ); - (!redirect).then_some(PageLoadWrite(())) + ) } /// Builds a permit without a request, for tests. @@ -226,6 +235,28 @@ mod tests { } } + #[test] + fn is_redirect_matches_only_write_skipping_statuses() { + for status in [ + http::StatusCode::MOVED_PERMANENTLY, + http::StatusCode::FOUND, + http::StatusCode::SEE_OTHER, + http::StatusCode::TEMPORARY_REDIRECT, + http::StatusCode::PERMANENT_REDIRECT, + ] { + assert!( + NavigationPermit::is_redirect(status), + "should treat {status} as a redirect" + ); + } + for status in [http::StatusCode::OK, http::StatusCode::NOT_MODIFIED] { + assert!( + !NavigationPermit::is_redirect(status), + "should not treat {status} as a redirect" + ); + } + } + fn permit() -> PageLoadWrite { NavigationPermit::for_test() .confirm(http::StatusCode::OK) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index ed0d2fbdb7..05bfbb92ed 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -208,21 +208,45 @@ impl PartnerRegistry { }) } - /// Builds the registry from `[ec]` settings and marks every source claimed - /// by an enabled identity provider as provider-owned. + /// Builds the registry from `[ec]` settings without provider ownership. + /// + /// For adapters with no identity-provider implementations: no source is + /// provider-owned, so browser-supplied IDs are ingested for every source. /// /// # Errors /// /// Returns the same errors as [`Self::from_config`]. pub fn from_ec(ec: &Ec) -> Result> { + Self::from_ec_with_kinds(ec, |_| false) + } + + /// Builds the registry from `[ec]` settings and marks every source claimed + /// by an enabled identity provider whose kind is registered as + /// provider-owned. + /// + /// An enabled provider with an unregistered kind never runs, so its + /// sources stay open to browser ingest and a warning is logged. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`]. + pub fn from_ec_with_kinds( + ec: &Ec, + is_registered_kind: impl Fn(&str) -> bool, + ) -> Result> { let mut registry = Self::from_config(&ec.partners)?; - for source in ec - .identity_providers - .iter() - .filter(|provider| provider.enabled) - .flat_map(|provider| &provider.sources) - { - registry.mark_provider_owned(source); + for provider in ec.identity_providers.iter().filter(|p| p.enabled) { + if !is_registered_kind(provider.kind()) { + log::warn!( + "Identity provider '{}' has unregistered kind '{}'; its sources stay browser-ingested", + provider.id, + provider.kind() + ); + continue; + } + for source in &provider.sources { + registry.mark_provider_owned(source); + } } Ok(registry) } @@ -499,7 +523,8 @@ mod tests { .expect("should parse disabled provider"), ]; - let registry = PartnerRegistry::from_ec(&ec).expect("should build registry"); + let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); assert!( registry.is_provider_owned("ids.example.com"), @@ -511,6 +536,34 @@ mod tests { ); } + #[test] + fn from_ec_skips_providers_with_unregistered_kinds() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![make_partner( + "ids.example.com", + &valid_api_token("ids-partner"), + )]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + ]; + + let with_kinds = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); + let plain = PartnerRegistry::from_ec(&ec).expect("should build registry"); + + assert!( + !with_kinds.is_provider_owned("ids.example.com"), + "should not claim sources for an unregistered kind" + ); + assert!( + !plain.is_provider_owned("ids.example.com"), + "should not claim sources without provider implementations" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 075b14b59e..1ccc721246 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -58,7 +58,17 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { let mut entry = base.clone(); apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); - let written = apply_lookups(&mut entry, input.lookups, input.now); + let written = if input.lookups.is_for(input.ec_id) { + apply_lookups(&mut entry, input.lookups, input.now) + } else { + if !input.lookups.is_empty() { + log::info!( + "EC write planner: discarding lookups made for another EC ID than '{}'", + super::log_id(input.ec_id) + ); + } + Vec::new() + }; for record in entry.ids.values_mut() { record.prune_expired(input.now); } @@ -111,23 +121,35 @@ fn serialized_len(entry: &KvEntry) -> usize { fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { while serialized_len(entry) > MAX_ENTRY_BYTES { - let Some(source) = written.iter().rev().find(|source| { + if let Some(source) = written.iter().rev().find(|source| { entry .ids .get(source.as_str()) .is_some_and(KvPartnerId::is_provider_written) - }) else { - log::warn!("EC write planner: row exceeds size cap with no provider UIDs left to drop"); + }) { + let record = entry + .ids + .get_mut(source.as_str()) + .expect("should hold a written source"); + record.drop_last_uid(); + if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + } + continue; + } + // No provider UIDs from this navigation are left: drop whole records, + // largest first, so every later write still fits the cap. + let Some(largest) = entry + .ids + .iter() + .max_by_key(|(_, record)| serde_json::to_vec(record).map_or(usize::MAX, |b| b.len())) + .map(|(source, _)| source.clone()) + else { + log::warn!("EC write planner: row exceeds size cap with no partner IDs left to drop"); return; }; - let record = entry - .ids - .get_mut(source.as_str()) - .expect("should hold a written source"); - record.drop_last_uid(); - if record.uid.is_empty() { - entry.ids.remove(source.as_str()); - } + log::warn!("EC write planner: dropping '{largest}' to fit the row size cap"); + entry.ids.remove(&largest); } } @@ -184,17 +206,20 @@ mod tests { ) }) .collect(); - LookupResults::new(vec![ProviderLookup { - provider: ProviderId::new("example-provider"), - status: LookupStatus::Found { - kept: 1, - dropped: DropCounts::default(), - }, - entries: vec![( - SourceDomain::try_from(source).expect("should build source"), - vec![KvEid::new(EidFields::default(), uids)], - )], - }]) + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), uids)], + )], + }], + ) } fn input<'a>( @@ -250,6 +275,36 @@ mod tests { ); } + #[test] + fn lookups_for_another_ec_id_are_discarded() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let lookups_for_old = LookupResults::new( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.OLD123", + found("ids.example.com", &["p-1"]).iter().cloned().collect(), + ); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups_for_old, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not attribute the old EC ID's lookups to the new row" + ); + } + #[test] fn unchanged_returning_row_plans_no_write() { let registry = registry(&[]); @@ -317,6 +372,35 @@ mod tests { ); } + #[test] + fn oversized_row_without_provider_uids_drops_records_until_it_fits() { + let registry = registry(&[]); + let mut entry = live_entry(); + for i in 0..40 { + entry.ids.insert( + format!("source{i:02}.example.com"), + KvPartnerId::single(format!("{i:03}{}", "x".repeat(500))), + ); + } + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let lookups = LookupResults::default(); + + let change = plan_write(&input(&snapshot, Some(&entry), &[], &lookups, ®istry)) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + assert!(!entry.ids.is_empty(), "should keep the records that fit"); + } + #[test] fn oversized_row_drops_provider_uids_until_it_fits() { let registry = registry(&["ids.example.com"]); diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 1ff50dedf8..831ed20729 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] error-stack = { workspace = true } +log = { workspace = true } trusted-server-core = { workspace = true } [dev-dependencies] diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 2b1154104a..dd6c086f43 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -15,7 +15,26 @@ use error_stack::Report; use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets, Settings}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::settings::{Ec, IdentityProviderConfig, ProviderSecrets, Settings}; + +/// Whether `kind` has a registered implementation. +/// +/// No kinds are registered yet. +#[must_use] +pub fn is_registered_kind(_kind: &str) -> bool { + false +} + +/// Builds the partner registry, marking only sources claimed by enabled +/// providers whose kind is registered as provider-owned. +/// +/// # Errors +/// +/// Returns the same errors as [`PartnerRegistry::from_ec`]. +pub fn partner_registry(ec: &Ec) -> Result> { + PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) +} /// Builds the provider for a configured `kind`. /// @@ -50,6 +69,17 @@ pub fn build_coordinator( .identity_providers .iter() .filter(|config| config.enabled) + .filter(|config| { + let registered = is_registered_kind(config.kind()); + if !registered { + log::warn!( + "Identity provider '{}' skipped: kind '{}' is not registered", + config.id, + config.kind() + ); + } + registered + }) .collect(); if enabled.is_empty() { return Ok(None); @@ -61,7 +91,7 @@ pub fn build_coordinator( .map(|provider| RegisteredProvider::new(config.clone(), provider)) }) .collect::, Report>>()?; - let partners = PartnerRegistry::from_ec(&settings.ec) + let partners = partner_registry(&settings.ec) .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; let budget = Duration::from_millis(settings.ec.identity_budget_ms); ProviderCoordinator::new(registered, Arc::new(partners), budget).map(Some) @@ -125,22 +155,23 @@ mod tests { } #[test] - fn enabled_unknown_kind_fails_the_build() { + fn enabled_unknown_kind_is_skipped_and_claims_no_sources() { let mut settings = create_test_settings(); settings.ec.identity_providers = vec![ toml::from_str( - "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"", + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", ) .expect("should parse"), ]; - let error = build_coordinator(&settings) - .err() - .expect("should reject unknown kind"); + let coordinator = + build_coordinator(&settings).expect("should skip unregistered provider kinds"); + let registry = partner_registry(&settings.ec).expect("should build registry"); + assert!(coordinator.is_none(), "should build no coordinator"); assert!( - matches!(error.current_context(), ProviderConfigError::UnknownKind), - "should report unknown kind" + !registry.is_provider_owned("ids.example.com"), + "should leave unregistered provider sources to browser ingest" ); } } From 49b73b217d9c0b9aecf30cc7bf8d057d304c683f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:07:28 +0530 Subject: [PATCH 05/58] Skip partner token length checks in deploy-time provider validation Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/config.rs | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 85bfd5cf0f..44a592f1f3 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -33,6 +33,7 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; +use crate::redacted::Redacted; use crate::settings::{AssetOriginAuth, Ec, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; @@ -279,7 +280,7 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Ec { + let mut ec = ec.clone(); + for (index, partner) in ec.partners.iter_mut().enumerate() { + if partner.api_token.is_some() { + partner.api_token = Some(Redacted::new(format!("{index:0>64}"))); + } + if partner.ts_pull_token.is_some() { + partner.ts_pull_token = Some(Redacted::new(format!("{index:1>64}"))); + } + } + ec +} + /// Checks the identity-provider rules core owns (spec §12.1). /// /// Unknown kinds and capability checks need the provider crate and are @@ -1355,6 +1374,16 @@ gam_network_id = "99999" validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); } + #[test] + fn deploy_validation_accepts_provider_with_unresolved_partner_token() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + settings.ec.partners[0].api_token = + Some(Redacted::new("example_partner_api_token".to_owned())); + + validate_settings_for_deploy(&settings) + .expect("should not length-check a partner token secret key name"); + } + #[test] fn runtime_validation_rejects_provider_endpoint_outside_allowlist() { let settings = settings_with_identity_provider(serde_json::json!({ From 9e2a2d72e0130932532496e5edf087ecc9d6c379 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:13:00 +0530 Subject: [PATCH 06/58] Stop warning about unregistered provider kinds where kinds are unknown Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/registry.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 05bfbb92ed..5fa2677167 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -212,12 +212,14 @@ impl PartnerRegistry { /// /// For adapters with no identity-provider implementations: no source is /// provider-owned, so browser-supplied IDs are ingested for every source. + /// Kind registration is unknown here, so no unregistered-kind warning is + /// logged; use [`Self::from_ec_with_kinds`] where kinds are known. /// /// # Errors /// /// Returns the same errors as [`Self::from_config`]. pub fn from_ec(ec: &Ec) -> Result> { - Self::from_ec_with_kinds(ec, |_| false) + Self::from_config(&ec.partners) } /// Builds the registry from `[ec]` settings and marks every source claimed From c559b8924e41c21c0c723383a837e25275e30bf1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 17:46:35 +0530 Subject: [PATCH 07/58] Write orphan-recovery lookups to the replacement EC row Orphan recovery minted a new EC ID and then discarded the navigation's provider results because they were looked up under the orphaned ID. Lookups depend on the browser, not the EC ID, and the orphan has no row, so retarget them to the replacement as the spec requires. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/results.rs | 12 +++ crates/trusted-server-core/src/ec/finalize.rs | 73 +++++++++++++++++++ crates/trusted-server-core/src/ec/mod.rs | 6 ++ 3 files changed, 91 insertions(+) diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 53e675ad57..7fff308ea9 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -149,6 +149,18 @@ impl LookupResults { self.ec_id.as_deref() == Some(ec_id) } + /// Retargets results made for `orphan` to `replacement`. + /// + /// Orphan recovery replaces a proven-absent EC ID with a new one for the + /// same browser. Lookups depend on the browser, not on the EC ID, and the + /// orphan has no row left, so its results belong to the replacement row. + /// Results made for any other ID are left untouched. + pub(crate) fn retarget_orphan(&mut self, orphan: &str, replacement: &str) { + if self.is_for(orphan) { + self.ec_id = Some(replacement.to_owned()); + } + } + /// Iterates results in provider-config order. pub fn iter(&self) -> impl Iterator { self.lookups.iter() diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index eba972a8a6..9e092f2433 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -325,6 +325,9 @@ fn recover_orphaned_ec( .device_signals() .map(super::device::DeviceSignals::to_kv_device); + // Spec: the recovered row is created with this navigation's provider + // results, which were looked up under the orphaned ID. + ec_context.retarget_orphan_lookups(&orphan_id, &new_id); let missing = EcKvSnapshot::Missing { ec_id: new_id.clone(), }; @@ -585,6 +588,11 @@ mod tests { use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; use crate::ec::EidSyncSource; + use crate::ec::coordinator::results::{ + DropCounts, LookupResults, LookupStatus, ProviderLookup, + }; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv_types::{KvEid, KvUid}; use crate::ec::page_load::NavigationPermit; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; @@ -1020,6 +1028,71 @@ mod tests { ); } + #[test] + fn finalize_recovery_writes_lookups_made_for_the_orphan() { + let settings = create_test_settings(); + let orphaned_ec = sample_ec_id("orphn2"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip( + Some(orphaned_ec.clone()), + consent, + Some("192.0.2.10".to_owned()), + ); + ec_context.set_recovery_eligible(true); + ec_context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: orphaned_ec.clone(), + }); + let uid = KvUid::new( + UidValue::try_from("provider-uid-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + ec_context.set_lookup_results(LookupResults::new( + &orphaned_ec, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("provider.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + }], + )); + let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let replacement = ec_context.ec_value().expect("should rotate orphan"); + assert_ne!(replacement, orphaned_ec, "should mint a replacement EC ID"); + let (entry, _) = graph + .get(replacement) + .expect("should read replacement") + .expect("should create replacement row"); + assert!( + entry.ids.contains_key("provider.example.com"), + "should write the orphan's provider results to the replacement row" + ); + } + #[test] fn finalize_rotates_orphaned_cookie_to_new_backed_ec() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 1663cd5d6b..2a832aebdd 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -639,6 +639,12 @@ impl EcContext { &self.lookup_results } + /// Retargets this navigation's lookups from an orphaned EC ID to its + /// replacement; see [`LookupResults::retarget_orphan`]. + pub(crate) fn retarget_orphan_lookups(&mut self, orphan: &str, replacement: &str) { + self.lookup_results.retarget_orphan(orphan, replacement); + } + /// Takes the row built by [`Self::generate_if_needed`]. pub(crate) fn take_pending_entry(&mut self) -> Option { self.pending_entry.take().map(|entry| *entry) From 4ab0a1c31e4b1a92095ff2918450a227016349e6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 17:49:49 +0530 Subject: [PATCH 08/58] Keep responses whose headers arrive in an expired deadline snapshot Collection that starts after its deadline takes one nonblocking snapshot. A request still in the headers state moved to the body state without reading bytes, which counted as no progress, so a response that had fully arrived was abandoned unread and reported as a timeout. Count entering the body state as progress. Signed-off-by: dhruv8sh --- .../src/platform/deadline.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index 6a8842d180..e4333c193f 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -262,9 +262,10 @@ pub(super) fn select_until( for index in 0..pending.len() { loop { let previous = pending[index].buffered_bytes(); + let was_headers = matches!(pending[index].state, Some(PendingState::Headers(_))); let result = pending[index].poll(); - let progressed = - matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; + let progressed = matches!(result, Ok(None)) + && snapshot_progressed(was_headers, previous, &pending[index]); match result { Ok(None) if progressed => {} Ok(None) => { @@ -337,6 +338,16 @@ pub(super) fn select_until( } } +/// Whether a nonblocking poll that returned no response advanced `item`. +/// +/// Arriving headers move a request into [`PendingState::Body`] without +/// reading any bytes; that is progress, or a response that fully arrived +/// before an expired collection would be abandoned unread. +fn snapshot_progressed(was_headers: bool, previous_bytes: usize, item: &DeadlinePending) -> bool { + let entered_body = was_headers && matches!(item.state, Some(PendingState::Body { .. })); + entered_body || item.buffered_bytes() > previous_bytes +} + fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { return; @@ -543,6 +554,20 @@ mod tests { ); } + #[test] + fn snapshot_counts_arriving_headers_as_progress() { + let pending = DeadlinePending::from_platform(buffered_pending(Vec::new(), "example")) + .expect("should resume a buffered pending request"); + assert!( + super::snapshot_progressed(true, 0, &pending), + "should keep polling a request whose headers just arrived" + ); + assert!( + !super::snapshot_progressed(false, 0, &pending), + "should stop polling a body that read nothing" + ); + } + #[test] fn expired_deadline_preserves_an_already_ready_body() { let pending = buffered_pending(b"ready".to_vec(), "example"); From e43ea9a93bda9a6002fde8976299dbc8776afb88 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 18:03:23 +0530 Subject: [PATCH 09/58] Surface arrived bodies once in an expired deadline snapshot A response body the host already holds can report not ready until the host services async I/O once, so an expired snapshot abandoned identity responses that arrived well before the deadline and reported them as timeouts. Give each pending body one host select capped at 1 ms before abandoning it. Signed-off-by: dhruv8sh --- .../src/platform/deadline.rs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index e4333c193f..6b074f385c 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -16,6 +16,9 @@ use super::{ // Bound payload retained across all partial bodies, as well as each response. const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; +/// Longest host wait an expired snapshot spends surfacing one arrived body. +const SNAPSHOT_SURFACE_TIMEOUT_MS: u32 = 1; + // Keep partially buffered responses alongside requests still awaiting headers so // an early header with a slow body cannot hold up a completed sibling response. struct DeadlinePending { @@ -215,7 +218,9 @@ fn body_is_ready(body: &BodyHandle) -> Result> { /// Dropping response bodies closes their host handles. Fastly has no pending /// request cancellation ABI: unfinished header requests are abandoned and /// never waited on past the auction deadline. If collection starts after the -/// deadline, one nonblocking snapshot preserves already available responses. +/// deadline, one snapshot preserves already available responses; it may spend +/// at most [`SNAPSHOT_SURFACE_TIMEOUT_MS`] per pending body letting the host +/// surface bytes that already arrived. /// /// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that /// capacity, the largest partial response fails with an attributed HTTP client @@ -260,12 +265,22 @@ pub(super) fn select_until( // Take one nonblocking snapshot. Fully buffered results are cached; // responses that would require a wait are abandoned, never retried. for index in 0..pending.len() { + let mut surfaced = false; loop { let previous = pending[index].buffered_bytes(); let was_headers = matches!(pending[index].state, Some(PendingState::Headers(_))); let result = pending[index].poll(); let progressed = matches!(result, Ok(None)) && snapshot_progressed(was_headers, previous, &pending[index]); + if matches!(result, Ok(None)) + && !progressed + && !surfaced + && let Some(PendingState::Body { body, .. }) = &pending[index].state + { + surfaced = true; + surface_arrived_body(body); + continue; + } match result { Ok(None) if progressed => {} Ok(None) => { @@ -338,6 +353,34 @@ pub(super) fn select_until( } } +/// Lets the host surface body bytes that have already arrived. +/// +/// A body whose bytes the host holds can still report not ready until the +/// host services async I/O once, so an expired snapshot would abandon a +/// response that fully arrived before the deadline. One select capped at +/// [`SNAPSHOT_SURFACE_TIMEOUT_MS`] runs at most once per pending body; a +/// failure leaves the body not ready. +fn surface_arrived_body(body: &BodyHandle) { + let handles = [ + // SAFETY: the live owned body retains the handle while select borrows it. + unsafe { body.as_u32() }, + ]; + let mut ready_index = 0; + // SAFETY: handles is a live slice, and ready_index is writable. The + // timeout is positive: the host interprets zero as an unbounded wait. + let status = unsafe { + fastly_sys::fastly_async_io::select( + handles.as_ptr(), + handles.len(), + SNAPSHOT_SURFACE_TIMEOUT_MS, + &mut ready_index, + ) + }; + if let Err(status) = status.result() { + log::debug!("Expired snapshot: body surface select failed: {status:?}"); + } +} + /// Whether a nonblocking poll that returned no response advanced `item`. /// /// Arriving headers move a request into [`PendingState::Body`] without From dfd160ab656a035853f257a3e850363a7084ca68 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 18:30:57 +0530 Subject: [PATCH 10/58] Fall back to browser IDs for provider-owned sources the provider did not fill Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 53 +++- crates/trusted-server-core/src/ec/finalize.rs | 117 +++++++++ .../trusted-server-core/src/ec/prebid_eids.rs | 7 +- .../trusted-server-core/src/ec/write_plan.rs | 236 +++++++++++++++++- docs/guide/configuration.md | 8 + 5 files changed, 402 insertions(+), 19 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index cd95bf667a..8fa406cbcb 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -272,7 +272,8 @@ impl ProviderCoordinator

{ self.providers.is_empty() } - /// Claimed sources lacking a live UID, minus sources needing input V1 + /// Claimed sources lacking a live provider-written UID (a browser + /// fallback does not count), minus sources needing input V1 /// never has; the flag reports whether any source was removed for input. fn due_sources( registered: &Registered

, @@ -285,7 +286,7 @@ impl ProviderCoordinator

{ for source in ®istered.policy.sources { let held = entry .and_then(|entry| entry.ids.get(source.as_str())) - .is_some_and(|record| record.has_live_uid(now)); + .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)); if held { continue; } @@ -560,8 +561,9 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::TcfConsent; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; use crate::ec::kv::KvIdentityGraph; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, ProviderUid, SourceCapability, @@ -829,6 +831,49 @@ mod tests { assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); } + fn provider_record(id: &str) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + #[test] + fn source_with_browser_fallback_uid_is_still_due() { + // Arrange + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("fallback"), + ); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up a source holding only a browser fallback" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + #[test] fn source_with_live_uid_is_not_due() { let coordinator = coordinator(""); @@ -837,7 +882,7 @@ mod tests { entry.consent.ok = true; entry .ids - .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + .insert("ids.example.com".to_owned(), provider_record("p-1")); let results = run( &coordinator, diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 9e092f2433..1fc5c1a074 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1583,6 +1583,123 @@ mod tests { ); } + #[test] + fn finalize_withholds_provider_owned_fallback_when_purpose_four_is_denied() { + // Arrange + let settings = create_test_settings(); + let ec_id = sample_ec_id("purp4o"); + let graph = KvIdentityGraph::in_memory("test_store"); + let purpose1_only_consent = purpose_one_only_consent(); + let live = KvEntry::new( + &purpose1_only_consent, + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row this request updates"); + let mut ec_context = EcContext::new_for_test_with_cookie( + Some(ec_id.clone()), + Some(ec_id.clone()), + true, + false, + purpose1_only_consent, + ); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { + id: "ID5_should_not_persist".to_owned(), + atype: Some(1), + ext: None, + }], + )]); + let mut registry = PartnerRegistry::from_config(&[make_partner("id5-sync.com")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("id5-sync.com"), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + !stored.ids.contains_key("id5-sync.com"), + "should keep the provider-owned fallback out of KV without Purpose 4" + ); + } + + #[test] + fn finalize_stores_browser_fallback_for_provider_owned_source() { + // Arrange + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut registry = PartnerRegistry::from_config(&[make_partner("sharedid.org")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("sharedid.org"), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("fallback-cookie-id"), + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should create the row"); + let record = stored + .ids + .get("sharedid.org") + .expect("should store the browser fallback"); + assert_eq!( + record.uid, "fallback-cookie-id", + "should store the browser uid" + ); + assert!( + !record.is_provider_written(), + "should mark the fallback as not provider-written" + ); + } + #[test] fn finalize_auction_confirmed_miss_does_not_create_a_row() { // The same path with a genuinely absent row must stay a no-op: a route diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 7be48e1e6d..29db7122fc 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,10 +157,9 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } - let mut updates = dedupe_partner_updates(updates); - // Provider-owned sources are written only by their identity provider. - updates.retain(|update| !registry.is_provider_owned(&update.partner_id)); - updates + // Provider-owned sources stay in the list: the write planner ingests them + // only as a fallback when the provider left the source empty. + dedupe_partner_updates(updates) } pub(crate) fn collect_prebid_eid_updates( diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 1ccc721246..60c60eb009 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -1,8 +1,14 @@ //! Plans a navigation's single row change. //! -//! Combines browser ingest (fill missing, never provider-owned sources) and -//! validated provider results (replace the provider's sources), prunes -//! expired UIDs and enforces [`MAX_ENTRY_BYTES`]. Pure: no I/O. +//! Combines browser ingest (fill missing) and validated provider results +//! (replace the provider's sources), prunes expired UIDs and enforces +//! [`MAX_ENTRY_BYTES`]. Pure: no I/O. +//! +//! A provider-owned source takes a browser UID only as a fallback: when the +//! row holds no live UID for it and the provider kept nothing for it on this +//! navigation. The fallback is a single-UID record without provider `eids`, +//! so the coordinator still treats the source as due and a later provider +//! result replaces it. use super::EcKvSnapshot; use super::coordinator::results::LookupResults; @@ -57,8 +63,26 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { }; let mut entry = base.clone(); - apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); - let written = if input.lookups.is_for(input.ec_id) { + let lookups_apply = input.lookups.is_for(input.ec_id); + let provider_filled: Vec<&SourceDomain> = if lookups_apply { + input + .lookups + .iter() + .flat_map(|lookup| lookup.entries.iter().map(|(source, _)| source)) + .collect() + } else { + Vec::new() + }; + apply_browser_ingest( + &mut entry, + input.browser_ingest, + &BrowserIngestRules { + registry: input.registry, + provider_filled: &provider_filled, + now: input.now, + }, + ); + let written = if lookups_apply { apply_lookups(&mut entry, input.lookups, input.now) } else { if !input.lookups.is_empty() { @@ -82,19 +106,46 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } } +/// What browser ingest needs to decide provider-owned fallbacks. +struct BrowserIngestRules<'a> { + registry: &'a PartnerRegistry, + /// Sources the provider returned entries for on this navigation. + provider_filled: &'a [&'a SourceDomain], + now: UnixSeconds, +} + fn apply_browser_ingest( entry: &mut KvEntry, updates: &[PartnerIdUpdate], - registry: &PartnerRegistry, + rules: &BrowserIngestRules<'_>, ) { for update in updates { - if registry.is_provider_owned(&update.partner_id) { + if !rules.registry.is_provider_owned(&update.partner_id) { + entry + .ids + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); continue; } - entry + let provider_filled = rules + .provider_filled + .iter() + .any(|source| source.as_str() == update.partner_id); + let held = entry .ids - .entry(update.partner_id.clone()) - .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + .get(&update.partner_id) + .is_some_and(|record| record.has_live_uid(rules.now)); + if provider_filled || held { + continue; + } + log::debug!( + "EC write planner: ingesting browser fallback for provider-owned source '{}'", + update.partner_id + ); + entry.ids.insert( + update.partner_id.clone(), + KvPartnerId::single(update.uid.clone()), + ); } } @@ -156,7 +207,7 @@ fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { #[cfg(test)] mod tests { use super::*; - use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; @@ -435,4 +486,167 @@ mod tests { assert!(kept.len() < 40, "should drop some uids"); assert_eq!(kept[0].id().as_str(), ids[0], "should drop from the end"); } + + fn ingest_owned() -> [PartnerIdUpdate; 1] { + [PartnerIdUpdate::new("ids.example.com", "browser-id")] + } + + fn failed_lookup(status: LookupStatus) -> LookupResults { + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status, + entries: Vec::new(), + }], + ) + } + + fn provider_entry(ids: &[&str]) -> KvEntry { + let mut entry = live_entry(); + let lookups = found("ids.example.com", ids); + let (_, eids) = lookups.iter().next().expect("should hold a lookup").entries[0].clone(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids(eids, UnixSeconds::new(1000)).expect("should build record"), + ); + entry + } + + #[test] + fn provider_owned_source_takes_browser_fallback_when_lookup_fails() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + + for status in [ + LookupStatus::NoMatch, + LookupStatus::Skipped(SkipReason::NotDue), + ] { + let lookups = failed_lookup(status); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!( + record.uid, "browser-id", + "should store the browser fallback" + ); + assert!( + !record.is_provider_written(), + "should mark the fallback as not provider-written" + ); + } + } + + #[test] + fn provider_owned_source_takes_browser_fallback_without_lookups() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert_eq!( + entry.ids["ids.example.com"].uid, "browser-id", + "should store the browser fallback when no provider ran" + ); + } + + #[test] + fn browser_fallback_is_not_stored_when_provider_found_the_source() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should store the provider uid"); + assert!(record.is_provider_written(), "should keep provider eids"); + } + + #[test] + fn browser_fallback_never_overwrites_a_live_provider_uid() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should leave the live provider uid untouched" + ); + } + + #[test] + fn browser_fallback_does_not_overwrite_an_existing_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("older")); + let snapshot = present(&entry); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn later_provider_result_replaces_browser_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("browser-id"), + ); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + None, + &ingest_owned(), + &lookups, + ®istry, + )) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should replace the fallback"); + assert!(record.is_provider_written(), "should store provider eids"); + } } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 60f47a9be6..8922920b21 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -803,6 +803,14 @@ Lookups run only for an existing or newly issued EC, only when consent allows EID persistence, and, when `tcf_vendor_id` is set, only with TCF vendor consent for that vendor. +A provider owns its `sources`: its results replace whatever the row holds for +them, and batch sync rejects them. Browser-resolved IDs (`ts-eids`, `sharedId`, +`/auction` body EIDs) for an owned source are stored only as a fallback, when +the row has no live provider-written UID for the source and the provider kept +nothing for it on that navigation. The fallback follows the normal browser +ingest rules, never overwrites an existing value, and does not stop later +lookups: the next successful lookup replaces it. + | Field | Type | Required | Description | | ----------------- | ------- | -------- | ----------------------------------------------------------------------- | | `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | From 1b84e3c07e09f00775bf06f8d5159bca243b80db Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:33:03 +0530 Subject: [PATCH 11/58] Use an example host in the identify record test Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/identify.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 26a6e195e4..18346249c7 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -788,7 +788,7 @@ mod tests { kv.create(&ec_id, &entry).expect("should seed KV entry"); let req = Request::builder() .method("GET") - .uri("https://edge.test-publisher.com/identify") + .uri("https://edge.example.com/identify") .header("authorization", format!("Bearer {VALID_API_TOKEN}")) .body(EdgeBody::empty()) .expect("should build test request"); From 7db9e8817489fd254f97f8853e89358241d0940a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:59:10 +0530 Subject: [PATCH 12/58] Send sec-fetch-mode navigate from the EC integration test client Page-load writes only run on real navigations, which browsers mark with both sec-fetch-dest: document and sec-fetch-mode: navigate. Without the mode header the consent withdrawal scenario never got a write permit, so the EC cookie was kept. Signed-off-by: dhruv8sh --- crates/trusted-server-integration-tests/tests/common/ec.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index 0a1f149c3b..1846fb9602 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -113,6 +113,7 @@ impl EcTestClient { .client .get(format!("{}{path}", self.base_url)) .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") .header("accept", "text/html"); for (key, value) in headers { builder = builder.header(*key, *value); From 7c35af40cfe0889cb86e04932b76c29aeaa9f6a6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 19:59:12 +0530 Subject: [PATCH 13/58] Skip identity lookups without EC consent A lookup sends the EC ID to the provider before finalize can decline the EC, so GPC, sale opt-out and unknown-jurisdiction requests must not dispatch. Matches the gate pull sync already applies. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 8fa406cbcb..75e92b7446 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -23,6 +23,7 @@ use self::validate::{ValidatedEntries, ValidationPolicy, validate}; use crate::consent::ConsentContext; use crate::consent::{allows_eid_persistence, has_tcf_vendor_consent}; use crate::ec::EcKvSnapshot; +use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::kv_types::KvEntry; use crate::ec::page_load::RowChange; @@ -433,9 +434,11 @@ impl IdentityCoordinator for ProviderCoordinator

{ return pending; } let entry = lookup.snapshot.entry_for(lookup.ec_id); - // A withdrawal tombstone or missing EID consent skips every provider. - let consented = - allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); + // Missing EC or EID consent, or a withdrawal tombstone, skips every + // provider: a lookup discloses the EC ID before finalize could decline it. + let consented = ec_consent_granted(lookup.consent) + && allows_eid_persistence(lookup.consent) + && entry.is_none_or(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); for (index, registered) in self.providers.iter().enumerate() { @@ -919,6 +922,53 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn missing_ec_consent_skips_every_provider() { + let coordinator = coordinator(""); + let cases = [ + ( + "GPC opt-out in a US state", + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..Default::default() + }, + ), + ( + "unknown jurisdiction", + ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "GDPR without a TC string", + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ]; + + for (label, consent) in cases { + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + + let results = run(&coordinator, &transport, &seeded_snapshot(&entry), &consent); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip for {label}" + ); + assert!( + transport.sent.borrow().is_empty(), + "should send nothing for {label}" + ); + } + } + #[test] fn missing_tcf_vendor_consent_skips_before_building_a_request() { let coordinator = coordinator("tcf_vendor_id = 9999"); @@ -942,6 +992,7 @@ mod tests { purposes[0] = true; purposes[3] = true; let consent = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, gdpr_applies: true, tcf: Some(TcfConsent { version: 2, From 2de6d0d1022591e8ba9bee4aea746196c3da933b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 18:24:59 +0530 Subject: [PATCH 14/58] Raise the UID length limit to 1024 bytes and drop oversized IDs on KV read LiveRamp envelopes are about 700 bytes, so the 512-byte limit dropped every one from KV and the auction. Rows written under a higher limit now load with the oversized IDs removed instead of failing as a whole. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/admin.rs | 2 +- .../src/ec/coordinator/validate.rs | 2 +- crates/trusted-server-core/src/ec/kv.rs | 31 ++-- crates/trusted-server-core/src/ec/kv_types.rs | 132 +++++++++++++++++- .../trusted-server-core/src/ec/pull_sync.rs | 4 +- docs/guide/integrations/prebid.md | 30 ++-- 6 files changed, 169 insertions(+), 32 deletions(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 383edddf15..7014721f0e 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -1417,7 +1417,7 @@ mod tests { #[test] fn eids_lookup_reports_configured_source_without_valid_uid() { - let oversized_uid = "x".repeat(513); + let oversized_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let cookie = eids_cookie_for(&serde_json::json!([{ "source": "bidstream.example", "uids": [ diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 25cd7332f1..e7c8107ea9 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -238,7 +238,7 @@ mod tests { let partners = registry(); let provider = ProviderId::new("example-provider"); let claimed = claimed(); - let long = "x".repeat(600); + let long = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let mut expired = uid("u-old"); expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 434c38bbb9..9a3f9492ee 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -267,12 +267,20 @@ impl KvIdentityGraph { ec_id: &str, body_bytes: &[u8], ) -> Result> { - let entry: KvEntry = + let mut entry: KvEntry = serde_json::from_slice(body_bytes).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: format!("Failed to deserialize entry for key '{}'", log_id(ec_id)), })?; + let dropped = entry.drop_unreadable_ids(); + if dropped > 0 { + log::warn!( + "Dropped {dropped} unreadable partner IDs loading key '{}'", + log_id(ec_id) + ); + } + entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -1289,7 +1297,7 @@ mod tests { } #[test] - fn deserialize_entry_rejects_invalid_legacy_values() { + fn deserialize_entry_drops_oversized_ids() { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), @@ -1297,14 +1305,19 @@ mod tests { "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), ), ); - let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); + entry.ids.insert( + "ssp_y".to_owned(), + crate::ec::kv_types::KvPartnerId::single("kept"), + ); + let body = serde_json::to_vec(&entry).expect("should serialize oversized entry payload"); - let err = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) - .expect_err("should reject invalid legacy entry values"); - let err_text = format!("{err}"); - assert!( - err_text.contains("Loaded invalid entry"), - "should report validation failure for loaded entries" + let loaded = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) + .expect("should load an entry holding an oversized ID"); + + assert_eq!( + loaded.ids.keys().collect::>(), + vec!["ssp_y"], + "should drop only the oversized ID" ); } diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 45ac8b6df6..b97a37892e 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -6,9 +6,11 @@ //! //! The schema is versioned (`v: 1`) to allow future migrations. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, de}; +use serde_json::value::RawValue; use super::eid_types::{ AgentType, EidFields, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, UidValue, @@ -34,9 +36,10 @@ pub const MAX_SEEN_DOMAINS: usize = 50; pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// Maximum allowed length (in bytes) for a partner UID across all sync -/// mechanisms (pixel, batch, pull). Defined centrally to ensure consistent -/// validation. -pub const MAX_UID_LENGTH: usize = 512; +/// mechanisms (pixel, batch, pull) and identity providers. Sized for +/// `LiveRamp` envelopes (about 700 bytes). Defined centrally to ensure +/// consistent validation. +pub const MAX_UID_LENGTH: usize = 1024; /// Full KV entry stored as the body of an EC identity graph record. /// @@ -187,6 +190,24 @@ impl KvPartnerId { *self != before } + /// Drops entries left without UIDs and resyncs `uid`. + /// + /// Returns whether the record still holds a UID within + /// [`MAX_UID_LENGTH`]. + fn retain_readable(&mut self) -> bool { + if self.eids.is_empty() { + return self.uid.len() <= MAX_UID_LENGTH; + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + !self.eids.is_empty() + } + /// Removes the last UID of the last entry and resyncs `uid`; used by the /// write planner's size cap. pub(crate) fn drop_last_uid(&mut self) { @@ -227,9 +248,39 @@ struct KvEidRepr { mm: Option, #[serde(default)] ext: Option, + #[serde(deserialize_with = "deserialize_readable_uids")] uids: Vec, } +/// The `id` of a stored UID, read without validating the rest of it. +#[derive(Deserialize)] +struct StoredId<'a> { + #[serde(borrow)] + id: Cow<'a, str>, +} + +/// Deserializes stored UIDs, skipping any whose `id` exceeds +/// [`MAX_UID_LENGTH`]. +/// +/// A row written under a higher limit then still loads; an entry left +/// without UIDs is removed by [`KvEntry::drop_unreadable_ids`]. Any other +/// invalid UID still fails the row. +fn deserialize_readable_uids<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let raw = Vec::>::deserialize(deserializer)?; + let mut uids = Vec::with_capacity(raw.len()); + for value in &raw { + let oversized = serde_json::from_str::>(value.get()) + .is_ok_and(|stored| stored.id.len() > MAX_UID_LENGTH); + if oversized { + continue; + } + uids.push(serde_json::from_str(value.get()).map_err(de::Error::custom)?); + } + Ok(uids) +} + impl From for KvEid { fn from(repr: KvEidRepr) -> Self { let KvEidRepr { @@ -504,6 +555,17 @@ pub(crate) fn validated_stored_domain(domain: &str) -> Option { } impl KvEntry { + /// Removes partner records holding no UID this build can serve. + /// + /// Lets this build load rows written by one with a higher + /// [`MAX_UID_LENGTH`] instead of rejecting the whole row. Returns the + /// number of records removed. + pub fn drop_unreadable_ids(&mut self) -> usize { + let before = self.ids.len(); + self.ids.retain(|_, record| record.retain_readable()); + before - self.ids.len() + } + /// Creates a new live entry from the current request context. /// /// `domain` is the publisher's apex domain (e.g. `"example.com"`), @@ -1543,6 +1605,68 @@ mod tests { assert!(entry.validate().is_err(), "should reject EID with no UIDs"); } + #[test] + fn validate_accepts_liveramp_sized_uid() { + let entry = KvEntry::minimal("liveramp.example.com", &"x".repeat(703), 1); + assert!( + entry.validate().is_ok(), + "should accept a 703-byte envelope-sized UID" + ); + } + + #[test] + fn drop_unreadable_ids_removes_oversized_single_uid() { + let mut entry = KvEntry::minimal("ids.example.com", "kept", 1); + entry.ids.insert( + "long.example.com".to_owned(), + KvPartnerId::single("x".repeat(MAX_UID_LENGTH + 1)), + ); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the oversized record"); + assert_eq!( + entry.ids.keys().collect::>(), + vec!["ids.example.com"], + "should keep the readable record" + ); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + + #[test] + fn deserialize_drops_oversized_provider_uids() { + let long = "x".repeat(MAX_UID_LENGTH + 1); + let json = serde_json::json!({ + "v": SCHEMA_VERSION, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "partial.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] }, + { "uids": [{ "id": "short", "atype": 1 }] } + ] }, + "gone.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] } + ] } + } + }) + .to_string(); + let mut entry: KvEntry = + serde_json::from_str(&json).expect("should parse a row with oversized UIDs"); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the record left without UIDs"); + let partial = entry + .ids + .get("partial.example.com") + .expect("should keep the record with a readable UID"); + assert_eq!(partial.uid, "short", "should resync uid to the kept UID"); + assert_eq!(partial.eids().len(), 1, "should drop the emptied entry"); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + #[test] fn provider_record_round_trips_eid_level_fields() { let ext = serde_json::value::to_raw_value(&serde_json::json!({ "k": 1 })) diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 4aa8617f62..24993c18f1 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -798,12 +798,12 @@ mod tests { #[test] fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(513); + let long_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let body = format!("{{\"uid\":\"{long_uid}\"}}"); let response = make_response(200, body.as_bytes()); let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding 512 bytes"); + assert!(uid.is_none(), "should reject uid exceeding MAX_UID_LENGTH"); } #[test] diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index b7a05ac7be..3b314f5bec 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -902,20 +902,20 @@ server-to-server ATS API. ### Degraded behavior -| Condition | Result | -| -------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | -| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | -| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | -| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | -| LiveRamp network resolution fails | The current auction continues without RampID | -| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | -| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | -| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | -| The resolved envelope exceeds the 512-byte EID value cap | The envelope is dropped from both the `/auction` payload and EC persistence; the auction continues | -| The CMP is callable but never returns a settled result | The managed entry is never seeded; no vendor call, no identity storage, and the auction continues | -| Two managed entries address one Prebid submodule | The later entry is dropped with a logged error; the first entry's configuration takes effect | -| A silent CMP stub is replaced by a working CMP | The shim re-subscribes to the replacement; a CMP that already answered once keeps its own wait | +| Condition | Result | +| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | +| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | +| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | +| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | +| LiveRamp network resolution fails | The current auction continues without RampID | +| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | +| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | +| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | +| The resolved envelope exceeds the 1024-byte EID value cap | The envelope is dropped from both the `/auction` payload and EC persistence; the auction continues | +| The CMP is callable but never returns a settled result | The managed entry is never seeded; no vendor call, no identity storage, and the auction continues | +| Two managed entries address one Prebid submodule | The later entry is dropped with a logged error; the first entry's configuration takes effect | +| A silent CMP stub is replaced by a working CMP | The shim re-subscribes to the replacement; a CMP that already answered once keeps its own wait | The TCF rows assume either the managed-ID automatic setup described above or a publisher-owned Prebid GDPR configuration. A CMP API and its policy remain @@ -932,7 +932,7 @@ domain, booleans, source names, counts, and status codes: `identityLink` entry with the test Placement ID. 2. With positive consent, confirm `idl_env` is created or refreshed. 3. Record the byte length of the resolved envelope — the length only, never the - value — and confirm it is at or below the 512-byte EID value cap. Anything + value — and confirm it is at or below the 1024-byte EID value cap. Anything above it is dropped as described in the degraded-behavior table. 4. Confirm `pbjs.getUserIdsAsEids()` reports source `liveramp.com` without recording its value. From ff5113dfa8c22b6b61fb6097e9e6ec46efb7eb02 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:31:47 +0530 Subject: [PATCH 15/58] Gate TCF vendor consent only where TCF applies and skip lookups on unreadable rows Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/consent/mod.rs | 9 +++ .../src/ec/coordinator/mod.rs | 71 +++++++++++++++---- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 83bf3f33db..5d232df8a9 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -296,6 +296,15 @@ pub fn has_tcf_vendor_consent(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bo effective_tcf(ctx).is_some_and(|tcf| tcf.has_vendor_consent(vendor_id.get())) } +/// Whether TCF rules apply and withhold vendor consent from `vendor_id`. +/// +/// Returns `false` when no TCF string applies and GDPR does not, so a TCF +/// vendor requirement only gates traffic that TCF governs. +#[must_use] +pub fn tcf_vendor_consent_denied(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + (ctx.gdpr_applies || effective_tcf(ctx).is_some()) && !has_tcf_vendor_consent(ctx, vendor_id) +} + /// Returns whether a server-side auction may be dispatched for this request. /// /// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 75e92b7446..1dec09044c 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -21,7 +21,7 @@ use self::transport::{ }; use self::validate::{ValidatedEntries, ValidationPolicy, validate}; use crate::consent::ConsentContext; -use crate::consent::{allows_eid_persistence, has_tcf_vendor_consent}; +use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; @@ -434,9 +434,12 @@ impl IdentityCoordinator for ProviderCoordinator

{ return pending; } let entry = lookup.snapshot.entry_for(lookup.ec_id); - // Missing EC or EID consent, or a withdrawal tombstone, skips every - // provider: a lookup discloses the EC ID before finalize could decline it. - let consented = ec_consent_granted(lookup.consent) + // Missing EC or EID consent, a withdrawal tombstone, or an unreadable + // row (which may hide a tombstone) skips every provider: a lookup + // discloses the EC ID before finalize could decline it. + let readable = !matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); + let consented = readable + && ec_consent_granted(lookup.consent) && allows_eid_persistence(lookup.consent) && entry.is_none_or(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); @@ -450,7 +453,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ let vendor_denied = registered .policy .tcf_vendor_id - .is_some_and(|vendor| !has_tcf_vendor_consent(lookup.consent, vendor)); + .is_some_and(|vendor| tcf_vendor_consent_denied(lookup.consent, vendor)); if !consented || vendor_denied { pending.early.push(skip(SkipReason::Consent)); continue; @@ -970,28 +973,42 @@ mod tests { } #[test] - fn missing_tcf_vendor_consent_skips_before_building_a_request() { - let coordinator = coordinator("tcf_vendor_id = 9999"); + fn failed_snapshot_skips_every_provider() { + let coordinator = coordinator(""); let transport = CannedTransport::new(ok_reply); + let snapshot = EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }; - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &snapshot, &granting()); assert_eq!( only_status(&results), LookupStatus::Skipped(SkipReason::Consent), - "should skip" + "should skip when the row could not be read" ); assert!(transport.sent.borrow().is_empty(), "should send nothing"); } #[test] - fn granted_tcf_vendor_consent_allows_the_lookup() { + fn tcf_vendor_id_does_not_gate_traffic_without_tcf() { let coordinator = coordinator("tcf_vendor_id = 9999"); let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert!( + !matches!(only_status(&results), LookupStatus::Skipped(_)), + "should look up when no TCF string applies outside GDPR" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + fn tcf_consent(vendor_consents: Vec) -> ConsentContext { let mut purposes = vec![false; 24]; purposes[0] = true; purposes[3] = true; - let consent = ConsentContext { + ConsentContext { jurisdiction: Jurisdiction::Gdpr, gdpr_applies: true, tcf: Some(TcfConsent { @@ -1006,14 +1023,40 @@ mod tests { last_updated_ds: 0, purpose_consents: purposes, purpose_legitimate_interests: vec![false; 24], - vendor_consents: vec![9999], + vendor_consents, vendor_legitimate_interests: Vec::new(), special_feature_opt_ins: vec![false; 12], }), ..ConsentContext::default() - }; + } + } + + #[test] + fn missing_tcf_vendor_consent_skips_before_building_a_request() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &tcf_consent(vec![1])); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn granted_tcf_vendor_consent_allows_the_lookup() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &consent); + let results = run( + &coordinator, + &transport, + &missing(), + &tcf_consent(vec![9999]), + ); assert!( matches!(only_status(&results), LookupStatus::Found { .. }), From f0e2fb065dfaf272875f3d2f2445af2489b83287 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:33:15 +0530 Subject: [PATCH 16/58] Mint a new EC only on navigations that may write its row Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 175898ae53..d3bdafb189 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -958,7 +958,10 @@ async fn dispatch_fallback( } } let is_publisher_navigation = ec.is_real_browser && is_navigation; + // A new EC is minted only when this request may also write its row, + // so a client without Fetch Metadata never receives a throwaway EC. if is_publisher_navigation + && has_permit && let Err(err) = ec .ec_context .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) @@ -2737,6 +2740,58 @@ mod tests { ); } + fn consented_navigation(mut request: Request) -> Request { + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + request + } + + #[test] + fn navigation_without_fetch_metadata_mints_no_ec() { + let router = test_router(); + let mut request = consented_navigation(empty_request(Method::GET, "/article")); + request + .headers_mut() + .insert(header::ACCEPT, HeaderValue::from_static("text/html")); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + + let mut response = route(&router, request); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_none(), + "should not mint an EC that no row can back" + ); + } + + #[test] + fn navigation_with_fetch_metadata_mints_an_ec() { + let router = test_router(); + + let mut response = route( + &router, + consented_navigation(navigation_request("/article")), + ); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_some(), + "should mint an EC on a page-load navigation" + ); + } + #[test] fn publisher_navigation_without_providers_dispatches_nothing() { let router = test_router(); From 97e680600572b24e1af18485e51fc99cac4ded33 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:35:07 +0530 Subject: [PATCH 17/58] Reject batch sync records for provider-owned sources Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/batch_sync.rs | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 4903589b29..66a894dad8 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -33,6 +33,7 @@ const REASON_INVALID_EC_ID: &str = "invalid_ec_id"; const REASON_INVALID_PARTNER_UID: &str = "invalid_partner_uid"; const REASON_INELIGIBLE: &str = "ineligible"; const REASON_KV_UNAVAILABLE: &str = "kv_unavailable"; +const REASON_PROVIDER_OWNED: &str = "provider_owned"; /// Maximum number of mappings allowed in a single batch request. const MAX_BATCH_SIZE: usize = 1000; @@ -167,7 +168,12 @@ fn handle_batch_sync_with_writer( } // 4. Process mappings with per-item validation and rejection reasons. - let (accepted, errors) = process_mappings(writer, &partner.source_domain, &body.mappings); + let (accepted, errors) = process_mappings_for_partner( + writer, + &partner.source_domain, + registry.is_provider_owned(&partner.source_domain), + &body.mappings, + ); let rejected = errors.len(); let status = if rejected > 0 { @@ -193,6 +199,30 @@ fn content_length_exceeds_limit(req: &Request, max_body_size: usize) - .is_some_and(|content_length| content_length > max_body_size) } +/// Processes mappings for one partner, rejecting all of them when an enabled +/// identity provider owns the partner's source (spec §8.5). +fn process_mappings_for_partner( + writer: &dyn BatchSyncWriter, + partner_id: &str, + provider_owned: bool, + mappings: &[SyncMapping], +) -> (usize, Vec) { + if !provider_owned { + return process_mappings(writer, partner_id, mappings); + } + log::info!( + "Batch sync: rejecting {} mappings for provider-owned source '{partner_id}'", + mappings.len() + ); + let errors = (0..mappings.len()) + .map(|index| MappingError { + index, + reason: REASON_PROVIDER_OWNED, + }) + .collect(); + (0, errors) +} + /// Validates all mappings, then processes each normalized EC ID once. /// /// Successful and eligibility outcomes fan out to every valid input in a @@ -544,6 +574,27 @@ mod tests { assert_eq!(errors[1].reason, REASON_INVALID_PARTNER_UID); } + #[test] + fn provider_owned_source_rejects_every_mapping() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Written)]); + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = + process_mappings_for_partner(&writer, "ids.example.com", true, &mappings); + + assert_eq!(accepted, 0, "should accept nothing"); + assert_eq!(errors.len(), 2, "should reject each mapping"); + assert!( + errors + .iter() + .all(|error| error.reason == REASON_PROVIDER_OWNED), + "should report provider_owned" + ); + } + #[test] fn process_mappings_aborts_on_kv_unavailable() { let writer = MockWriter::new(vec![ From 665f8d3bb414d8b2f8319899c32bec9fe22d964b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:34:20 +0530 Subject: [PATCH 18/58] Skip pull sync for provider-owned sources Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/registry.rs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 5fa2677167..435e164b80 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -305,12 +305,16 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } - /// Returns all partners with `pull_sync_enabled = true`. + /// Returns all partners with `pull_sync_enabled = true` whose source no + /// identity provider owns. + /// + /// A provider writes full-fidelity records for the sources it owns, so a + /// pull-sync write there would overwrite them with a single UID. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { self.by_source_domain .values() - .filter(|p| p.pull_sync_enabled) + .filter(|p| p.pull_sync_enabled && !p.provider_owned) .collect() } @@ -700,6 +704,25 @@ mod tests { assert!(result.is_err(), "should reject invalid source domain"); } + #[test] + fn pull_enabled_partners_excludes_provider_owned_sources() { + let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); + pull_partner.api_token = None; + pull_partner.pull_sync_enabled = true; + pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); + pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; + pull_partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); + let mut registry = + PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); + + registry.mark_provider_owned("pull.example.com"); + + assert!( + registry.pull_enabled_partners().is_empty(), + "should not pull sources an identity provider owns" + ); + } + #[test] fn pull_enabled_partners_filters_correctly() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); From c4684f6a1e3a70f26b1ba5f67a82e63960f49403 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:34:50 +0530 Subject: [PATCH 19/58] Document navigation-only browser EID writes and the narrowed TCF vendor gate Signed-off-by: dhruv8sh --- docs/guide/configuration.md | 8 ++++---- docs/guide/edge-cookies.md | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 0ff626d2e5..3cc9402a78 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -800,12 +800,12 @@ finalized. Results go into the navigation's single EC row write. A lookup that times out or fails leaves the row unchanged. Lookups run only for an existing or newly issued EC, only when consent allows -EID persistence, and, when `tcf_vendor_id` is set, only with TCF vendor consent -for that vendor. +EID persistence, and, when `tcf_vendor_id` is set and TCF applies to the +request, only with TCF vendor consent for that vendor. A provider owns its `sources`: its results replace whatever the row holds for -them, and batch sync rejects them. Browser-resolved IDs (`ts-eids`, `sharedId`, -`/auction` body EIDs) for an owned source are stored only as a fallback, when +them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs +(`ts-eids`, `sharedId`) for an owned source are stored only as a fallback, when the row has no live provider-written UID for the source and the provider kept nothing for it on that navigation. The fallback follows the normal browser ingest rules, never overwrites an existing value, and does not stop later diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..527268a57f 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -148,13 +148,12 @@ flowchart LR ### Prebid EID Flow -Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a response. It collects matched partner UIDs from three request-local sources, applied in this order so that, when two sources carry the same partner, the later source's UID is the candidate offered to KV: +Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a top-level document navigation, the only request that writes KV. It collects matched partner UIDs from two request-local sources, applied in this order so that, when both carry the same partner, the later source's UID is the candidate offered to KV: 1. the `ts-eids` cookie, -2. the `eids` array in the `/auction` request body, -3. the `sharedId` cookie. +2. the `sharedId` cookie. -The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. +The `eids` array in the `/auction` request body goes to bidders on that request only and is never stored. The `ts-eids` cookie bridges Prebid user ID modules with the identity graph on the next navigation. These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. @@ -173,8 +172,8 @@ sequenceDiagram TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] TSJS->>B: document.cookie = "ts-eids=..." - Note over B,TS: Next eligible EID-sync request - B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids + Note over B,TS: Next page load + B->>TS: Top-level document navigation TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) ``` @@ -183,11 +182,11 @@ Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shap The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. -Returning-user cookie persistence runs only on publisher document navigations, `POST /auction`, and admitted `GET /_ts/page-bids` SPA navigations. Static assets, analytics, integration requests, filter short circuits, and other subresources do not decode or persist these cookies. New EC creation remains eligible regardless of route because its backing row must include the request's initial IDs. +Cookie persistence runs only on top-level document navigations (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`). `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. -Each eligible request attempts at most one conditional cookie update. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. +Each navigation attempts at most one conditional write. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. -Browser cookies and the `/auction` request body do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. +Browser cookies do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. ### EID Seeding and Prebid Bidstream Forwarding From 09cad520c0d7dde93aaa7e380a26bd8122162a30 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:35:53 +0530 Subject: [PATCH 20/58] Skip the snapshot preload for a freshly generated EC Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/publisher.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b70fad1355..2a68d2ab8f 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3458,8 +3458,9 @@ struct EcSnapshotPreloadInput { } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { - // An earlier step of this request (identity-provider dispatch) already - // read the row for an existing EC, so a second read adds nothing. + // An earlier step of this request already settled the row snapshot + // (identity-provider dispatch, or EC generation's authoritative miss), so + // a read adds nothing. if input.snapshot_already_read { return false; } @@ -5313,10 +5314,12 @@ pub async fn handle_publisher_request( .is_some_and(|registry| !registry.is_empty()), eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), - // A freshly generated EC keeps its refresh read: the Add-confirmed - // snapshot may still upgrade to the stored row. - snapshot_already_read: !ec_context.ec_generated() - && !matches!(ec_context.kv_snapshot(), crate::ec::EcKvSnapshot::NotRead), + // A freshly generated EC already holds an authoritative `Missing` + // snapshot: its row is only created by this navigation's write. + snapshot_already_read: !matches!( + ec_context.kv_snapshot(), + crate::ec::EcKvSnapshot::NotRead + ), }); let mut pending_origin = None; // A shared-template hit skips the origin entirely, and an in-flight request From 17072f45c57b56e90a2bc99adb3171b0afbe074a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:40:55 +0530 Subject: [PATCH 21/58] Build the partner registry once per app state Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 79 ++++++++++++++++--- .../trusted-server-adapter-fastly/src/main.rs | 17 +++- .../src/lib.rs | 47 +++++------ 3 files changed, 106 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d3bdafb189..f9187ab0d5 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -88,7 +88,7 @@ use std::cell::RefCell; use std::net::IpAddr; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use edgezero_adapter_fastly::context::FastlyRequestContext; @@ -123,6 +123,7 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::provider::ClientSignals; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; @@ -151,7 +152,7 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use trusted_server_identity_providers::{build_coordinator, partner_registry}; +use trusted_server_identity_providers::{build_coordinator_with, partner_registry}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -200,6 +201,27 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, + /// Partner registry built on first use and reused for the state's lifetime. + pub(crate) partner_registry: OnceLock>, +} + +impl AppState { + /// Returns the partner registry, building it on first use. + /// + /// A failed build is not cached, so every caller sees its error. + /// + /// # Errors + /// + /// Returns the same errors as [`partner_registry`]. + pub(crate) fn partner_registry( + &self, + ) -> Result, Report> { + if let Some(registry) = self.partner_registry.get() { + return Ok(Arc::clone(registry)); + } + let registry = Arc::new(partner_registry(&self.settings.ec)?); + Ok(Arc::clone(self.partner_registry.get_or_init(|| registry))) + } } /// Build the application state, loading settings and constructing all per-application components. @@ -245,6 +267,7 @@ pub(crate) fn build_state_from_settings( registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, + partner_registry: OnceLock::new(), })) } @@ -383,7 +406,23 @@ async fn dispatch_identity_lookups( req: &Request, ec: &mut EcRequestState, ) -> Option { - let coordinator = match build_coordinator(&state.settings) { + if !state + .settings + .ec + .identity_providers + .iter() + .any(|config| config.enabled) + { + return None; + } + let partners = match state.partner_registry() { + Ok(partners) => partners, + Err(report) => { + log::error!("Identity providers disabled for this request: {report:?}"); + return None; + } + }; + let coordinator = match build_coordinator_with(&state.settings, partners) { Ok(Some(coordinator)) => coordinator, Ok(None) => return None, Err(report) => { @@ -678,7 +717,8 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = partner_registry(&state.settings.ec) + let response = state + .partner_registry() .and_then(|registry| match handler { NamedRouteHandler::AdminEcLookup => { // Deliberately do not use an EC request-state graph: that @@ -759,7 +799,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; handle_identify( &state.settings, &kv, @@ -773,11 +813,11 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; handle_auction( &state.settings, @@ -797,11 +837,11 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; let auction = AuctionDispatch { integration_registry: &state.registry, @@ -844,7 +884,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = partner_registry(&state.settings.ec)?; + let partner_registry = state.partner_registry()?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -985,13 +1025,13 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match partner_registry(&state.settings.ec) { + match state.partner_registry() { Ok(partner_registry) => { let auction = AuctionDispatch { integration_registry: &state.registry, orchestrator: &state.orchestrator, slots, - registry: Some(&partner_registry), + registry: Some(&*partner_registry), }; match handle_publisher_request( &state.settings, @@ -1749,6 +1789,7 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + partner_registry: std::sync::OnceLock::new(), }); TrustedServerApp::routes_for_state(&state) } @@ -2753,6 +2794,19 @@ mod tests { request } + #[test] + fn partner_registry_is_built_once_per_state() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + + let first = state.partner_registry().expect("should build the registry"); + let second = state.partner_registry().expect("should reuse the registry"); + + assert!( + Arc::ptr_eq(&first, &second), + "should return the cached registry" + ); + } + #[test] fn navigation_without_fetch_metadata_mints_no_ec() { let router = test_router(); @@ -2993,6 +3047,7 @@ mod tests { let result = crate::apply_edgezero_ec_finalize( &settings, + None, &mut state, Some(super::IdentityLookups { coordinator, diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index efff0d4952..7c8340bdb8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -553,7 +553,13 @@ fn edgezero_main( if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, identity, &mut response) { + match apply_edgezero_ec_finalize( + settings, + app_state.as_deref(), + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -574,6 +580,7 @@ fn edgezero_main( Ok(settings) => { match apply_edgezero_ec_finalize( &settings, + None, &mut ec_state, identity, &mut response, @@ -657,10 +664,11 @@ fn apply_entry_point_finalize_headers( /// panics. fn apply_edgezero_ec_finalize( settings: &Settings, + state: Option<&AppState>, ec_state: &mut EcFinalizeState, identity: Option, response: &mut HttpResponse, -) -> Result> { +) -> Result, Report> { // A redirect skips the write, so do not wait for lookups it would discard. let identity = identity.filter(|_| { let redirect = NavigationPermit::is_redirect(response.status()); @@ -679,7 +687,10 @@ fn apply_edgezero_ec_finalize( let results = futures::executor::block_on(coordinator.collect(&transport, pending)); ec_state.ec_context.set_lookup_results(results); } - let partner_registry = partner_registry(&settings.ec)?; + let partner_registry = match state { + Some(state) => state.partner_registry()?, + None => Arc::new(partner_registry(&settings.ec)?), + }; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index dd6c086f43..d7206687ed 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -64,37 +64,40 @@ pub fn build( pub fn build_coordinator( settings: &Settings, ) -> Result, Report> { - let enabled: Vec<&IdentityProviderConfig> = settings + let partners = partner_registry(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + build_coordinator_with(settings, Arc::new(partners)) +} + +/// Builds the coordinator for every enabled provider in `settings` against an +/// already built partner registry. +/// +/// Enabled providers with an unregistered kind are skipped; building the +/// registry with [`partner_registry`] already warned about them. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. +pub fn build_coordinator_with( + settings: &Settings, + partners: Arc, +) -> Result, Report> { + let registered = settings .ec .identity_providers .iter() - .filter(|config| config.enabled) - .filter(|config| { - let registered = is_registered_kind(config.kind()); - if !registered { - log::warn!( - "Identity provider '{}' skipped: kind '{}' is not registered", - config.id, - config.kind() - ); - } - registered - }) - .collect(); - if enabled.is_empty() { - return Ok(None); - } - let registered = enabled - .into_iter() + .filter(|config| config.enabled && is_registered_kind(config.kind())) .map(|config| { build(config, &config.secrets) .map(|provider| RegisteredProvider::new(config.clone(), provider)) }) .collect::, Report>>()?; - let partners = partner_registry(&settings.ec) - .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + if registered.is_empty() { + return Ok(None); + } let budget = Duration::from_millis(settings.ec.identity_budget_ms); - ProviderCoordinator::new(registered, Arc::new(partners), budget).map(Some) + ProviderCoordinator::new(registered, partners, budget).map(Some) } #[cfg(test)] From 54079ae99c6c33e472e6fa112f234dce173977db Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:42:52 +0530 Subject: [PATCH 22/58] Remove EID resolution, create-if-absent and proxy inputs left unused Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-axum/src/app.rs | 3 - .../src/app.rs | 3 - .../trusted-server-adapter-fastly/src/app.rs | 2 - crates/trusted-server-adapter-spin/src/app.rs | 3 - crates/trusted-server-core/src/ec/eids.rs | 197 +++--------------- crates/trusted-server-core/src/ec/kv.rs | 54 ----- .../src/integrations/registry.rs | 30 +-- 7 files changed, 29 insertions(+), 263 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 314580e79a..be6589a103 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -224,15 +224,12 @@ async fn dispatch_fallback( } if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); return state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services, req, }) diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 54029b36c7..b0c9599e34 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -535,15 +535,12 @@ fn build_router(state: &Arc) -> RouterService { EdgeCacheHeader::CloudflareCdnCacheControl, ) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index f9187ab0d5..675a1fc898 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -959,8 +959,6 @@ async fn dispatch_fallback( method: &method, path: &path, settings: &state.settings, - kv: ec.kv_graph.as_ref(), - ec_context: &mut ec.ec_context, services, req, }) diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index f5fec2efc7..9a35062fc7 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -817,15 +817,12 @@ fn build_router(state: &Arc) -> RouterService { let result = if method == Method::GET && path.starts_with("/static/tsjs=") { handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 904594c934..8bd39614a8 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -1,8 +1,8 @@ //! Shared EID resolution and formatting helpers. //! -//! Used by both `/_ts/api/v1/identify` and `/auction` to resolve source-domain -//! keyed IDs from KV entries, convert them to `OpenRTB` EID structures, and -//! build base64-encoded response headers. +//! Used by both `/_ts/api/v1/identify` and `/auction` to view source-domain +//! keyed IDs from KV entries as `OpenRTB` EID structures, and to build +//! base64-encoded response headers. use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use error_stack::{Report, ResultExt as _}; @@ -17,67 +17,6 @@ use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. pub const MAX_EIDS_HEADER_BYTES: usize = 4096; -/// A source-domain keyed ID resolved from a KV entry against the partner registry. -/// -/// Only includes partners with `bidstream_enabled = true` and a non-empty UID. -pub struct ResolvedPartnerId { - /// The partner's identity source domain and EC KV `ids` key. - pub source_domain: String, - /// The synced user ID value. - pub uid: String, - /// `OpenRTB` agent type for this partner's identifiers. - pub openrtb_atype: i32, -} - -/// Resolves source-domain keyed IDs from a KV entry against the partner registry. -/// -/// Filters to partners with `bidstream_enabled = true` and non-empty UIDs, -/// sorted deterministically by source domain. -#[must_use] -pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec { - let mut resolved = Vec::new(); - - for (source_domain, partner_uid) in &entry.ids { - if partner_uid.uid.is_empty() { - continue; - } - - let Some(partner) = registry.get(source_domain) else { - continue; - }; - if !partner.bidstream_enabled { - continue; - } - - resolved.push(ResolvedPartnerId { - source_domain: partner.source_domain.clone(), - uid: partner_uid.uid.clone(), - openrtb_atype: partner.openrtb_atype, - }); - } - - resolved.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - resolved -} - -/// Converts resolved partner IDs to `OpenRTB` `Eid` entries. -#[must_use] -pub fn to_eids(resolved: &[ResolvedPartnerId]) -> Vec { - resolved - .iter() - .map(|item| { - Eid::new( - item.source_domain.clone(), - vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], - ) - }) - .collect() -} - /// A borrowed EID entry: a single legacy UID or a full-fidelity provider entry. #[derive(Debug, Clone, Copy)] pub enum EidView<'row> { @@ -179,28 +118,11 @@ impl KvEntry { } } -/// Builds a base64-encoded EID header value, truncating if needed. -/// -/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds -/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, partners are removed -/// from the end of the deterministic partner ordering until it fits. -/// -/// # Errors -/// -/// Returns an error if JSON serialization fails. -pub fn build_eids_header( - resolved: &[ResolvedPartnerId], -) -> Result<(String, bool), Report> { - let eids = to_eids(resolved); - encode_eids_header(&eids) -} - /// Encodes a pre-built EID slice into a base64 header value with truncation. /// -/// Like [`build_eids_header`] but operates on already-constructed `Eid` values -/// (e.g., from `UserInfo.eids` in the auction response path). -/// -/// Returns `(encoded_value, was_truncated)`. +/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds +/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, EIDs are removed from the +/// end until it fits. /// /// # Errors /// @@ -273,89 +195,30 @@ mod tests { } } - #[test] - fn resolve_partner_ids_sorts_by_source_domain() { - let partners = vec![ - make_test_partner("zeta.example.com"), - make_test_partner("alpha.example.com"), - ]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); - - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "zeta.example.com".to_owned(), - super::super::kv_types::KvPartnerId::single("uid-z".to_owned()), - ); - entry.ids.insert( - "alpha.example.com".to_owned(), - super::super::kv_types::KvPartnerId::single("uid-a".to_owned()), - ); - - let resolved = resolve_partner_ids(®istry, &entry); - let source_domains: Vec<&str> = resolved - .iter() - .map(|item| item.source_domain.as_str()) - .collect(); - - assert_eq!( - source_domains, - vec!["alpha.example.com", "zeta.example.com"], - "should sort deterministically by source domain" - ); + fn header_eid(source_domain: String, uid: String) -> Eid { + Eid::new( + source_domain, + vec![Uid { + id: uid, + atype: Some(3), + ext: None, + }], + ) } #[test] - fn to_eids_maps_resolved_ids_correctly() { - let resolved = vec![ - ResolvedPartnerId { - uid: "LR_xyz".to_owned(), - source_domain: "liveramp.com".to_owned(), - openrtb_atype: 3, - }, - ResolvedPartnerId { - uid: "ID5_abc".to_owned(), - source_domain: "id5-sync.com".to_owned(), - openrtb_atype: 1, - }, - ResolvedPartnerId { - uid: "pair-id".to_owned(), - source_domain: "google.com".to_owned(), - openrtb_atype: 571187, - }, - ]; - - let eids = to_eids(&resolved); - - assert_eq!(eids.len(), 3, "should produce one EID per resolved partner"); - assert_eq!(eids[0].source, "liveramp.com"); - assert_eq!(eids[0].uids[0].id, "LR_xyz"); - assert_eq!(eids[0].uids[0].atype, Some(3)); - assert_eq!(eids[1].source, "id5-sync.com"); - assert_eq!(eids[1].uids[0].id, "ID5_abc"); - assert_eq!(eids[1].uids[0].atype, Some(1)); - assert_eq!(eids[2].source, "google.com", "should preserve PAIR source"); - assert_eq!(eids[2].uids[0].id, "pair-id", "should preserve PAIR ID"); - assert_eq!( - eids[2].uids[0].atype, - Some(571187), - "should preserve PAIR vendor-specific atype" - ); - } - - #[test] - fn build_eids_header_truncates_when_too_large() { - let mut resolved = Vec::new(); - for idx in 0..64 { - resolved.push(ResolvedPartnerId { - uid: format!("uid_{}", "x".repeat(100)), - source_domain: format!("partner-{idx}.example.com"), - openrtb_atype: 3, - }); - } + fn encode_eids_header_truncates_when_too_large() { + let eids: Vec = (0..64) + .map(|idx| { + header_eid( + format!("partner-{idx}.example.com"), + format!("uid_{}", "x".repeat(100)), + ) + }) + .collect(); let (encoded, truncated) = - build_eids_header(&resolved).expect("should build truncated header"); + encode_eids_header(&eids).expect("should build truncated header"); assert!(truncated, "should report truncation for large payload"); assert!( @@ -365,15 +228,11 @@ mod tests { } #[test] - fn build_eids_header_fits_without_truncation() { - let resolved = vec![ResolvedPartnerId { - uid: "u1".to_owned(), - source_domain: "ssp.com".to_owned(), - openrtb_atype: 3, - }]; + fn encode_eids_header_fits_without_truncation() { + let eids = vec![header_eid("ssp.example.com".to_owned(), "u1".to_owned())]; let (encoded, truncated) = - build_eids_header(&resolved).expect("should build header without truncation"); + encode_eids_header(&eids).expect("should build header without truncation"); assert!(!truncated, "should not truncate small payload"); assert!(!encoded.is_empty(), "should produce non-empty value"); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 9a3f9492ee..7d074d1145 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -62,15 +62,6 @@ pub enum UpsertResult { Unchanged, } -/// Outcome of atomically creating an identity-graph root when absent. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CreateIfAbsentOutcome { - /// The candidate entry was persisted. - Written, - /// A row already exists for the candidate key. - AlreadyExists, -} - /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PartnerIdUpdate { @@ -341,23 +332,6 @@ impl KvIdentityGraph { } } - /// Atomically creates an entry while preserving collision as normal control flow. - /// - /// # Errors - /// - /// Returns [`TrustedServerError::KvStore`] when serialization or store I/O fails. - pub fn create_if_absent( - &self, - ec_id: &str, - entry: &KvEntry, - ) -> Result> { - let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; - match self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? { - EcKvWriteOutcome::Written => Ok(CreateIfAbsentOutcome::Written), - EcKvWriteOutcome::PreconditionFailed => Ok(CreateIfAbsentOutcome::AlreadyExists), - } - } - /// Low-level write with shared error context. fn write_entry( &self, @@ -1746,34 +1720,6 @@ mod tests { ); } - #[test] - fn create_if_absent_reports_written_and_collision() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should create absent entry"), - CreateIfAbsentOutcome::Written - ); - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should report collision"), - CreateIfAbsentOutcome::AlreadyExists - ); - } - - #[test] - fn create_if_absent_propagates_store_error() { - let kv = KvIdentityGraph::failing("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - assert!( - kv.create_if_absent(&ec_id, &live_entry()).is_err(), - "should preserve store failures instead of reporting a collision" - ); - } - #[test] fn create_or_revive_revives_tombstone() { let kv = KvIdentityGraph::in_memory("test_store"); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index fc40711240..0460984238 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -10,8 +10,6 @@ use matchit::Router; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; -use crate::ec::EcContext; -use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; @@ -785,15 +783,11 @@ impl IntegrationMetadata { /// Inputs to [`IntegrationRegistry::handle_proxy`]. /// -/// Bundled into a struct so the dispatch surface stays within the project's -/// 7-argument cap; `ec_context` and `req` participate in the borrow so the -/// whole thing shares one lifetime. +/// Bundled into a struct so the dispatch surface shares one lifetime. pub struct ProxyDispatchInput<'a> { pub method: &'a Method, pub path: &'a str, pub settings: &'a Settings, - pub kv: Option<&'a KvIdentityGraph>, - pub ec_context: &'a mut EcContext, pub services: &'a RuntimeServices, pub req: Request, } @@ -1028,8 +1022,6 @@ impl IntegrationRegistry { method, path, settings, - kv: _, - ec_context: _, services, mut req, } = input; @@ -1825,14 +1817,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { method: &http::Method::GET, path: "/integrations/test/echo", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &noop_services(), req, })) @@ -2179,8 +2167,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = noop_services(); // Call handle_proxy (uses futures executor in test environment) @@ -2188,8 +2174,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); @@ -2229,8 +2213,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("evil;injected"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2238,8 +2220,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2278,8 +2258,6 @@ mod tests { )) .expect("should build Cookie header"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2287,8 +2265,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2324,8 +2300,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2333,8 +2307,6 @@ mod tests { method: &Method::POST, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); From a987581b9e7defec4c3383b304bc6b7dc5fbf19f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:44:03 +0530 Subject: [PATCH 23/58] Track the row size incrementally while enforcing the size cap Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/write_plan.rs | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 60c60eb009..5046bf82da 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -170,37 +170,73 @@ fn serialized_len(entry: &KvEntry) -> usize { serde_json::to_vec(entry).map_or(usize::MAX, |bytes| bytes.len()) } +/// Serialized bytes one `ids` member adds to the row: its record, its +/// escaped key, the colon and a separating comma. +fn member_len(source: &str, record: &KvPartnerId) -> usize { + let key = serde_json::to_vec(source).map_or(usize::MAX, |bytes| bytes.len()); + let value = serde_json::to_vec(record).map_or(usize::MAX, |bytes| bytes.len()); + key.saturating_add(value).saturating_add(2) +} + +/// Shrinks `entry` until it serializes within [`MAX_ENTRY_BYTES`]. +/// +/// # Performance +/// +/// Tracks the row size by the bytes each drop removes and serializes the +/// whole row again only when that estimate reaches the cap, so the cost is +/// linear in the number of dropped UIDs and records. fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { - while serialized_len(entry) > MAX_ENTRY_BYTES { - if let Some(source) = written.iter().rev().find(|source| { + let mut size = serialized_len(entry); + // Drop this navigation's provider UIDs first, newest source last-in first. + while size > MAX_ENTRY_BYTES { + let Some(source) = written.iter().rev().find(|source| { entry .ids .get(source.as_str()) .is_some_and(KvPartnerId::is_provider_written) - }) { - let record = entry - .ids - .get_mut(source.as_str()) - .expect("should hold a written source"); - record.drop_last_uid(); - if record.uid.is_empty() { - entry.ids.remove(source.as_str()); - } - continue; - } - // No provider UIDs from this navigation are left: drop whole records, - // largest first, so every later write still fits the cap. - let Some(largest) = entry + }) else { + break; + }; + let record = entry .ids - .iter() - .max_by_key(|(_, record)| serde_json::to_vec(record).map_or(usize::MAX, |b| b.len())) - .map(|(source, _)| source.clone()) - else { + .get_mut(source.as_str()) + .expect("should hold a written source"); + let before = member_len(source.as_str(), record); + record.drop_last_uid(); + let after = if record.uid.is_empty() { + entry.ids.remove(source.as_str()); + 0 + } else { + member_len(source.as_str(), record) + }; + size = size.saturating_sub(before.saturating_sub(after)); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } + } + if size <= MAX_ENTRY_BYTES { + return; + } + // No provider UIDs from this navigation are left: drop whole records, + // largest first, so every later write still fits the cap. + let mut by_size: Vec<(usize, String)> = entry + .ids + .iter() + .map(|(source, record)| (member_len(source, record), source.clone())) + .collect(); + // A stable ascending sort pops the last of equally large records first. + by_size.sort_by_key(|(len, _)| *len); + while size > MAX_ENTRY_BYTES { + let Some((len, largest)) = by_size.pop() else { log::warn!("EC write planner: row exceeds size cap with no partner IDs left to drop"); return; }; log::warn!("EC write planner: dropping '{largest}' to fit the row size cap"); entry.ids.remove(&largest); + size = size.saturating_sub(len); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } } } From e681663a8d2d2579fa70231f66f59b6a42a2c1c8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 4 Nov 2026 00:45:11 +0530 Subject: [PATCH 24/58] Reuse the core clock for lookups and use should-style expect messages Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-fastly/src/app.rs | 4 +--- crates/trusted-server-core/src/ec/finalize.rs | 12 ++++++------ crates/trusted-server-core/src/ec/kv.rs | 6 +++--- crates/trusted-server-core/src/ec/mod.rs | 3 ++- crates/trusted-server-core/src/ec/pull_sync.rs | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 675a1fc898..aef63f6009 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -446,9 +446,7 @@ async fn dispatch_identity_lookups( .headers() .get(header::USER_AGENT) .and_then(|value| value.to_str().ok()); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); + let now = trusted_server_core::ec::current_timestamp(); let pending = coordinator .dispatch( &transport, diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 1fc5c1a074..41105abe24 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1290,7 +1290,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("sharedid.org"), "a subresource response must not persist request EID cookies" @@ -1348,7 +1348,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), permit.then_some(cookie_id), @@ -1438,7 +1438,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( stored.ids.is_empty(), "an auction request must leave the row unchanged" @@ -1513,7 +1513,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( stored.ids.is_empty(), "should persist neither the auction body nor the cookie EIDs" @@ -1575,7 +1575,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("id5-sync.com"), "denying TCF Purpose 4 must keep the EID out of KV even though Purpose 1 \ @@ -1640,7 +1640,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("id5-sync.com"), "should keep the provider-owned fallback out of KV without Purpose 4" diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 7d074d1145..2adcba566f 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -1931,7 +1931,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read root after rejected hard delete") - .expect("rejected hard delete should retain root"); + .expect("should retain the root after a rejected hard delete"); assert!( !stored.consent.ok, "rejected hard delete should leave the tombstoned root in place" @@ -2520,7 +2520,7 @@ mod tests { let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("ssp_x").map(|id| id.uid.as_str()), Some("uid-1"), @@ -2597,7 +2597,7 @@ mod tests { let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!(stored.ids.is_empty(), "no update should reach the store"); } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 2a832aebdd..f90940d5be 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -780,7 +780,8 @@ impl EcContext { /// native and `wasm32-wasip1` targets and to a JS-backed clock on /// `wasm32-unknown-unknown` (Cloudflare Workers), where `std::time` is not /// available. -pub(crate) fn current_timestamp() -> u64 { +#[must_use] +pub fn current_timestamp() -> u64 { checked_current_timestamp().unwrap_or(0) } diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 24993c18f1..4fac813dd5 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -1186,7 +1186,7 @@ mod tests { let (entry, _) = graph .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!( entry.ids.is_empty(), "no partner UID may be written back onto a tombstoned row" From ef7ad61aada55347724a9c7e9097f9d5072a4b36 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 01:22:25 +0530 Subject: [PATCH 25/58] Collect provider IDs after the response and keep provider refresh state Providers set to collect = "after_response" look up after the page is sent and store their results with one compare-and-swap follow-up write (Fastly only). Providers may keep private refresh state with a UID, generate IDs, pass request context to their reply parser and report an opt-out. Refresh is due only while the UID is live. Unusable refresh state is dropped without dropping the UID. The admin EC view redacts refresh state. Operators can bound lookups per provider with rate_limit_per_hour. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 121 ++++-- .../trusted-server-adapter-fastly/src/main.rs | 102 ++++- crates/trusted-server-core/src/ec/admin.rs | 47 +++ .../src/ec/after_response.rs | 369 ++++++++++++++++++ .../src/ec/coordinator/mod.rs | 292 +++++++++++++- .../src/ec/coordinator/results.rs | 7 + .../src/ec/coordinator/transport.rs | 2 + .../src/ec/coordinator/validate.rs | 91 ++++- crates/trusted-server-core/src/ec/finalize.rs | 133 ++++++- crates/trusted-server-core/src/ec/kv.rs | 64 ++- crates/trusted-server-core/src/ec/kv_types.rs | 87 +++++ crates/trusted-server-core/src/ec/mod.rs | 32 +- .../trusted-server-core/src/ec/page_load.rs | 43 +- crates/trusted-server-core/src/ec/provider.rs | 45 +++ .../trusted-server-core/src/ec/write_plan.rs | 50 ++- crates/trusted-server-core/src/settings.rs | 37 +- .../src/lib.rs | 109 +++++- 17 files changed, 1517 insertions(+), 114 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/after_response.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index aef63f6009..5364bfc182 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -148,7 +148,7 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::settings::{ProxyAssetRoute, Settings}; +use trusted_server_core::settings::{CollectTiming, ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; @@ -368,31 +368,97 @@ pub(crate) struct EcFinalizeState { } /// Identity lookups dispatched for this navigation, collected before finalize. -/// -/// In-flight platform requests are not `Send`, so they cannot ride in -/// response extensions with [`EcFinalizeState`]. They travel through a -/// request-local slot instead: the entry point clears it before routing and -/// takes it after the router returns. pub(crate) struct IdentityLookups { pub(crate) coordinator: ProviderCoordinator, pub(crate) pending: PendingLookups, } -thread_local! { - static PENDING_IDENTITY_LOOKUPS: RefCell> = const { RefCell::new(None) }; +/// What a navigation's after-response identity lookups need once the response +/// has been sent. +/// +/// The router owns the request, so the request signals the providers read are +/// copied here. The coordinator is built and the lookups dispatched only after +/// the response, so neither delays the page. +pub(crate) struct AfterResponseLookups { + pub(crate) ec_id: String, + pub(crate) client_ip: Option, + pub(crate) user_agent: Option, +} + +/// Identity work a navigation hands to the entry point. +/// +/// In-flight platform requests are not `Send`, so they cannot ride in +/// response extensions with [`EcFinalizeState`]. They travel through one +/// request-local slot instead: the entry point clears it before routing and +/// takes it after the router returns. +#[derive(Default)] +pub(crate) struct NavigationLookups { + /// Page-phase lookups, collected before EC finalization. + pub(crate) page: Option, + /// After-response lookups, dispatched once the response has been sent. + pub(crate) after_response: Option, } -/// Stores this request's dispatched identity lookups for the entry point. -fn stash_identity_lookups(lookups: IdentityLookups) { - PENDING_IDENTITY_LOOKUPS.with(|slot| *slot.borrow_mut() = Some(lookups)); +thread_local! { + static NAVIGATION_LOOKUPS: RefCell = RefCell::new(NavigationLookups::default()); } -/// Takes the identity lookups dispatched for the current request, if any. +/// Takes the identity work stored for the current request. /// /// The entry point calls this before routing (discarding anything a prior /// request left behind) and after the router returns. -pub(crate) fn take_identity_lookups() -> Option { - PENDING_IDENTITY_LOOKUPS.with(|slot| slot.borrow_mut().take()) +pub(crate) fn take_navigation_lookups() -> NavigationLookups { + NAVIGATION_LOOKUPS.with(|slot| std::mem::take(&mut *slot.borrow_mut())) +} + +/// Request signals lent to identity providers. +struct ProviderRequestSignals<'a> { + client_ip: Option, + user_agent: Option<&'a str>, +} + +impl<'a> ProviderRequestSignals<'a> { + fn read(req: &'a Request, ec: &EcRequestState) -> Self { + Self { + client_ip: ec + .ec_context + .client_ip() + .and_then(|ip| ip.parse::().ok()), + user_agent: req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()), + } + } +} + +/// Captures the request signals for after-response identity lookups on a +/// permitted publisher navigation. +/// +/// Returns `None` when no after-response provider is enabled, the request has +/// no EC, or the identity graph is unavailable. +fn prepare_after_response_lookups( + state: &AppState, + req: &Request, + ec: &EcRequestState, +) -> Option { + let has_after_response_provider = state + .settings + .ec + .identity_providers + .iter() + .any(|config| config.enabled && config.collect == CollectTiming::AfterResponse); + if !has_after_response_provider { + return None; + } + ec.kv_graph.as_ref()?; + let ec_id = ec.ec_context.ec_value()?.to_owned(); + let signals = ProviderRequestSignals::read(req, ec); + Some(AfterResponseLookups { + ec_id, + client_ip: signals.client_ip, + user_agent: signals.user_agent.map(str::to_owned), + }) } /// Starts identity-provider lookups for a permitted publisher navigation. @@ -438,14 +504,7 @@ async fn dispatch_identity_lookups( } let transport = PlatformLookupTransport::new(services, state.settings.proxy.certificate_check); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - let client_ip = ec - .ec_context - .client_ip() - .and_then(|ip| ip.parse::().ok()); - let user_agent = req - .headers() - .get(header::USER_AGENT) - .and_then(|value| value.to_str().ok()); + let signals = ProviderRequestSignals::read(req, ec); let now = trusted_server_core::ec::current_timestamp(); let pending = coordinator .dispatch( @@ -455,8 +514,8 @@ async fn dispatch_identity_lookups( snapshot: ec.ec_context.kv_snapshot(), consent: ec.ec_context.consent(), client: ClientSignals { - ip: client_ip, - user_agent, + ip: signals.client_ip, + user_agent: signals.user_agent, }, publisher_domain: &state.settings.publisher.domain, rate_limiter: &limiter, @@ -1011,7 +1070,15 @@ async fn dispatch_fallback( && has_permit && let Some(lookups) = dispatch_identity_lookups(state, services, &req, &mut ec).await { - stash_identity_lookups(lookups); + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().page = Some(lookups)); + } + // After-response providers only capture request signals here; they + // are built and dispatched once the response has been sent. + if is_publisher_navigation + && has_permit + && let Some(lookups) = prepare_after_response_lookups(state, &req, &ec) + { + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().after_response = Some(lookups)); } // Run the server-side auction with the configured creative- @@ -2850,7 +2917,7 @@ mod tests { let _state = take_finalize_state(&mut response); assert!( - super::take_identity_lookups().is_none(), + super::take_navigation_lookups().page.is_none(), "should dispatch no identity lookups without enabled providers" ); } @@ -2922,6 +2989,7 @@ mod tests { url: url::Url::parse("https://ids.example.com/lookup").expect("should parse"), headers: edgezero_core::http::HeaderMap::new(), body: Bytes::new(), + context: None, }) } @@ -2942,6 +3010,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::Unspecified, + refresh: None, }], }])) } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 7c8340bdb8..6a59e836fd 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,12 +13,16 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::after_response::commit_after_response_lookups; use trusted_server_core::ec::coordinator::IdentityCoordinator as _; +use trusted_server_core::ec::coordinator::LookupContext; use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -31,7 +35,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; -use trusted_server_identity_providers::partner_registry; +use trusted_server_identity_providers::{build_after_response_coordinator, partner_registry}; mod app; mod backend; @@ -48,8 +52,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, IdentityLookups, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, take_identity_lookups, + AfterResponseLookups, AppState, EcFinalizeState, IdentityLookups, NavigationLookups, + RuntimeStoreConfig, TrustedServerApp, load_settings_from_config_store, take_navigation_lookups, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -496,7 +500,7 @@ fn edgezero_main( } core_req.extensions_mut().insert(client_info); // Drop lookups a previous request on this sandbox left behind. - drop(take_identity_lookups()); + drop(take_navigation_lookups()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -525,7 +529,10 @@ fn edgezero_main( let ec_state = response.extensions_mut().remove::(); // Identity lookups dispatched on a publisher navigation are collected in // EC finalization, after the router's executor has returned. - let identity = take_identity_lookups(); + let NavigationLookups { + page: identity, + after_response, + } = take_navigation_lookups(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); @@ -567,6 +574,12 @@ fn edgezero_main( counters.as_ref(), ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + run_after_response_lookups( + settings, + &partner_registry, + &mut ec_state, + after_response, + ); return; } Err(e) => { @@ -596,6 +609,12 @@ fn edgezero_main( &partner_registry, &ec_state, ); + run_after_response_lookups( + &settings, + &partner_registry, + &mut ec_state, + after_response, + ); return; } Err(e) => { @@ -735,6 +754,79 @@ fn run_edgezero_pull_sync_after_send( ); } +/// Runs after-response identity lookups and applies their follow-up write. +/// +/// Called once the response has been sent, so provider latency never delays +/// the page. Skipped unless finalize confirmed the navigation (it issues the +/// follow-up permit) and the EC is still the one the lookups were prepared +/// for. +fn run_after_response_lookups( + settings: &Settings, + partner_registry: &PartnerRegistry, + ec_state: &mut EcFinalizeState, + lookups: Option, +) { + let Some(lookups) = lookups else { + return; + }; + let Some(permit) = ec_state.ec_context.take_after_response_permit() else { + log::debug!( + "After-response lookups skipped: no permit (consent, redirect or failed page-load write)" + ); + return; + }; + if ec_state.ec_context.ec_value() != Some(lookups.ec_id.as_str()) { + log::debug!("After-response lookups skipped: EC changed during the request"); + return; + } + let coordinator = match build_after_response_coordinator(settings) { + Ok(Some(coordinator)) => coordinator, + Ok(None) => return, + Err(report) => { + log::error!("After-response identity providers disabled for this request: {report:?}"); + return; + } + }; + let Some(graph) = maybe_identity_graph(settings) else { + return; + }; + let snapshot = graph.load_snapshot(&lookups.ec_id); + let transport = + PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()); + let pending = futures::executor::block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &lookups.ec_id, + snapshot: &snapshot, + consent: ec_state.ec_context.consent(), + client: ClientSignals { + ip: lookups.client_ip, + user_agent: lookups.user_agent.as_deref(), + }, + publisher_domain: &settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + )); + if pending.is_empty() { + return; + } + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + let outcome = commit_after_response_lookups( + &graph, + permit, + partner_registry, + &lookups.ec_id, + &snapshot, + &results, + ); + log::info!("After-response lookups: write outcome={outcome}"); +} + fn prepare_pull_sync_after_send( context: Option, graph_factory: F, diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 7014721f0e..a0d8151a36 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -395,6 +395,7 @@ fn build_lookup_response( match serde_json::from_slice::(&lookup.body) { Ok(mut entry_json) => { add_iso_timestamp_companions(&mut entry_json); + redact_refresh_state(&mut entry_json); payload.entry = Some(entry_json); match serde_json::from_slice::(&lookup.body) { @@ -471,6 +472,30 @@ fn add_iso_timestamp_companions(entry_json: &mut JsonValue) { } } +/// Placeholder served in place of a provider's private refresh state. +const REDACTED: &str = "[redacted]"; + +/// Replaces every provider refresh `state` in the entry with [`REDACTED`]. +/// +/// Refresh state can hold a provider credential (for example a refresh +/// token), so the admin view shows only that it exists and when it is due. +fn redact_refresh_state(value: &mut JsonValue) { + match value { + JsonValue::Object(object) => { + if let Some(state) = object + .get_mut("refresh") + .and_then(JsonValue::as_object_mut) + .and_then(|refresh| refresh.get_mut("state")) + { + *state = JsonValue::String(REDACTED.to_owned()); + } + object.values_mut().for_each(redact_refresh_state); + } + JsonValue::Array(items) => items.iter_mut().for_each(redact_refresh_state), + _ => {} + } +} + /// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). /// /// Returns `None` for values outside the representable date range. @@ -1585,4 +1610,26 @@ mod tests { assert_eq!(view.skipped[0].source_domain, "expired.example"); assert_eq!(view.skipped[0].reason, "expired", "should report expiry"); } + + #[test] + fn redacts_provider_refresh_state() { + let mut entry = serde_json::json!({ + "ids": {"token.example": {"eids": [{"source": "token.example", "uids": [ + {"id": "token", "refresh": {"state": "secret-refresh", "from": 1_000}} + ]}]}} + }); + + redact_refresh_state(&mut entry); + + let refresh = &entry["ids"]["token.example"]["eids"][0]["uids"][0]["refresh"]; + assert_eq!(refresh["state"], REDACTED, "should hide the refresh state"); + assert_eq!( + refresh["from"], 1_000, + "should keep when the refresh is due" + ); + assert_eq!( + entry["ids"]["token.example"]["eids"][0]["uids"][0]["id"], "token", + "should keep the UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs new file mode 100644 index 0000000000..c6ecba1031 --- /dev/null +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -0,0 +1,369 @@ +//! The follow-up write of after-response identity lookups. +//! +//! Providers configured with `collect = "after_response"` are dispatched only +//! after the navigation's response has been sent, so their latency never +//! delays the page. Their validated results reach the row in one follow-up +//! update, guarded by the row's current generation, and are served from the +//! next page load. Browser ingest never runs here: it belongs to the +//! navigation's own write. + +use super::coordinator::results::LookupResults; +use super::eid_types::UnixSeconds; +use super::kv::KvIdentityGraph; +use super::page_load::{AfterResponseWrite, RowChange}; +use super::registry::PartnerRegistry; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, current_timestamp, log_id}; + +/// What the follow-up write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum AfterResponseOutcome { + /// The row was updated with the after-response results. + #[display("written")] + Written, + /// The results would not change the row. + #[display("unchanged")] + Unchanged, + /// The row is missing, withdrawn or unreadable, so nothing was written. + #[display("no_row")] + NoRow, + /// The row changed since it was read; the results are dropped. + #[display("conflict")] + Conflict, + /// The store rejected the write. + #[display("failed")] + Failed, +} + +/// Applies after-response lookup results to the row of `ec_id`. +/// +/// Plans an update from `lookups` alone against `snapshot`, the row read +/// after the response was sent, and writes it once, guarded by the snapshot's +/// generation. The `permit` is consumed whether or not a write is +/// made. +/// +/// # Performance +/// +/// At most one conditional KV write; the row is read by the caller. +/// Runs after the response has been sent, so it adds no page latency. +#[must_use] +pub fn commit_after_response_lookups( + graph: &KvIdentityGraph, + permit: AfterResponseWrite, + registry: &PartnerRegistry, + ec_id: &str, + snapshot: &EcKvSnapshot, + lookups: &LookupResults, +) -> AfterResponseOutcome { + if lookups.is_empty() { + return AfterResponseOutcome::Unchanged; + } + if !matches!(snapshot, EcKvSnapshot::Present { .. }) { + log::info!( + "After-response lookups: no row for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + } + let change = plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry: None, + browser_ingest: &[], + lookups, + registry, + now: UnixSeconds::new(current_timestamp()), + }); + let Some(change) = change else { + // A withdrawn row also plans no change. + return match snapshot.entry_for(ec_id) { + Some(entry) if entry.consent.ok => AfterResponseOutcome::Unchanged, + _ => AfterResponseOutcome::NoRow, + }; + }; + let RowChange::Update { entry, expected } = change else { + log::warn!( + "After-response lookups: unexpected row change for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + }; + match graph.commit_after_response(permit, ec_id, &entry, expected) { + Ok(()) => AfterResponseOutcome::Written, + Err(report) => { + log::warn!( + "After-response write failed for '{}': {}", + log_id(ec_id), + report.current_context() + ); + match report.current_context() { + super::page_load::KvWriteError::Conflict => AfterResponseOutcome::Conflict, + _ => AfterResponseOutcome::Failed, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn registry() -> PartnerRegistry { + let partners: Vec = ["browser.example.com", "ids.example.com"] + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, + "source_domain": source, + "bidstream_enabled": true, + })) + .expect("should build partner") + }) + .collect(); + let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + assert!( + registry.mark_provider_owned("ids.example.com"), + "should mark the provider source as owned" + ); + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn found(ec_id: &str, id: &str) -> LookupResults { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + LookupResults::new( + ec_id, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("ids.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + ) + } + + fn stored_uid(graph: &KvIdentityGraph, source: &str) -> Option { + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .and_then(|entry| entry.ids.get(source).map(|record| record.uid.clone())) + } + + #[test] + fn writes_after_response_results_to_an_existing_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "browser.example.com".to_owned(), + KvPartnerId::single("b-1".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-1"), + "should keep the browser UID written by the page load" + ); + } + + #[test] + fn replaces_a_browser_fallback_for_an_owned_source() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("from-browser".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should replace the browser fallback with the provider UID" + ); + } + + #[test] + fn skips_the_write_when_the_row_already_holds_the_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let first = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + assert_eq!(first, AfterResponseOutcome::Written, "should write first"); + + let second = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + second, + AfterResponseOutcome::Unchanged, + "should not rewrite identical results" + ); + } + + #[test] + fn writes_nothing_without_a_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not create a row" + ); + assert!( + graph.load_snapshot(EC_ID).entry_for(EC_ID).is_none(), + "should leave the row absent" + ); + } + + #[test] + fn writes_nothing_to_a_withdrawn_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not revive a tombstone" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store the provider UID on a tombstone" + ); + } + + #[test] + fn drops_results_made_for_another_ec() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let other = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ABC123"; + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &found(other, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should discard results dispatched for another EC" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store results for another EC" + ); + } + + #[test] + fn skips_the_write_without_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &graph.load_snapshot(EC_ID), + &LookupResults::new(EC_ID, Vec::new()), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should make no write without results" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 1dec09044c..40de27dc8b 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -29,7 +29,8 @@ use crate::ec::kv_types::KvEntry; use crate::ec::page_load::RowChange; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, - ProviderResponse, SourceInput, destination_allowed, parse_sources, validate_enabled_configs, + ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, + validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; use crate::ec::registry::PartnerRegistry; @@ -103,6 +104,10 @@ pub struct PendingLookups { now: UnixSeconds, /// Providers that finished at dispatch (skipped or failed to send). early: Vec>, + /// Providers whose IDs were generated at dispatch, validated. + generated: Vec>, + /// Provider state to hand back with each in-flight request's response. + contexts: Vec>, } impl Default for PendingLookups { @@ -113,6 +118,8 @@ impl Default for PendingLookups { ec_id: String::new(), now: UnixSeconds::new(0), early: Vec::new(), + generated: Vec::new(), + contexts: Vec::new(), } } } @@ -122,15 +129,16 @@ impl std::fmt::Debug for PendingLookups { f.debug_struct("PendingLookups") .field("requests", &self.requests.len()) .field("early", &self.early.len()) + .field("generated", &self.generated.len()) .finish_non_exhaustive() } } impl PendingLookups { - /// Whether nothing was dispatched or skipped. + /// Whether nothing was dispatched, generated or skipped. #[must_use] pub fn is_empty(&self) -> bool { - self.requests.is_empty() && self.early.is_empty() + self.requests.is_empty() && self.early.is_empty() && self.generated.is_empty() } } @@ -285,10 +293,16 @@ impl ProviderCoordinator

{ let mut input_blocked = false; let mut due = Vec::new(); for source in ®istered.policy.sources { - let held = entry + let record = entry .and_then(|entry| entry.ids.get(source.as_str())) - .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)); - if held { + .filter(|record| record.is_provider_written()); + // A UID whose provider asked for a refresh makes its source due; + // the stored refresh state is the lookup's input. + if record.is_some_and(|record| record.refresh_due(now)) { + due.push(source.clone()); + continue; + } + if record.is_some_and(|record| record.has_live_uid(now)) { continue; } let needs_input = capabilities @@ -331,10 +345,17 @@ impl ProviderCoordinator

{ status: LookupStatus, entries: ValidatedEntries, ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let cleared = if status == LookupStatus::Skipped(SkipReason::OptedOut) { + registered.policy.sources.to_vec() + } else { + Vec::new() + }; ProviderLookup { - provider: self.providers[provider.get()].id.clone(), + provider: registered.id.clone(), status, entries, + cleared, } } @@ -342,6 +363,7 @@ impl ProviderCoordinator

{ &self, provider: ProviderIndex, response: Result>, + context: Option<&str>, now: UnixSeconds, ) -> ProviderLookup { let registered = &self.providers[provider.get()]; @@ -378,12 +400,20 @@ impl ProviderCoordinator

{ status: response.status, headers: &response.headers, body: &response.body, + context, }); let eids = match parsed { Ok(LookupOutcome::Found(eids)) => eids, Ok(LookupOutcome::NoMatch) => { return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); } + Ok(LookupOutcome::OptedOut) => { + return self.lookup( + provider, + LookupStatus::Skipped(SkipReason::OptedOut), + Vec::new(), + ); + } Err(report) => { log::debug!( "Identity lookup: provider={} response not parsed: {}", @@ -393,6 +423,17 @@ impl ProviderCoordinator

{ return self.lookup(provider, LookupStatus::InvalidResponse, Vec::new()); } }; + self.validated(provider, eids, now) + } + + /// Validates a provider's entries and reports them as one lookup. + fn validated( + &self, + provider: ProviderIndex, + eids: Vec>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; let policy = ValidationPolicy { provider: ®istered.id, claimed: ®istered.policy.sources, @@ -467,10 +508,6 @@ impl IdentityCoordinator for ProviderCoordinator

{ })); continue; } - if Self::rate_limited(registered, lookup) { - pending.early.push(skip(SkipReason::RateLimited)); - continue; - } let input = LookupInput { ec_id: lookup.ec_id, consent: lookup.consent, @@ -480,7 +517,18 @@ impl IdentityCoordinator for ProviderCoordinator

{ current: OwnedEntries::new(entry, ®istered.policy.sources), hashed_email: None, }; - let request = match registered.provider.build_request(&input) { + if let Some(eids) = registered.provider.generate(&input) { + pending.generated.push(Tagged { + provider, + value: self.validated(provider, eids, lookup.now), + }); + continue; + } + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } + let mut request = match registered.provider.build_request(&input) { Ok(request) => request, Err(report) => { log::debug!( @@ -496,8 +544,14 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::Destination)); continue; } + let context = request.context.take(); match transport.start(request, deadline).await { - Ok(value) => pending.requests.push(Tagged { provider, value }), + Ok(value) => { + pending.requests.push(Tagged { provider, value }); + if let Some(value) = context { + pending.contexts.push(Tagged { provider, value }); + } + } Err(report) => { log::warn!( "Identity lookup: provider={} send failed: {}", @@ -525,6 +579,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ ec_id, now, early, + generated, + contexts, } = pending; let responses = match deadline { Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, @@ -535,8 +591,15 @@ impl IdentityCoordinator for ProviderCoordinator

{ for Tagged { provider, value } in early { slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); } + for Tagged { provider, value } in generated { + slots[provider.get()] = Some(value); + } for Tagged { provider, value } in responses { - slots[provider.get()] = Some(self.interpret(provider, value, now)); + let context = contexts + .iter() + .find(|tagged| tagged.provider == provider) + .map(|tagged| tagged.value.as_str()); + slots[provider.get()] = Some(self.interpret(provider, value, context, now)); } let lookups: Vec = slots.into_iter().flatten().collect(); for lookup in &lookups { @@ -569,7 +632,7 @@ mod tests { use crate::consent::types::TcfConsent; use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; use crate::ec::kv::KvIdentityGraph; - use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; use crate::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, ProviderUid, SourceCapability, @@ -584,6 +647,9 @@ mod tests { type Reply = fn() -> Result>; + /// Context the echo provider sets on its request and expects back. + const ECHO_CONTEXT: &str = "echo-context"; + struct EchoProvider(ProviderCapabilities); impl IdentityProvider for EchoProvider { @@ -604,6 +670,7 @@ mod tests { .expect("should parse"), headers: http::HeaderMap::new(), body: bytes::Bytes::new(), + context: Some(ECHO_CONTEXT.to_owned()), }) } @@ -611,8 +678,14 @@ mod tests { &self, response: ProviderResponse<'body>, ) -> Result, Report> { + if response.context != Some(ECHO_CONTEXT) { + return Err(Report::new(ProviderError::InvalidResponse)); + } let id = std::str::from_utf8(response.body) .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + if id == "optout" { + return Ok(LookupOutcome::OptedOut); + } Ok(LookupOutcome::Found(vec![ProviderEid { source: Cow::Borrowed("ids.example.com"), inserter: None, @@ -624,6 +697,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::Unspecified, + refresh: None, }], }])) } @@ -767,8 +841,8 @@ mod tests { } } - fn run_with( - coordinator: &ProviderCoordinator, + fn run_with( + coordinator: &ProviderCoordinator

, transport: &CannedTransport, snapshot: &EcKvSnapshot, consent: &ConsentContext, @@ -792,8 +866,8 @@ mod tests { }) } - fn run( - coordinator: &ProviderCoordinator, + fn run( + coordinator: &ProviderCoordinator

, transport: &CannedTransport, snapshot: &EcKvSnapshot, consent: &ConsentContext, @@ -1206,4 +1280,184 @@ mod tests { "should report invalid config" ); } + + fn refreshable_record(from: u64) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(from), + }); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + fn snapshot_holding(record: KvPartnerId) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert("ids.example.com".to_owned(), record); + seeded_snapshot(&entry) + } + + #[test] + fn a_due_refresh_makes_a_live_source_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should refresh a live UID whose refresh time has passed" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); + } + + #[test] + fn a_future_refresh_leaves_the_source_held() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(2000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh before the refresh time" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + fn optout_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"optout")) + } + + #[test] + fn an_opted_out_reply_clears_the_sources() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(optout_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should report the opt-out" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + + /// An echo provider that generates its ID instead of looking it up. + struct GeneratingProvider(EchoProvider); + + impl IdentityProvider for GeneratingProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + Some(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed("generated-1"), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }]) + } + } + + fn generating_coordinator() -> ProviderCoordinator { + ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + GeneratingProvider(echo_provider(SourceInput::Anonymous)), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator") + } + + #[test] + fn a_generated_id_replaces_the_lookup_for_a_due_source() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "generated-1", + "should store the generated ID" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn a_held_source_is_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(provider_record("p-1")), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the held ID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 7fff308ea9..c8e6578f71 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -63,6 +63,9 @@ pub enum SkipReason { Destination, /// The provider's hourly rate limit is reached. RateLimited, + /// The vendor reports an opt-out; the provider's stored UIDs are + /// removed. + OptedOut, } /// One outcome per provider per navigation. @@ -106,6 +109,7 @@ impl LookupStatus { Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", Self::Skipped(SkipReason::Destination) => "destination_rejected", + Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", } } } @@ -119,6 +123,9 @@ pub struct ProviderLookup { pub status: LookupStatus, /// Validated entries per returned source; empty unless `Found`. pub entries: Vec<(SourceDomain, Vec)>, + /// Sources whose stored records are removed, because the user opted + /// out of the provider; empty unless `Skipped(OptedOut)`. + pub cleared: Vec, } /// Validated results of one navigation's lookups. diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs index e01f2c6a13..9402d6aab2 100644 --- a/crates/trusted-server-core/src/ec/coordinator/transport.rs +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -174,6 +174,7 @@ impl LookupTransport for PlatformLookupTransport<'_> { url, headers, body, + context: _, } = request; let host = url .host_str() @@ -317,6 +318,7 @@ mod tests { url: url::Url::parse(&format!("https://ids.example.com/{path}")).expect("should parse"), headers: http::HeaderMap::new(), body: bytes::Bytes::new(), + context: None, } } diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index e7c8107ea9..a189fcd25a 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -7,10 +7,13 @@ use crate::ec::eid_types::{ AgentType, EidFields, EidValidationError, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, SourceDomain, UidValue, UnixSeconds, }; -use crate::ec::kv_types::{KvEid, KvUid}; +use crate::ec::kv_types::{KvEid, KvRefresh, KvUid}; use crate::ec::provider::{EidLevelFields, EidShape, ProviderEid, ProviderExpiry, ProviderUid}; use crate::ec::registry::PartnerRegistry; +/// Largest private refresh state a provider may keep with one UID. +const MAX_REFRESH_STATE_BYTES: usize = 2048; + /// Policy one response is validated against. pub(crate) struct ValidationPolicy<'a> { /// Owner recorded on every kept UID. @@ -126,21 +129,43 @@ fn stored_uid( let id = UidValue::try_from(uid.id.into_owned())?; let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; let ext = uid.ext.map(RawJson::try_from).transpose()?; - let expiry = match uid.expiry { - ProviderExpiry::Unspecified => Expiry::Never, - ProviderExpiry::At(at) => Expiry::At(at), - ProviderExpiry::After(after) => Expiry::At(policy.now.saturating_add_secs(after.as_secs())), + let expiry = match resolve_expiry(uid.expiry, policy.now) { + Some(at) => Expiry::At(at), + None => Expiry::Never, }; if !expiry.is_live(policy.now) { return Err(EidValidationError::Expired); } - Ok(KvUid::new( - id, - atype, - ext, - Some(policy.provider.clone()), - expiry, - )) + let stored = KvUid::new(id, atype, ext, Some(policy.provider.clone()), expiry); + let Some(refresh) = uid.refresh else { + return Ok(stored); + }; + // Unusable refresh state is dropped, never the UID: a refresh with no + // stated time, or with empty or oversized state, is simply not stored. + let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; + match resolve_expiry(refresh.from, policy.now) { + Some(from) if usable => Ok(stored.with_refresh(KvRefresh { + state: refresh.state.into_owned(), + from, + })), + _ => { + log::debug!( + "Identity lookup: provider={} drops unusable refresh state", + policy.provider + ); + Ok(stored) + } + } +} + +/// Resolves a provider-stated time against `now`. `None` when no time was +/// stated. +fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { + match expiry { + ProviderExpiry::Unspecified => None, + ProviderExpiry::At(at) => Some(at), + ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), + } } fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { @@ -154,6 +179,7 @@ fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::ec::provider::ProviderRefresh; use crate::settings::EcPartner; use serde_json::value::RawValue; use std::num::NonZeroUsize; @@ -173,6 +199,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::Unspecified, + refresh: None, } } @@ -233,6 +260,46 @@ mod tests { ); } + #[test] + fn keeps_the_uid_when_its_refresh_state_is_unusable() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let in_a_minute = ProviderExpiry::After(Duration::from_secs(60)); + let cases = [ + ( + "state".to_owned(), + in_a_minute, + Some(UnixSeconds::new(1060)), + ), + (String::new(), in_a_minute, None), + ("x".repeat(MAX_REFRESH_STATE_BYTES + 1), in_a_minute, None), + ("state".to_owned(), ProviderExpiry::Unspecified, None), + ]; + + for (state, from, expected) in cases { + let mut refreshed = uid("u-1"); + refreshed.refresh = Some(ProviderRefresh { + state: Cow::Owned(state), + from, + }); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![refreshed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop no UID"); + assert_eq!( + entries[0].1[0].uids()[0] + .refresh() + .map(|refresh| refresh.from), + expected, + "should keep only usable refresh state" + ); + } + } + #[test] fn drops_invalid_values_by_reason() { let partners = registry(); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 41105abe24..055e4b2950 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -60,9 +60,10 @@ pub fn ec_finalize_response( response: &mut Response, ) { ec_context.validate_pull_sync_marker(settings, registry); - let permit = ec_context + let (permit, after_response) = ec_context .take_navigation_permit() - .and_then(|permit| permit.confirm(response.status())); + .and_then(|permit| permit.confirm_with_after_response(response.status())) + .unzip(); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -106,10 +107,17 @@ pub fn ec_finalize_response( updates: &updates, had_eid_updates, }; - if ec_context.ec_generated() { - commit_generated(&commit, ec_context, permit, response); + let page_write_ok = if ec_context.ec_generated() { + commit_generated(&commit, ec_context, permit, response) } else if ec_context.ec_was_present() { - commit_returning(&commit, ec_context, permit, response); + commit_returning(&commit, ec_context, permit, response) + } else { + false + }; + // The follow-up write is allowed only once consent allows the EC and + // the page-load write succeeded or had nothing to change. + if page_write_ok && let Some(after_response) = after_response { + ec_context.set_after_response_permit(after_response); } } else if ec_context.ec_generated() { // Do not emit a generated EC when there is no KV graph: that would @@ -162,19 +170,21 @@ fn present_snapshot(ec_id: &str, entry: KvEntry) -> EcKvSnapshot { } /// Creates the row of a newly generated EC and issues its cookie on success. +/// +/// Returns whether the row was created. fn commit_generated( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: Option, response: &mut Response, -) { +) -> bool { let Some(new_entry) = ec_context.take_pending_entry() else { log::warn!("Skipping generated EC cookie because no pending row was built"); - return; + return false; }; let Some(permit) = permit else { log::info!("Skipping generated EC: not a page-load navigation"); - return; + return false; }; let Some(change) = commit.plan( ec_context, @@ -183,12 +193,13 @@ fn commit_generated( Some(&new_entry), ) else { log::warn!("Skipping generated EC: no row change was planned"); - return; + return false; }; match commit.graph.commit_page_load(permit, commit.ec_id, change) { Ok(committed) => { ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); set_ec_cookie_on_response(commit.settings, ec_context, response); + true } Err(report) => { log::warn!( @@ -196,19 +207,23 @@ fn commit_generated( log_id(commit.ec_id), report.current_context() ); + false } } } /// Applies a returning EC's page-load write, or recovers an orphaned EC. +/// +/// Returns whether the row is live afterwards: written, recovered, or left +/// unchanged because there was nothing to write. fn commit_returning( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: Option, response: &mut Response, -) { +) -> bool { let Some(permit) = permit else { - return; + return false; }; // An unread snapshot is read only when this navigation has something to // write, or when consent gating emptied real EID data on a request that @@ -225,27 +240,38 @@ fn commit_returning( if ec_context.recovery_eligible() { confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); } - return; + return matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }); } - update_returning(commit, ec_context, permit); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Failed { .. }) { + return false; + } + update_returning(commit, ec_context, permit) } /// Writes the planned update for a returning EC whose row snapshot is held. +/// +/// Returns `false` only when a planned write failed. fn update_returning( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: PageLoadWrite, -) { +) -> bool { let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { - return; + return true; }; match commit.graph.commit_page_load(permit, commit.ec_id, change) { - Ok(committed) => ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)), - Err(report) => log::info!( - "Page-load write skipped for '{}': {}", - log_id(commit.ec_id), - report.current_context() - ), + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + true + } + Err(report) => { + log::info!( + "Page-load write skipped for '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + false + } } } @@ -815,6 +841,10 @@ mod tests { .expect("should replace the stale miss with a tombstone snapshot"); assert!(!snapshot_entry.consent.ok && snapshot_entry.ids.is_empty()); assert_eq!(ec_context.kv_snapshot().generation_for(&ec_id), None); + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups after a withdrawal" + ); } #[test] @@ -1065,6 +1095,7 @@ mod tests { SourceDomain::try_from("provider.example.com").expect("should build source"), vec![KvEid::new(EidFields::default(), vec![uid])], )], + cleared: Vec::new(), }], )); let graph = KvIdentityGraph::in_memory("test_store"); @@ -2770,4 +2801,64 @@ mod tests { entry.consent.ok = true; entry } + + #[test] + fn finalize_allows_after_response_lookups_on_a_live_row() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after1"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ) + .expect("should seed the identity"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_some(), + "should allow after-response lookups when the page write had nothing to change" + ); + } + + #[test] + fn finalize_withholds_after_response_lookups_when_the_row_read_failed() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after2"); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups when the page phase failed" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 2adcba566f..b0dc976387 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -22,7 +22,7 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; -use super::page_load::{KvWriteError, PageLoadWrite, RowChange}; +use super::page_load::{AfterResponseWrite, Generation, KvWriteError, PageLoadWrite, RowChange}; use super::{EcKvSnapshot, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -389,23 +389,61 @@ impl KvIdentityGraph { EcKvWriteMode::IfGenerationMatch(expected.get()), ), }; + self.write_row(ec_id, &entry, ttl, mode)?; + if is_tombstone { + // Deliberate exception to the one-write-per-page-load rule: + // the withdrawal-completion marker is a separate key that + // predates the rule. Later withdrawals use it to avoid a + // strongly consistent existence check on a row already + // withdrawn, and it never touches the identity row. + self.record_withdrawal_completion(ec_id, entry.consent.updated); + } + Ok(entry) + } + + /// Applies a navigation's follow-up update after its response was sent. + /// + /// The row is replaced only while it is still at `expected`, so a + /// concurrent write is never overwritten. The `permit` is consumed. + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, or [`KvWriteError::Store`] on a serialization or store failure, + /// including a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_after_response( + &self, + permit: AfterResponseWrite, + ec_id: &str, + entry: &KvEntry, + expected: Generation, + ) -> Result<(), Report> { + permit.spend(); + self.write_row( + ec_id, + entry, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ) + } + + /// Serializes `entry` and writes it with `mode`, mapping a failed + /// precondition to [`KvWriteError::AlreadyExists`] for a create and + /// [`KvWriteError::Conflict`] otherwise. + fn write_row( + &self, + ec_id: &str, + entry: &KvEntry, + ttl: Duration, + mode: EcKvWriteMode, + ) -> Result<(), Report> { let (body, metadata) = - Self::serialize_entry(&entry, self.store_name()).change_context(KvWriteError::Store)?; + Self::serialize_entry(entry, self.store_name()).change_context(KvWriteError::Store)?; let outcome = self .write_entry(ec_id, &body, &metadata, ttl, mode) .change_context(KvWriteError::Store)?; match outcome { - EcKvWriteOutcome::Written => { - if is_tombstone { - // Deliberate exception to the one-write-per-page-load rule: - // the withdrawal-completion marker is a separate key that - // predates the rule. Later withdrawals use it to avoid a - // strongly consistent existence check on a row already - // withdrawn, and it never touches the identity row. - self.record_withdrawal_completion(ec_id, entry.consent.updated); - } - Ok(entry) - } + EcKvWriteOutcome::Written => Ok(()), EcKvWriteOutcome::PreconditionFailed => Err(Report::new(match mode { EcKvWriteMode::Add => KvWriteError::AlreadyExists, _ => KvWriteError::Conflict, diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index b97a37892e..c6f401f40d 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -157,6 +157,15 @@ impl KvPartnerId { !self.eids.is_empty() } + /// Whether a UID in the record asks its provider for a refresh by `now`. + #[must_use] + pub fn refresh_due(&self, now: UnixSeconds) -> bool { + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.refresh_due(now)) + } + /// Whether the record holds a UID that may be served at `now`. #[must_use] pub fn has_live_uid(&self, now: UnixSeconds) -> bool { @@ -338,6 +347,29 @@ pub struct KvUid { skip_serializing_if = "Expiry::is_never" )] expiry: Expiry, + /// Private refresh state; never served, mirrored or forwarded. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. +/// +/// It stays in the row only: outbound EIDs are built from the UID's `id`, +/// `atype` and `ext`, so this state never reaches bidders or cookies. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework. + pub state: String, + /// When the owning provider should be asked to refresh the UID. + pub from: UnixSeconds, +} + +impl core::fmt::Debug for KvRefresh { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("KvRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } } impl KvUid { @@ -356,9 +388,36 @@ impl KvUid { ext, owner, expiry, + refresh: None, } } + /// Attaches the owning provider's private refresh state. + #[must_use] + pub fn with_refresh(mut self, refresh: KvRefresh) -> Self { + self.refresh = Some(refresh); + self + } + + /// The owning provider's private refresh state. + #[must_use] + pub fn refresh(&self) -> Option<&KvRefresh> { + self.refresh.as_ref() + } + + /// Whether the owning provider asked to refresh this live UID by `now`. + /// + /// An expired UID is never refresh-due: its source is due anyway, and a + /// refresh that keeps failing must not outlive the UID. + #[must_use] + pub fn refresh_due(&self, now: UnixSeconds) -> bool { + self.is_live(now) + && self + .refresh + .as_ref() + .is_some_and(|refresh| refresh.from <= now) + } + /// The UID value. #[must_use] pub fn id(&self) -> &UidValue { @@ -1732,4 +1791,32 @@ mod tests { assert!(entry.validate().is_err(), "should reject oversized {field}"); } } + + #[test] + fn refresh_is_due_only_while_the_uid_is_live() { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + None, + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(1000), + }); + + assert!( + !uid.refresh_due(UnixSeconds::new(999)), + "should wait for `from`" + ); + assert!( + uid.refresh_due(UnixSeconds::new(1000)), + "should be due at `from`" + ); + assert!( + !uid.refresh_due(UnixSeconds::new(5000)), + "should stop refreshing once the UID expires" + ); + } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f90940d5be..84c3147393 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -38,6 +38,7 @@ mod auth; pub mod admin; +pub mod after_response; pub mod batch_sync; pub mod consent; pub mod cookies; @@ -97,7 +98,7 @@ use device::DeviceSignals; use self::coordinator::results::LookupResults; use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; -use self::page_load::NavigationPermit; +use self::page_load::{AfterResponseWrite, NavigationPermit}; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. @@ -264,6 +265,17 @@ impl Clone for PermitSlot { } } +/// Holds the navigation's after-response permit. Like [`PermitSlot`], +/// cloning an [`EcContext`] never copies it. +#[derive(Debug, Default)] +struct AfterResponsePermitSlot(Option); + +impl Clone for AfterResponsePermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during @@ -308,6 +320,9 @@ pub struct EcContext { eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. navigation_permit: PermitSlot, + /// Permit for the navigation's follow-up write after its response, set + /// by finalize when it confirms the navigation permit. + after_response_permit: AfterResponsePermitSlot, /// Validated provider lookup results collected for this navigation. lookup_results: LookupResults, /// New row built by [`Self::generate_if_needed`], written by finalize. @@ -395,6 +410,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, }) @@ -623,6 +639,17 @@ impl EcContext { self.navigation_permit.0.take() } + /// Stores the after-response permit confirmed with the navigation permit. + pub(crate) fn set_after_response_permit(&mut self, permit: AfterResponseWrite) { + self.after_response_permit = AfterResponsePermitSlot(Some(permit)); + } + + /// Takes the after-response permit; returns `None` when finalize did not + /// confirm a navigation, or after the first call. + pub fn take_after_response_permit(&mut self) -> Option { + self.after_response_permit.0.take() + } + /// Takes the navigation permit, for adapter tests. #[cfg(any(test, feature = "test-utils"))] pub fn take_navigation_permit_for_test(&mut self) -> Option { @@ -709,6 +736,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, } @@ -737,6 +765,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, } @@ -768,6 +797,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), lookup_results: LookupResults::default(), pending_entry: None, } diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs index 7453d93e7f..5eddc555d5 100644 --- a/crates/trusted-server-core/src/ec/page_load.rs +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -3,7 +3,12 @@ //! A request classified as a top-level document navigation receives a //! [`NavigationPermit`]. Once the final response status is known it becomes a //! [`PageLoadWrite`], which [`KvIdentityGraph::commit_page_load`] consumes. -//! Neither type is `Clone`, so one navigation makes at most one write. +//! Neither type is `Clone`, so one navigation makes at most one write before +//! its response. +//! +//! The one exception is [`AfterResponseWrite`], confirmed alongside the page +//! load's permit. It allows one follow-up update after the response has been +//! sent, carrying only the results of after-response identity lookups. //! //! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load @@ -57,6 +62,19 @@ impl NavigationPermit { (!Self::is_redirect(status)).then_some(PageLoadWrite(())) } + /// Confirms the permit like [`Self::confirm`] and also issues the + /// navigation's [`AfterResponseWrite`]. + /// + /// Returns `None` for a redirect, which makes neither write. + #[must_use] + pub fn confirm_with_after_response( + self, + status: StatusCode, + ) -> Option<(PageLoadWrite, AfterResponseWrite)> { + self.confirm(status) + .map(|permit| (permit, AfterResponseWrite(()))) + } + /// Whether `status` is a redirect (301, 302, 303, 307, 308) on which the /// page load's write is skipped. /// @@ -91,6 +109,29 @@ impl PageLoadWrite { pub(crate) fn spend(self) {} } +/// Proof that the current navigation may make one follow-up update after its +/// response has been sent. +/// +/// Issued only by [`NavigationPermit::confirm_with_after_response`] and +/// consumed by [`KvIdentityGraph::commit_after_response`]. It is not `Clone`, +/// so a navigation makes at most one follow-up write. +/// +/// [`KvIdentityGraph::commit_after_response`]: super::kv::KvIdentityGraph::commit_after_response +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +impl AfterResponseWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} + + /// Builds a permit without a navigation, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + /// KV generation a conditional write expects. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Generation(u64); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index d2be63fb7a..28b2970180 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -57,6 +57,17 @@ pub trait IdentityProvider { &self, response: ProviderResponse<'body>, ) -> Result, Report>; + + /// Creates IDs for due sources without a lookup, for a provider whose + /// IDs the operator owns rather than a vendor server. + /// + /// Runs when a source is due. A returned entry goes through the same + /// validation and write plan as a lookup result, and no request is built. + /// The default creates nothing. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + let _ = lookup; + None + } } impl IdentityProvider for Box

{ @@ -77,6 +88,10 @@ impl IdentityProvider for Box

{ ) -> Result, Report> { (**self).parse_response(response) } + + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + (**self).generate(lookup) + } } /// What a provider can return, used by the coordinator to apply shared policy. @@ -206,6 +221,9 @@ pub struct ProviderRequest { pub headers: http::HeaderMap, /// Request body. pub body: bytes::Bytes, + /// Provider state returned with the response in + /// [`ProviderResponse::context`]; never sent. + pub context: Option, } impl fmt::Debug for ProviderRequest { @@ -226,6 +244,8 @@ pub struct ProviderResponse<'body> { pub headers: &'body http::HeaderMap, /// Response body. pub body: &'body [u8], + /// The [`ProviderRequest::context`] of the request this answers. + pub context: Option<&'body str>, } /// A parsed provider response. @@ -235,6 +255,8 @@ pub enum LookupOutcome<'body> { Found(Vec>), /// The provider knows no ID for this visitor. NoMatch, + /// The visitor opted out with the provider; its stored UIDs are removed. + OptedOut, } /// One entry as returned by a provider, before validation. @@ -265,6 +287,28 @@ pub struct ProviderUid<'body> { pub ext: Option<&'body RawValue>, /// Expiry as the provider states it. pub expiry: ProviderExpiry, + /// Private refresh state kept with the UID in the row and never served. + pub refresh: Option>, +} + +/// Refresh state a provider asks the framework to keep with a UID. +/// +/// The stored UID's source becomes due again at `from`, and the state is +/// handed back through [`LookupInput::current`]. `Debug` hides the state, +/// which may hold a credential. +pub struct ProviderRefresh<'body> { + /// Provider-defined state, such as a refresh token. + pub state: Cow<'body, str>, + /// When the provider should be asked to refresh the UID. + pub from: ProviderExpiry, +} + +impl fmt::Debug for ProviderRefresh<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } } /// Expiry as a provider states it. The coordinator turns it into @@ -587,6 +631,7 @@ mod tests { .expect("should parse"), headers, body: bytes::Bytes::new(), + context: None, }; let printed = format!("{request:?}"); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 5046bf82da..fb00d8ca38 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -1,7 +1,7 @@ //! Plans a navigation's single row change. //! //! Combines browser ingest (fill missing) and validated provider results -//! (replace the provider's sources), prunes expired UIDs and enforces +//! (replace the provider's sources, or remove them on an opt-out), prunes expired UIDs and enforces //! [`MAX_ENTRY_BYTES`]. Pure: no I/O. //! //! A provider-owned source takes a browser UID only as a fallback: when the @@ -68,7 +68,13 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { input .lookups .iter() - .flat_map(|lookup| lookup.entries.iter().map(|(source, _)| source)) + .flat_map(|lookup| { + lookup + .entries + .iter() + .map(|(source, _)| source) + .chain(&lookup.cleared) + }) .collect() } else { Vec::new() @@ -162,6 +168,14 @@ fn apply_lookups( written.push(source.clone()); } } + for source in &lookup.cleared { + if entry.ids.remove(source.as_str()).is_some() { + log::info!( + "EC write planner: removing '{source}' after provider '{}' opt-out", + lookup.provider + ); + } + } } written } @@ -305,6 +319,7 @@ mod tests { SourceDomain::try_from(source).expect("should build source"), vec![KvEid::new(EidFields::default(), uids)], )], + cleared: Vec::new(), }], ) } @@ -534,10 +549,41 @@ mod tests { provider: ProviderId::new("example-provider"), status, entries: Vec::new(), + cleared: Vec::new(), }], ) } + #[test] + fn opt_out_removes_the_provider_sources_and_blocks_browser_fallback() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + }], + ); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should remove the opted-out source without a browser fallback" + ); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index af233cee06..8ea999d82a 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; -use std::num::NonZeroU16; +use std::num::{NonZeroU16, NonZeroU32}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -605,6 +605,12 @@ pub struct IdentityProviderConfig { /// TCF vendor ID that must have vendor consent, when set. #[serde(default)] pub tcf_vendor_id: Option, + /// When the provider's lookup runs on a navigation. + #[serde(default)] + pub collect: CollectTiming, + /// Most lookups per EC per hour, when set; unset means unlimited. + #[serde(default)] + pub rate_limit_per_hour: Option, /// Secret Store key references, resolved at startup. #[serde(default)] pub secrets: ProviderSecrets, @@ -628,6 +634,20 @@ impl IdentityProviderConfig { } } +/// When an identity provider's lookup runs on a navigation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectTiming { + /// Before the response, within `identity_budget_ms`; results reach the + /// navigation's own write and response. + #[default] + Page, + /// After the response has been sent, within + /// `identity_after_response_budget_ms`; results reach the row in one + /// follow-up write and are served from the next page load. + AfterResponse, +} + /// Secret values for one identity provider. /// /// `EdgeZero` secret paths address fixed field names, so V1 supports one @@ -686,6 +706,15 @@ pub struct Ec { )] pub identity_budget_ms: u64, + /// Time budget, in milliseconds, for lookups of providers with + /// `collect = "after_response"`, measured from their dispatch after the + /// response has been sent. + #[serde( + default = "Ec::default_identity_after_response_budget_ms", + deserialize_with = "from_value_or_str" + )] + pub identity_after_response_budget_ms: u64, + /// Server-side identity providers. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub identity_providers: Vec, @@ -725,6 +754,12 @@ impl Ec { 200 } + /// Default after-response identity lookup budget (2 s). + #[must_use] + pub const fn default_identity_after_response_budget_ms() -> u64 { + 2000 + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index d7206687ed..cb6e08118e 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -2,8 +2,9 @@ //! //! Each provider kind is a module implementing //! [`trusted_server_core::ec::provider::ProviderKind`]. [`build`] maps a -//! configured `kind` to its implementation; [`build_coordinator`] builds the -//! coordinator for every enabled provider. +//! configured `kind` to its implementation; [`build_coordinator`] and +//! [`build_after_response_coordinator`] build the coordinators for enabled +//! providers collected before and after the response. //! //! Dependencies point one way: this crate depends on core, and core never //! depends on it. @@ -16,7 +17,9 @@ use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvid use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::settings::{Ec, IdentityProviderConfig, ProviderSecrets, Settings}; +use trusted_server_core::settings::{ + CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, +}; /// Whether `kind` has a registered implementation. /// @@ -53,14 +56,16 @@ pub fn build( .attach(format!("kind `{kind}` is not registered"))) } -/// Builds the coordinator for every enabled provider in `settings`. +/// Builds the coordinator for enabled providers collected before the +/// response (`collect = "page"`, the default). /// -/// Returns `None` when no provider is enabled. +/// Returns `None` when no such provider is enabled. /// /// # Errors /// -/// Returns [`ProviderConfigError`] when a provider cannot be built or fails -/// the coordinator's configuration checks. +/// Returns [`ProviderConfigError`] when a provider cannot be built or the +/// enabled providers, taken together, fail the coordinator's configuration +/// checks. pub fn build_coordinator( settings: &Settings, ) -> Result, Report> { @@ -69,8 +74,8 @@ pub fn build_coordinator( build_coordinator_with(settings, Arc::new(partners)) } -/// Builds the coordinator for every enabled provider in `settings` against an -/// already built partner registry. +/// Builds the page-phase coordinator against an already built partner +/// registry. /// /// Enabled providers with an unregistered kind are skipped; building the /// registry with [`partner_registry`] already warned about them. @@ -83,20 +88,65 @@ pub fn build_coordinator_with( settings: &Settings, partners: Arc, ) -> Result, Report> { + build_phase_coordinator( + settings, + partners, + CollectTiming::Page, + Duration::from_millis(settings.ec.identity_budget_ms), + ) +} + +/// Builds the coordinator for enabled providers collected after the response +/// has been sent (`collect = "after_response"`). +/// +/// Its budget is `identity_after_response_budget_ms`. Returns `None` when no +/// such provider is enabled. +/// +/// # Errors +/// +/// Returns the same errors as [`build_coordinator`]. +pub fn build_after_response_coordinator( + settings: &Settings, +) -> Result, Report> { + let partners = partner_registry(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + build_phase_coordinator( + settings, + Arc::new(partners), + CollectTiming::AfterResponse, + Duration::from_millis(settings.ec.identity_after_response_budget_ms), + ) +} + +fn build_phase_coordinator( + settings: &Settings, + partners: Arc, + timing: CollectTiming, + budget: Duration, +) -> Result, Report> { + // Source ownership across both phases is checked when settings load + // (`config::validate_identity_providers`), so only this phase's providers + // are built and checked here. let registered = settings .ec .identity_providers .iter() - .filter(|config| config.enabled && is_registered_kind(config.kind())) + .filter(|config| { + config.enabled && config.collect == timing && is_registered_kind(config.kind()) + }) .map(|config| { - build(config, &config.secrets) - .map(|provider| RegisteredProvider::new(config.clone(), provider)) + build(config, &config.secrets).map(|provider| { + let registered = RegisteredProvider::new(config.clone(), provider); + match config.rate_limit_per_hour { + Some(per_hour) => registered.with_rate_limit(per_hour), + None => registered, + } + }) }) .collect::, Report>>()?; if registered.is_empty() { return Ok(None); } - let budget = Duration::from_millis(settings.ec.identity_budget_ms); ProviderCoordinator::new(registered, partners, budget).map(Some) } @@ -177,4 +227,37 @@ mod tests { "should leave unregistered provider sources to browser ingest" ); } + + #[test] + fn rejects_an_unknown_collect_timing() { + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\ncollect = \"later\"", + ) + .expect_err("should reject an unknown `collect` value"); + } + + #[test] + fn parses_an_optional_hourly_rate_limit() { + let limited: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 2", + ) + .expect("should parse a rate limit"); + let unset: IdentityProviderConfig = + toml::from_str("id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"") + .expect("should parse without a rate limit"); + + assert_eq!( + limited.rate_limit_per_hour.map(std::num::NonZeroU32::get), + Some(2), + "should keep the hourly limit" + ); + assert!( + unset.rate_limit_per_hour.is_none(), + "should leave the limit unset by default" + ); + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 0", + ) + .expect_err("should reject a zero limit"); + } } From d45b5087543e893c8bf7e5954584b023c9507f7f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:29:02 +0530 Subject: [PATCH 26/58] Never store browser UIDs for provider-owned sources Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 14 +- .../trusted-server-core/src/ec/prebid_eids.rs | 3 +- .../trusted-server-core/src/ec/write_plan.rs | 120 ++++++------------ docs/guide/configuration.md | 7 +- 4 files changed, 48 insertions(+), 96 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 055e4b2950..2e28f92a23 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1679,7 +1679,7 @@ mod tests { } #[test] - fn finalize_stores_browser_fallback_for_provider_owned_source() { + fn finalize_skips_browser_uid_for_provider_owned_source() { // Arrange let settings = create_test_settings(); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1717,17 +1717,9 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("should create the row"); - let record = stored - .ids - .get("sharedid.org") - .expect("should store the browser fallback"); - assert_eq!( - record.uid, "fallback-cookie-id", - "should store the browser uid" - ); assert!( - !record.is_provider_written(), - "should mark the fallback as not provider-written" + !stored.ids.contains_key("sharedid.org"), + "should not store a browser UID for a provider-owned source" ); } diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 29db7122fc..768a431188 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,8 +157,7 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } - // Provider-owned sources stay in the list: the write planner ingests them - // only as a fallback when the provider left the source empty. + // Provider-owned sources stay in the list; the write planner skips them. dedupe_partner_updates(updates) } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index fb00d8ca38..c1b566a54a 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -4,11 +4,9 @@ //! (replace the provider's sources, or remove them on an opt-out), prunes expired UIDs and enforces //! [`MAX_ENTRY_BYTES`]. Pure: no I/O. //! -//! A provider-owned source takes a browser UID only as a fallback: when the -//! row holds no live UID for it and the provider kept nothing for it on this -//! navigation. The fallback is a single-UID record without provider `eids`, -//! so the coordinator still treats the source as due and a later provider -//! result replaces it. +//! Browser ingest skips provider-owned sources entirely (spec §8.4), so a +//! source the provider cleared on an opt-out is never refilled from the +//! browser's `ts-eids` cookie. use super::EcKvSnapshot; use super::coordinator::results::LookupResults; @@ -64,30 +62,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { let mut entry = base.clone(); let lookups_apply = input.lookups.is_for(input.ec_id); - let provider_filled: Vec<&SourceDomain> = if lookups_apply { - input - .lookups - .iter() - .flat_map(|lookup| { - lookup - .entries - .iter() - .map(|(source, _)| source) - .chain(&lookup.cleared) - }) - .collect() - } else { - Vec::new() - }; - apply_browser_ingest( - &mut entry, - input.browser_ingest, - &BrowserIngestRules { - registry: input.registry, - provider_filled: &provider_filled, - now: input.now, - }, - ); + apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); let written = if lookups_apply { apply_lookups(&mut entry, input.lookups, input.now) } else { @@ -112,46 +87,24 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } } -/// What browser ingest needs to decide provider-owned fallbacks. -struct BrowserIngestRules<'a> { - registry: &'a PartnerRegistry, - /// Sources the provider returned entries for on this navigation. - provider_filled: &'a [&'a SourceDomain], - now: UnixSeconds, -} - +/// Fills missing sources from the browser, skipping provider-owned ones. fn apply_browser_ingest( entry: &mut KvEntry, updates: &[PartnerIdUpdate], - rules: &BrowserIngestRules<'_>, + registry: &PartnerRegistry, ) { for update in updates { - if !rules.registry.is_provider_owned(&update.partner_id) { - entry - .ids - .entry(update.partner_id.clone()) - .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + if registry.is_provider_owned(&update.partner_id) { + log::trace!( + "EC write planner: skipping browser UID for provider-owned source '{}'", + update.partner_id + ); continue; } - let provider_filled = rules - .provider_filled - .iter() - .any(|source| source.as_str() == update.partner_id); - let held = entry + entry .ids - .get(&update.partner_id) - .is_some_and(|record| record.has_live_uid(rules.now)); - if provider_filled || held { - continue; - } - log::debug!( - "EC write planner: ingesting browser fallback for provider-owned source '{}'", - update.partner_id - ); - entry.ids.insert( - update.partner_id.clone(), - KvPartnerId::single(update.uid.clone()), - ); + .entry(update.partner_id.clone()) + .or_insert_with(|| KvPartnerId::single(update.uid.clone())); } } @@ -596,7 +549,7 @@ mod tests { } #[test] - fn provider_owned_source_takes_browser_fallback_when_lookup_fails() { + fn browser_ingest_skips_a_provider_owned_source() { // Arrange let registry = registry(&["ids.example.com"]); let snapshot = present(&live_entry()); @@ -604,32 +557,43 @@ mod tests { for status in [ LookupStatus::NoMatch, + LookupStatus::Timeout, LookupStatus::Skipped(SkipReason::NotDue), + LookupStatus::Skipped(SkipReason::OptedOut), ] { let lookups = failed_lookup(status); // Act - let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) - .expect("should plan update"); + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); // Assert - let RowChange::Update { entry, .. } = change else { - panic!("should update existing row") - }; - let record = &entry.ids["ids.example.com"]; - assert_eq!( - record.uid, "browser-id", - "should store the browser fallback" - ); assert!( - !record.is_provider_written(), - "should mark the fallback as not provider-written" + change.is_none(), + "should not store a browser UID for a provider-owned source" ); } } #[test] - fn provider_owned_source_takes_browser_fallback_without_lookups() { + fn opted_out_source_is_not_refilled_from_the_browser_on_the_next_navigation() { + // Arrange: the row after an opt-out holds nothing for the source. + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not restore an opted-out provider UID from the browser cookie" + ); + } + + #[test] + fn new_row_skips_browser_uid_for_a_provider_owned_source() { let registry = registry(&["ids.example.com"]); let snapshot = EcKvSnapshot::Missing { ec_id: EC_ID.to_owned(), @@ -650,9 +614,9 @@ mod tests { let RowChange::Create(entry) = change else { panic!("should create a new row") }; - assert_eq!( - entry.ids["ids.example.com"].uid, "browser-id", - "should store the browser fallback when no provider ran" + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not store a browser UID when creating the row" ); } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 3cc9402a78..e52648eae8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -805,11 +805,8 @@ request, only with TCF vendor consent for that vendor. A provider owns its `sources`: its results replace whatever the row holds for them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs -(`ts-eids`, `sharedId`) for an owned source are stored only as a fallback, when -the row has no live provider-written UID for the source and the provider kept -nothing for it on that navigation. The fallback follows the normal browser -ingest rules, never overwrites an existing value, and does not stop later -lookups: the next successful lookup replaces it. +(`ts-eids`, `sharedId`) are never stored for an owned source, so a source a +provider cleared on an opt-out is not refilled from the browser. | Field | Type | Required | Description | | ----------------- | ------- | -------- | ----------------------------------------------------------------------- | From e171d1d89a5ae3fc991d3b7b469c948f1e578503 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:31:19 +0530 Subject: [PATCH 27/58] Hand refresh state only to the provider that wrote it Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 37 ++++++++++- crates/trusted-server-core/src/ec/kv_types.rs | 9 ++- crates/trusted-server-core/src/ec/provider.rs | 61 ++++++++++++++++--- 3 files changed, 93 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 40de27dc8b..3fd940b030 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -298,7 +298,7 @@ impl ProviderCoordinator

{ .filter(|record| record.is_provider_written()); // A UID whose provider asked for a refresh makes its source due; // the stored refresh state is the lookup's input. - if record.is_some_and(|record| record.refresh_due(now)) { + if record.is_some_and(|record| record.refresh_due_for(®istered.id, now)) { due.push(source.clone()); continue; } @@ -514,7 +514,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ client: lookup.client, publisher_domain: lookup.publisher_domain, due_sources: &due, - current: OwnedEntries::new(entry, ®istered.policy.sources), + current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), hashed_email: None, }; if let Some(eids) = registered.provider.generate(&input) { @@ -1282,11 +1282,15 @@ mod tests { } fn refreshable_record(from: u64) -> KvPartnerId { + refreshable_record_owned_by("example-provider", from) + } + + fn refreshable_record_owned_by(owner: &str, from: u64) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from("p-1").expect("should build uid"), AgentType::try_from(1).expect("should build atype"), None, - Some(ProviderId::new("example-provider")), + Some(ProviderId::new(owner)), Expiry::At(UnixSeconds::new(5000)), ) .with_refresh(KvRefresh { @@ -1346,6 +1350,33 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn another_providers_refresh_state_is_never_handed_over() { + // Arrange: a live UID with due refresh state written by a provider that + // no longer owns the source. + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record_owned_by("former-provider", 900)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh a UID another provider wrote" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not send another provider's refresh state" + ); + } + fn optout_reply() -> Result> { Ok(reply(http::StatusCode::OK, b"optout")) } diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index c6f401f40d..c709f066bb 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -157,13 +157,16 @@ impl KvPartnerId { !self.eids.is_empty() } - /// Whether a UID in the record asks its provider for a refresh by `now`. + /// Whether a UID `owner` wrote asks it for a refresh by `now`. + /// + /// UIDs another provider wrote never make the record refresh-due for + /// `owner`, so their refresh state is never handed to it. #[must_use] - pub fn refresh_due(&self, now: UnixSeconds) -> bool { + pub fn refresh_due_for(&self, owner: &ProviderId, now: UnixSeconds) -> bool { self.eids .iter() .flat_map(KvEid::uids) - .any(|uid| uid.refresh_due(now)) + .any(|uid| uid.owner() == Some(owner) && uid.refresh_due(now)) } /// Whether the record holds a UID that may be served at `now`. diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 28b2970180..89f328e06a 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -22,7 +22,7 @@ use serde_json::value::RawValue; use url::Url; use crate::consent::ConsentContext; -use crate::ec::eid_types::{SourceDomain, UnixSeconds}; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::kv_types::{KvEid, KvEntry}; use crate::ec::registry::PartnerRegistry; use crate::settings::{IdentityProviderConfig, ProviderSecrets}; @@ -170,27 +170,46 @@ pub struct ClientSignals<'a> { } /// This provider's stored entries, borrowed from the row. +/// +/// Only entries whose every UID this provider wrote are visible, so another +/// provider's refresh state never reaches it. #[derive(Debug, Clone, Copy)] pub struct OwnedEntries<'a> { entry: Option<&'a KvEntry>, sources: &'a [SourceDomain], + owner: &'a ProviderId, } impl<'a> OwnedEntries<'a> { - /// Views the entries of `entry` for `sources`. + /// Views the entries of `entry` for `sources` that `owner` wrote. #[must_use] - pub fn new(entry: Option<&'a KvEntry>, sources: &'a [SourceDomain]) -> Self { - Self { entry, sources } + pub fn new( + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], + owner: &'a ProviderId, + ) -> Self { + Self { + entry, + sources, + owner, + } } /// Iterates `(source, entry)` pairs in source order. pub fn iter(&self) -> impl Iterator + 'a { let entry = self.entry; + let owner = self.owner; self.sources.iter().flat_map(move |source| { entry .and_then(|entry| entry.ids.get_key_value(source.as_str())) .into_iter() - .flat_map(|(key, record)| record.eids().iter().map(move |eid| (key.as_str(), eid))) + .flat_map(move |(key, record)| { + record + .eids() + .iter() + .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == Some(owner))) + .map(move |eid| (key.as_str(), eid)) + }) }) } } @@ -705,14 +724,15 @@ mod tests { "consent": { "ok": true, "updated": 1 }, "geo": { "country": "US" }, "ids": { - "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3 }] }] }, - "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3 }] }] } + "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3, "owner": "example-provider" }] }] }, + "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3, "owner": "example-provider" }] }] } } })) .expect("should build entry"); let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); - let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources) + let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources, &owner) .iter() .map(|(source, _)| source) .collect(); @@ -723,4 +743,29 @@ mod tests { "should yield owned sources only" ); } + + #[test] + fn owned_entries_hides_entries_another_provider_wrote() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [ + { "uids": [{ "id": "u-1", "atype": 3, "owner": "former-provider" }] }, + { "uids": [{ "id": "u-2", "atype": 3 }] } + ] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); + + let count = OwnedEntries::new(Some(&entry), &sources, &owner) + .iter() + .count(); + + assert_eq!(count, 0, "should hide entries this provider did not write"); + } } From cbe3a2bc63bff8be8ee50ebffdfc3f7170f80f32 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:33:51 +0530 Subject: [PATCH 28/58] Remember a provider opt-out so the provider is not called again for 30 days Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 53 +++++++++++++++ crates/trusted-server-core/src/ec/kv_types.rs | 38 +++++++++++ .../trusted-server-core/src/ec/write_plan.rs | 64 ++++++++++++++++++- docs/guide/configuration.md | 4 ++ 4 files changed, 156 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 3fd940b030..0705e8d14f 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -499,6 +499,12 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::Consent)); continue; } + // A remembered opt-out keeps the provider from being called until + // it lapses. + if entry.is_some_and(|entry| entry.opted_out(®istered.id, lookup.now)) { + pending.early.push(skip(SkipReason::OptedOut)); + continue; + } let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); if due.is_empty() { pending.early.push(skip(if input_blocked { @@ -1350,6 +1356,53 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn remembered_opt_out_skips_the_provider_until_it_lapses() { + // Arrange + let coordinator = coordinator(""); + let provider = ProviderId::new("example-provider"); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.opt_outs.insert(provider.clone(), 1001); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::OptedOut), + "should skip a provider whose opt-out is remembered" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not call a provider the visitor opted out of" + ); + + // Arrange: the opt-out lapsed. + entry.opt_outs.insert(provider, 1000); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should call the provider again once the opt-out lapses" + ); + } + #[test] fn another_providers_refresh_state_is_never_handed_over() { // Arrange: a live UID with due refresh state written by a provider that diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index c709f066bb..b24e91e6c0 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -41,6 +41,10 @@ pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// consistent validation. pub const MAX_UID_LENGTH: usize = 1024; +/// How long a provider opt-out keeps that provider from being called again, +/// in seconds. Matches the longest lifetime a provider UID may have. +pub const OPT_OUT_MEMORY_SECONDS: u64 = 30 * 24 * 60 * 60; + /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). @@ -69,6 +73,10 @@ pub struct KvEntry { /// Populated by pixel sync, batch sync, and pull sync operations. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub ids: BTreeMap, + /// Provider ID → Unix time until which that provider's opt-out is + /// honoured without calling the provider again. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub opt_outs: BTreeMap, } /// Consent state within a KV entry. @@ -658,7 +666,35 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), + } + } + + /// Whether `provider` opted out and must not be called again by `now`. + #[must_use] + pub fn opted_out(&self, provider: &ProviderId, now: UnixSeconds) -> bool { + self.opt_outs + .get(provider) + .is_some_and(|until| *until > now.get()) + } + + /// Remembers an opt-out by `provider` for [`OPT_OUT_MEMORY_SECONDS`]. + /// + /// An opt-out already remembered keeps its end time, so a provider skipped + /// because of it is asked again once the time passes. + pub fn remember_opt_out(&mut self, provider: &ProviderId, now: UnixSeconds) { + if self.opted_out(provider, now) { + return; } + self.opt_outs.insert( + provider.clone(), + now.saturating_add_secs(OPT_OUT_MEMORY_SECONDS).get(), + ); + } + + /// Forgets opt-outs whose time has passed. + pub fn prune_opt_outs(&mut self, now: UnixSeconds) { + self.opt_outs.retain(|_, until| *until > now.get()); } /// Creates a minimal live entry. @@ -688,6 +724,7 @@ impl KvEntry { device: None, network: None, ids, + opt_outs: BTreeMap::new(), } } @@ -720,6 +757,7 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), } } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index c1b566a54a..0a6e087521 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -9,7 +9,7 @@ //! browser's `ts-eids` cookie. use super::EcKvSnapshot; -use super::coordinator::results::LookupResults; +use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; use super::eid_types::{SourceDomain, UnixSeconds}; use super::kv::PartnerIdUpdate; use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; @@ -77,6 +77,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { for record in entry.ids.values_mut() { record.prune_expired(input.now); } + entry.prune_opt_outs(input.now); entry.ids.retain(|_, record| !record.uid.is_empty()); enforce_size_cap(&mut entry, &written); @@ -121,6 +122,9 @@ fn apply_lookups( written.push(source.clone()); } } + if lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) { + entry.remember_opt_out(&lookup.provider, now); + } for source in &lookup.cleared { if entry.ids.remove(source.as_str()).is_some() { log::info!( @@ -213,7 +217,7 @@ mod tests { use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::KvIdentityGraph; - use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid, OPT_OUT_MEMORY_SECONDS}; use crate::settings::EcPartner; const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; @@ -537,6 +541,61 @@ mod tests { ); } + #[test] + fn opt_out_is_remembered_without_extending_it() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let provider = ProviderId::new("example-provider"); + let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + + // Act: the first opt-out is remembered. + let first = plan_write(&input( + &present(&provider_entry(&["p-1"])), + None, + &ingest, + &lookups, + ®istry, + )) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = first else { + panic!("should update existing row") + }; + let until = entry.opt_outs[&provider]; + assert_eq!( + until, + 2000 + OPT_OUT_MEMORY_SECONDS, + "should remember the opt-out for the memory period" + ); + + // Act: a skip because of the remembered opt-out changes nothing. + let again = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert + assert!(again.is_none(), "should not extend a remembered opt-out"); + } + + #[test] + fn lapsed_opt_out_is_pruned() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!(entry.opt_outs.is_empty(), "should forget a lapsed opt-out"); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); @@ -559,7 +618,6 @@ mod tests { LookupStatus::NoMatch, LookupStatus::Timeout, LookupStatus::Skipped(SkipReason::NotDue), - LookupStatus::Skipped(SkipReason::OptedOut), ] { let lookups = failed_lookup(status); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e52648eae8..5574ef53d7 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -808,6 +808,10 @@ them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs (`ts-eids`, `sharedId`) are never stored for an owned source, so a source a provider cleared on an opt-out is not refilled from the browser. +When a provider reports an opt-out, its sources are cleared and the row +remembers the opt-out for 30 days. The provider is not called for that EC +again until the 30 days pass. + | Field | Type | Required | Description | | ----------------- | ------- | -------- | ----------------------------------------------------------------------- | | `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | From ca915dbb3340d69c5044524fa044ce52db53a2a6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:45:07 +0530 Subject: [PATCH 29/58] Add an empty owned-entries view for providers with nothing stored Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 89f328e06a..302b120b25 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -177,7 +177,7 @@ pub struct ClientSignals<'a> { pub struct OwnedEntries<'a> { entry: Option<&'a KvEntry>, sources: &'a [SourceDomain], - owner: &'a ProviderId, + owner: Option<&'a ProviderId>, } impl<'a> OwnedEntries<'a> { @@ -191,7 +191,17 @@ impl<'a> OwnedEntries<'a> { Self { entry, sources, - owner, + owner: Some(owner), + } + } + + /// A view holding no entries, for a provider with nothing stored. + #[must_use] + pub fn empty() -> Self { + Self { + entry: None, + sources: &[], + owner: None, } } @@ -207,7 +217,7 @@ impl<'a> OwnedEntries<'a> { record .eids() .iter() - .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == Some(owner))) + .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == owner)) .map(move |eid| (key.as_str(), eid)) }) }) @@ -744,6 +754,15 @@ mod tests { ); } + #[test] + fn empty_owned_entries_yields_nothing() { + assert_eq!( + OwnedEntries::empty().iter().count(), + 0, + "should yield no entries" + ); + } + #[test] fn owned_entries_hides_entries_another_provider_wrote() { let entry: KvEntry = serde_json::from_value(json!({ From 97a69f1e828a5d89a1944b83d62b62aa52f4968d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 20:09:49 +0530 Subject: [PATCH 30/58] Run after-response lookups only for an existing consented row An EC minted on the navigation, or one whose row is missing or withdrawn when read after the response, is skipped before any provider is called, so no vendor is sent an EC ID whose row cannot take the result. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 12 ++++- .../src/ec/after_response.rs | 50 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 6a59e836fd..0256c8d9e9 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,7 +13,9 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; -use trusted_server_core::ec::after_response::commit_after_response_lookups; +use trusted_server_core::ec::after_response::{ + after_response_lookups_allowed, commit_after_response_lookups, +}; use trusted_server_core::ec::coordinator::IdentityCoordinator as _; use trusted_server_core::ec::coordinator::LookupContext; use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; @@ -791,6 +793,14 @@ fn run_after_response_lookups( return; }; let snapshot = graph.load_snapshot(&lookups.ec_id); + if !after_response_lookups_allowed( + &snapshot, + &lookups.ec_id, + ec_state.ec_context.ec_generated(), + ) { + log::debug!("After-response lookups skipped: new EC or no live row"); + return; + } let transport = PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index c6ecba1031..37952eaadc 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -35,6 +35,24 @@ pub enum AfterResponseOutcome { Failed, } +/// Whether after-response lookups may be dispatched for `ec_id`. +/// +/// They run only for an EC that existed before this navigation and whose row, +/// read after the response was sent, is live and consented. An EC minted on +/// this navigation is skipped: no partner can know it yet, and its new row +/// may not be readable so soon. Its lookups run from the next navigation. +#[must_use] +pub fn after_response_lookups_allowed( + snapshot: &EcKvSnapshot, + ec_id: &str, + ec_generated: bool, +) -> bool { + !ec_generated + && snapshot + .entry_for(ec_id) + .is_some_and(|entry| entry.consent.ok) +} + /// Applies after-response lookup results to the row of `ec_id`. /// /// Plans an update from `lookups` alone against `snapshot`, the row read @@ -366,4 +384,36 @@ mod tests { "should make no write without results" ); } + + #[test] + fn lookups_run_only_for_an_existing_consented_row() { + // Arrange + let graph = KvIdentityGraph::in_memory("after_response_allowed"); + let missing = graph.load_snapshot(EC_ID); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let present = graph.load_snapshot(EC_ID); + let withdrawn_graph = KvIdentityGraph::in_memory("after_response_withdrawn"); + withdrawn_graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let withdrawn = withdrawn_graph.load_snapshot(EC_ID); + + // Act + Assert + assert!( + after_response_lookups_allowed(&present, EC_ID, false), + "should run for a returning EC with a live row" + ); + assert!( + !after_response_lookups_allowed(&present, EC_ID, true), + "should not run for an EC minted on this navigation" + ); + assert!( + !after_response_lookups_allowed(&missing, EC_ID, false), + "should not run when the row does not exist" + ); + assert!( + !after_response_lookups_allowed(&withdrawn, EC_ID, false), + "should not run for a withdrawn row" + ); + } } From a6adc2610367b909827fff1b30d1b0b5f7d9088f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 21:22:13 +0530 Subject: [PATCH 31/58] Re-read the row just before the after-response write The follow-up write was guarded by the generation read before the lookups were dispatched, up to the full lookup budget earlier, so any write from another request in that window dropped the results. The row is now read just before the write, so only the gap between that read and the write can conflict. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 10 +-- .../src/ec/after_response.rs | 65 ++++++++++++++----- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 0256c8d9e9..1d4c9057cb 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -826,14 +826,8 @@ fn run_after_response_lookups( return; } let results = futures::executor::block_on(coordinator.collect(&transport, pending)); - let outcome = commit_after_response_lookups( - &graph, - permit, - partner_registry, - &lookups.ec_id, - &snapshot, - &results, - ); + let outcome = + commit_after_response_lookups(&graph, permit, partner_registry, &lookups.ec_id, &results); log::info!("After-response lookups: write outcome={outcome}"); } diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index 37952eaadc..f7e9206ce3 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -55,27 +55,28 @@ pub fn after_response_lookups_allowed( /// Applies after-response lookup results to the row of `ec_id`. /// -/// Plans an update from `lookups` alone against `snapshot`, the row read -/// after the response was sent, and writes it once, guarded by the snapshot's -/// generation. The `permit` is consumed whether or not a write is -/// made. +/// Re-reads the row just before writing, plans an update from `lookups` +/// alone against it, and writes it once, guarded by that read's generation. +/// Reading here rather than before dispatch keeps the window in which another +/// write can conflict to the gap between this read and the write. The +/// `permit` is consumed whether or not a write is made. /// /// # Performance /// -/// At most one conditional KV write; the row is read by the caller. -/// Runs after the response has been sent, so it adds no page latency. +/// One KV read and at most one conditional KV write. Runs after the response +/// has been sent, so it adds no page latency. #[must_use] pub fn commit_after_response_lookups( graph: &KvIdentityGraph, permit: AfterResponseWrite, registry: &PartnerRegistry, ec_id: &str, - snapshot: &EcKvSnapshot, lookups: &LookupResults, ) -> AfterResponseOutcome { if lookups.is_empty() { return AfterResponseOutcome::Unchanged; } + let snapshot = &graph.load_snapshot(ec_id); if !matches!(snapshot, EcKvSnapshot::Present { .. }) { log::info!( "After-response lookups: no row for '{}', skipping write", @@ -205,7 +206,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -226,6 +226,48 @@ mod tests { ); } + #[test] + fn writes_against_the_row_as_read_just_before_the_write() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-reread"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read_before_dispatch = graph.load_snapshot(EC_ID); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + assert_ne!( + graph.load_snapshot(EC_ID), + read_before_dispatch, + "should change the row after the dispatch read" + ); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should not conflict with a write made while the lookups ran" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + } + #[test] fn replaces_a_browser_fallback_for_an_owned_source() { let graph = KvIdentityGraph::in_memory("after-response"); @@ -241,7 +283,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -266,7 +307,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); assert_eq!(first, AfterResponseOutcome::Written, "should write first"); @@ -276,7 +316,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -296,7 +335,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -323,7 +361,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(EC_ID, "p-1"), ); @@ -349,7 +386,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &found(other, "p-1"), ); @@ -374,7 +410,6 @@ mod tests { AfterResponseWrite::for_test(), ®istry(), EC_ID, - &graph.load_snapshot(EC_ID), &LookupResults::new(EC_ID, Vec::new()), ); From eb2c868a3aaa86249d4a775173e736553d522c52 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 21:22:13 +0530 Subject: [PATCH 32/58] Drop provider refresh state that is already due A refresh time at or before now made the source due on every navigation until the UID expired. Such state is now dropped like other unusable refresh state; the UID itself is kept. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/validate.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index a189fcd25a..367e049e55 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -141,10 +141,11 @@ fn stored_uid( return Ok(stored); }; // Unusable refresh state is dropped, never the UID: a refresh with no - // stated time, or with empty or oversized state, is simply not stored. + // stated time, with empty or oversized state, or due already (which + // would make the source due on every navigation) is simply not stored. let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; match resolve_expiry(refresh.from, policy.now) { - Some(from) if usable => Ok(stored.with_refresh(KvRefresh { + Some(from) if usable && from > policy.now => Ok(stored.with_refresh(KvRefresh { state: refresh.state.into_owned(), from, })), @@ -275,6 +276,21 @@ mod tests { (String::new(), in_a_minute, None), ("x".repeat(MAX_REFRESH_STATE_BYTES + 1), in_a_minute, None), ("state".to_owned(), ProviderExpiry::Unspecified, None), + ( + "state".to_owned(), + ProviderExpiry::After(Duration::ZERO), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(1000)), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(999)), + None, + ), ]; for (state, from, expected) in cases { From aade0ffda349e932fa2f5b18df6c4f312e9ad7bb Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:53:24 +0530 Subject: [PATCH 33/58] Document collect timing, the after-response budget and rate limit granularity Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/settings.rs | 3 ++ docs/guide/configuration.md | 55 +++++++++++++--------- trusted-server.example.toml | 9 ++++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 8ea999d82a..148d774624 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -609,6 +609,9 @@ pub struct IdentityProviderConfig { #[serde(default)] pub collect: CollectTiming, /// Most lookups per EC per hour, when set; unset means unlimited. + /// + /// The Fastly limiter enforces this as a per-minute budget of at least + /// one, so values below 60 behave like 60 per hour. #[serde(default)] pub rate_limit_per_hour: Option, /// Secret Store key references, resolved at startup. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 5574ef53d7..50fa54276f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -749,16 +749,17 @@ Legacy consent-store records are not read or migrated into `ec.ec_store`. Their `passphrase` is a key name in `trusted_server_secrets`; the resolved value must be at least 32 bytes. Keep it stable to preserve EC identifier continuity. -| Field | Type | Required | Description | -| ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | -| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | -| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | -| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | -| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | -| `partners` | Array | No | Static partner registry entries | -| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | -| `identity_providers` | Array | No | Server-side identity providers (see below) | +| Field | Type | Required | Description | +| ----------------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | +| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | +| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | +| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | +| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | +| `partners` | Array | No | Static partner registry entries | +| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | +| `identity_after_response_budget_ms` | Integer | No | Time budget for `collect = "after_response"` lookups (default 2000) | +| `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. @@ -812,17 +813,19 @@ When a provider reports an opt-out, its sources are cleared and the row remembers the opt-out for 30 days. The provider is not called for that EC again until the 30 days pass. -| Field | Type | Required | Description | -| ----------------- | ------- | -------- | ----------------------------------------------------------------------- | -| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | -| `kind` | String | Yes | Provider implementation built into this release | -| `enabled` | Boolean | No | Whether the provider runs (default `true`) | -| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | -| `allowed_domains` | Array | Yes | Hosts the provider may call | -| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | -| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | -| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | -| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | +| Field | Type | Required | Description | +| --------------------- | ------- | -------- | ----------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation built into this release | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | +| `allowed_domains` | Array | Yes | Hosts the provider may call | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC per hour; unset means unlimited. See the note below | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | **Example**: @@ -841,6 +844,16 @@ sources = ["ids.example.com"] token = "example_provider_token" ``` +`collect = "after_response"` lookups run after the response is sent, within +`identity_after_response_budget_ms`, and only for an EC whose row already +existed before the navigation. Their results reach the row in a follow-up write. + +::: warning Rate limit granularity +On Fastly, `rate_limit_per_hour` is enforced as a per-minute budget rounded up +to at least one lookup per minute, so any value below 60 behaves like 60 per +hour. Values of 60 and above are enforced as `ceil(value / 60)` per minute. +::: + **Validation**: configuration validation fails when an enabled provider's `endpoint` is not HTTPS or its host is not in `allowed_domains`, when a source has no `[[ec.partners]]` entry, or when two enabled providers claim the same diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e0cec612cf..6418f21eaf 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -110,6 +110,9 @@ pull_sync_concurrency = 3 # Optional server-side identity providers, looked up on publisher navigations # before the origin fetch. Every enabled provider shares one time budget: # identity_budget_ms = 200 # default 200 +# Budget for providers with collect = "after_response", which run after the +# response is sent and write their results in a follow-up write: +# identity_after_response_budget_ms = 2000 # # [[ec.identity_providers]] # id = "example-provider" @@ -124,6 +127,12 @@ pull_sync_concurrency = 3 # sources = ["ids.example.com"] # Optional: require TCF vendor consent for this provider. # tcf_vendor_id = 9999 +# "page" (default) collects before the response; "after_response" collects +# after it is sent, only for an EC whose row already exists. +# collect = "page" +# Optional: most lookups per EC per hour. On Fastly this is enforced per +# minute with a minimum of one, so values below 60 behave like 60. +# rate_limit_per_hour = 60 # Secret Store key name, resolved like other secret references. # [ec.identity_providers.secrets] # token = "example_provider_token" From 49ec524ef0fa4791a23ad1d949e0ac82e44fe5f1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:54:44 +0530 Subject: [PATCH 34/58] Generate IDs only for due sources without a live UID Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 0705e8d14f..11f4c41855 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -281,6 +281,13 @@ impl ProviderCoordinator

{ self.providers.is_empty() } + /// Whether the row holds a live provider-written UID for `source`. + fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { + entry + .and_then(|entry| entry.ids.get(source)) + .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)) + } + /// Claimed sources lacking a live provider-written UID (a browser /// fallback does not count), minus sources needing input V1 /// never has; the flag reports whether any source was removed for input. @@ -523,12 +530,20 @@ impl IdentityCoordinator for ProviderCoordinator

{ current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), hashed_email: None, }; - if let Some(eids) = registered.provider.generate(&input) { - pending.generated.push(Tagged { - provider, - value: self.validated(provider, eids, lookup.now), + if let Some(mut eids) = registered.provider.generate(&input) { + // Only a due source without a live UID may take a generated + // ID: a refresh-due source keeps its UID and goes to lookup. + eids.retain(|eid| { + due.iter().any(|source| source.as_str() == eid.source) + && !Self::holds_live_uid(entry, &eid.source, lookup.now) }); - continue; + if !eids.is_empty() { + pending.generated.push(Tagged { + provider, + value: self.validated(provider, eids, lookup.now), + }); + continue; + } } if Self::rate_limited(registered, lookup) { pending.early.push(skip(SkipReason::RateLimited)); @@ -1526,6 +1541,35 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send no lookup"); } + #[test] + fn a_refresh_due_live_source_is_looked_up_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup + .entries + .iter() + .flat_map(|(_, eids)| eids) + .flat_map(KvEid::uids) + .all(|uid| uid.id().as_str() != "generated-1"), + "should not replace the live UID with a generated one" + ); + assert_eq!( + transport.sent.borrow().len(), + 1, + "should send the refresh lookup instead" + ); + } + #[test] fn a_held_source_is_not_regenerated() { let coordinator = generating_coordinator(); From dbb9fa60465a8771de885b5b08fca50f6934da6f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:56:00 +0530 Subject: [PATCH 35/58] Reuse the partner registry and core clock for after-response lookups Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-fastly/src/main.rs | 10 +++++----- crates/trusted-server-identity-providers/src/lib.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 1d4c9057cb..134889694b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -19,6 +19,7 @@ use trusted_server_core::ec::after_response::{ use trusted_server_core::ec::coordinator::IdentityCoordinator as _; use trusted_server_core::ec::coordinator::LookupContext; use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::current_timestamp; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; @@ -764,7 +765,7 @@ fn run_edgezero_pull_sync_after_send( /// for. fn run_after_response_lookups( settings: &Settings, - partner_registry: &PartnerRegistry, + partner_registry: &Arc, ec_state: &mut EcFinalizeState, lookups: Option, ) { @@ -781,7 +782,8 @@ fn run_after_response_lookups( log::debug!("After-response lookups skipped: EC changed during the request"); return; } - let coordinator = match build_after_response_coordinator(settings) { + let coordinator = match build_after_response_coordinator(settings, Arc::clone(partner_registry)) + { Ok(Some(coordinator)) => coordinator, Ok(None) => return, Err(report) => { @@ -804,9 +806,7 @@ fn run_after_response_lookups( let transport = PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); + let now = current_timestamp(); let pending = futures::executor::block_on(coordinator.dispatch( &transport, &LookupContext { diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index cb6e08118e..17f083be0f 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -97,22 +97,22 @@ pub fn build_coordinator_with( } /// Builds the coordinator for enabled providers collected after the response -/// has been sent (`collect = "after_response"`). +/// has been sent (`collect = "after_response"`), against an already built +/// partner registry. /// /// Its budget is `identity_after_response_budget_ms`. Returns `None` when no /// such provider is enabled. /// /// # Errors /// -/// Returns the same errors as [`build_coordinator`]. +/// Returns the same errors as [`build_coordinator_with`]. pub fn build_after_response_coordinator( settings: &Settings, + partners: Arc, ) -> Result, Report> { - let partners = partner_registry(&settings.ec) - .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; build_phase_coordinator( settings, - Arc::new(partners), + partners, CollectTiming::AfterResponse, Duration::from_millis(settings.ec.identity_after_response_budget_ms), ) From 1826524b7b9c650f7bd3ad42a118315675a4e7f4 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:57:05 +0530 Subject: [PATCH 36/58] Hide the body and context from provider response debug output Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 302b120b25..e0e50a9841 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -265,7 +265,10 @@ impl fmt::Debug for ProviderRequest { } /// A provider response, borrowed from the transport. -#[derive(Debug, Clone, Copy)] +/// +/// Its `Debug` output omits the body, headers and context, which can hold +/// identifiers or a reply-decryption key. +#[derive(Clone, Copy)] pub struct ProviderResponse<'body> { /// Response status. pub status: http::StatusCode, @@ -277,6 +280,16 @@ pub struct ProviderResponse<'body> { pub context: Option<&'body str>, } +impl fmt::Debug for ProviderResponse<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderResponse") + .field("status", &self.status) + .field("body_len", &self.body.len()) + .field("has_context", &self.context.is_some()) + .finish_non_exhaustive() + } +} + /// A parsed provider response. #[derive(Debug)] pub enum LookupOutcome<'body> { @@ -672,6 +685,29 @@ mod tests { assert!(printed.contains("ids.example.com"), "should print host"); } + #[test] + fn response_debug_hides_body_and_context() { + let headers = http::HeaderMap::new(); + let response = ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body: b"example-uid", + context: Some("example-key"), + }; + + let printed = format!("{response:?}"); + + assert!( + !printed.contains("example-uid"), + "should not print the body" + ); + assert!( + !printed.contains("example-key"), + "should not print the context" + ); + assert!(printed.contains("200"), "should print the status"); + } + #[test] fn validate_enabled_configs_accepts_valid_config() { validate_enabled_configs([&config(&json!({}))], &partners()) From bc65fb9b1c4a83e6fd2136a519cd81de2f4e17c7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:58:42 +0530 Subject: [PATCH 37/58] Plan no row write when pruning is the only change Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/write_plan.rs | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 0a6e087521..3b543e2a56 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -74,6 +74,8 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } Vec::new() }; + // Pruning alone never triggers a write; it rides along with a real change. + let changed = entry != base; for record in entry.ids.values_mut() { record.prune_expired(input.now); } @@ -83,7 +85,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { match expected { None => Some(RowChange::Create(entry)), - Some(_) if entry == base => None, + Some(_) if !changed || entry == base => None, Some(expected) => Some(RowChange::Update { entry, expected }), } } @@ -578,7 +580,7 @@ mod tests { } #[test] - fn lapsed_opt_out_is_pruned() { + fn lapsed_opt_out_alone_plans_no_write() { let registry = registry(&["ids.example.com"]); let ingest: [PartnerIdUpdate; 0] = []; let mut entry = live_entry(); @@ -587,6 +589,21 @@ mod tests { .insert(ProviderId::new("example-provider"), 1000); let lookups = LookupResults::default(); + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!(change.is_none(), "should not write only to prune"); + } + + #[test] + fn lapsed_opt_out_is_pruned_with_a_real_change() { + let registry = registry(&["ids.example.com"]); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) .expect("should plan update"); @@ -596,6 +613,44 @@ mod tests { assert!(entry.opt_outs.is_empty(), "should forget a lapsed opt-out"); } + #[test] + fn not_due_skip_with_an_expired_uid_plans_no_write() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!( + change.is_none(), + "should not write only to prune an expired UID" + ); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); From b334e75e4ce756d75f3ef2f594420d7c8a97e7bc Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:00:44 +0530 Subject: [PATCH 38/58] Test after-response write conflicts and a row withdrawn before the write Signed-off-by: dhruv8sh --- .../src/ec/after_response.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index f7e9206ce3..59abf86484 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -128,7 +128,9 @@ mod tests { use super::*; use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::TombstoneOutcome; use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use crate::ec::page_load::{Generation, KvWriteError}; use crate::settings::EcPartner; const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; @@ -268,6 +270,72 @@ mod tests { ); } + #[test] + fn follow_up_write_conflicts_when_the_row_changed_after_its_read() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-conflict"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + let report = result.expect_err("should refuse a stale generation"); + assert!( + matches!(report.current_context(), KvWriteError::Conflict), + "should report a conflict" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + } + + #[test] + fn follow_up_write_never_revives_a_row_withdrawn_after_its_read() { + // Arrange: scenario 23, the row is tombstoned between read and CAS. + let graph = KvIdentityGraph::in_memory("after-response-withdrawn"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + assert_eq!( + graph + .write_withdrawal_tombstone(EC_ID, drop) + .expect("should withdraw the row"), + TombstoneOutcome::Written, + "should tombstone the row" + ); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + assert!(result.is_err(), "should refuse to overwrite the tombstone"); + assert!( + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .is_none_or(|entry| !entry.consent.ok), + "should leave the row withdrawn" + ); + } + #[test] fn replaces_a_browser_fallback_for_an_owned_source() { let graph = KvIdentityGraph::in_memory("after-response"); From d83509cfc7088dffd75ebb8f1bc4dd4aeb5366b8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 10 Nov 2026 13:21:27 +0530 Subject: [PATCH 39/58] Address review of server-side identity provider collection Restore withdrawal tombstones on every finalized request, keep browser, batch and pull ingest at 512-byte UIDs, withhold a freshly minted EC from bidders and providers, and fail startup on an unknown or unsupported provider. Also gate stored provider IDs on TCF vendor consent in the bidstream and /identify, report full rows from batch sync, validate provider ids, sources and budgets, always verify provider certificates, and update the operator docs and CHANGELOG. Signed-off-by: dhruv8sh --- AGENTS.md | 1 + CHANGELOG.md | 12 +- Cargo.lock | 1 - crates/trusted-server-adapter-axum/src/app.rs | 1 + .../src/app.rs | 1 + .../trusted-server-adapter-fastly/src/app.rs | 269 ++++++----- .../trusted-server-adapter-fastly/src/main.rs | 23 +- crates/trusted-server-adapter-spin/src/app.rs | 1 + .../src/auction/endpoints.rs | 33 +- crates/trusted-server-core/src/config.rs | 6 +- crates/trusted-server-core/src/ec/admin.rs | 17 +- .../src/ec/after_response.rs | 2 +- .../trusted-server-core/src/ec/batch_sync.rs | 28 ++ .../src/ec/coordinator/mod.rs | 141 ++++-- .../src/ec/coordinator/results.rs | 6 +- .../src/ec/coordinator/transport.rs | 18 +- .../src/ec/coordinator/validate.rs | 2 +- .../trusted-server-core/src/ec/eid_types.rs | 34 +- crates/trusted-server-core/src/ec/eids.rs | 55 ++- crates/trusted-server-core/src/ec/finalize.rs | 351 ++++++++------ crates/trusted-server-core/src/ec/identify.rs | 58 ++- crates/trusted-server-core/src/ec/kv.rs | 455 ++++++++++++++---- .../trusted-server-core/src/ec/kv_backend.rs | 62 +++ crates/trusted-server-core/src/ec/kv_types.rs | 58 ++- crates/trusted-server-core/src/ec/mod.rs | 8 +- .../trusted-server-core/src/ec/page_load.rs | 27 -- crates/trusted-server-core/src/ec/provider.rs | 116 ++++- .../trusted-server-core/src/ec/pull_sync.rs | 1 + crates/trusted-server-core/src/ec/registry.rs | 57 ++- .../trusted-server-core/src/ec/write_plan.rs | 66 ++- crates/trusted-server-core/src/publisher.rs | 155 +++++- crates/trusted-server-core/src/settings.rs | 82 +++- .../trusted-server-core/src/test_support.rs | 29 ++ .../Cargo.toml | 1 - .../src/lib.rs | 34 +- docs/guide/api-reference.md | 6 +- docs/guide/architecture.md | 3 +- docs/guide/auction-orchestration.md | 6 +- docs/guide/configuration.md | 65 ++- docs/guide/edge-cookies.md | 10 +- docs/guide/integrations/prebid.md | 44 +- trusted-server.example.toml | 30 +- 42 files changed, 1683 insertions(+), 692 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d90eda1f2d..d3aafb14a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ crates/ trusted-server-adapter-cloudflare/ # Cloudflare Workers entry point (wasm32-unknown-unknown binary) trusted-server-adapter-spin/ # Fermyon Spin entry point (wasm32-wasip1 component) trusted-server-cli/ # Host-target `ts` operator CLI + trusted-server-identity-providers/ # Server-side identity provider kinds (used by the Fastly adapter) trusted-server-js/ # TypeScript/JS build — per-integration IIFE bundles lib/ # TS source, Vitest tests, esbuild pipeline ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d5d0b9ea8..c1dc4349f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 measured from dispatch, including delayed headers and trickling bodies on all adapters. Complete available responses survive a slow publisher origin; unfinished providers time out and mediation is skipped after budget exhaustion. - Provider bids that are already complete at the final nonblocking snapshot remain + Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. + On Fastly the final snapshot may wait up to 1 ms per partially received body so + the host can surface bytes that already arrived, and keeps a response whose + headers arrived by the cutoff. +- Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, pull sync skips them, and browser IDs never fill them. Set `tcf_vendor_id` to gate both lookups and the provider's stored IDs in bidstream and `/identify` on TCF vendor consent. See [Identity providers](docs/guide/configuration.md#identity-providers). +- Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its size limit, and continues with the remaining groups. +- **Breaking:** An EC is now minted, and the EC row written, only on a top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, integration proxies and `/auction` no longer mint one, and prefetches and prerenders never mint or write. Each navigation makes at most one conditional page-load write; a consent withdrawal's tombstone is still written on any request that carries the EC cookie. +- **Breaking:** Browser EIDs from the `/auction` request body are no longer stored, and `/auction` and `/_ts/page-bids` no longer write browser EIDs from `ts-eids` or `sharedId`. Those cookies are ingested on the next top-level navigation. +- **Breaking:** An EC minted on the current navigation is no longer sent as `user.id` to bidders, and identity providers are not called for it, until a later navigation finds its row. +- **Breaking:** Axum, Cloudflare and Spin refuse to start when an enabled identity provider is configured, and Fastly refuses to start when an enabled provider names a `kind` this build does not include. +- **Breaking (rollback):** Stored provider UIDs may be up to 1024 bytes (browser, batch and pull ingest stay at 512). A binary from before this release cannot read an EC row holding a longer UID, so roll back only after such rows expire or are rewritten. A config blob that sets `ec.identity_providers` or a non-default identity budget cannot be loaded by an older binary; restore the defaults and re-push before rolling back. - Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. - Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/Cargo.lock b/Cargo.lock index 116d45a14c..d9cf35f9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6743,7 +6743,6 @@ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ "error-stack", - "log", "toml", "trusted-server-core", ] diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index be6589a103..8e3143bdaa 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -91,6 +91,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Axum")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index b0c9599e34..3f96bc9217 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -204,6 +204,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Cloudflare")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 5364bfc182..82e9aee082 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -88,7 +88,7 @@ use std::cell::RefCell; use std::net::IpAddr; -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use edgezero_adapter_fastly::context::FastlyRequestContext; @@ -99,7 +99,7 @@ use edgezero_core::http::{ HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, }; use edgezero_core::router::RouterService; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -148,11 +148,13 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::settings::{CollectTiming, ProxyAssetRoute, Settings}; +use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use trusted_server_identity_providers::{build_coordinator_with, partner_registry}; +use trusted_server_identity_providers::{ + build_after_response_coordinator, build_coordinator_with, partner_registry, +}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -201,26 +203,18 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, - /// Partner registry built on first use and reused for the state's lifetime. - pub(crate) partner_registry: OnceLock>, + /// Partner registry, built with the state. + pub(crate) partner_registry: Arc, + /// Coordinator for providers collected before the response, if any. + pub(crate) page_identity: Option>, + /// Coordinator for providers collected after the response, if any. + pub(crate) after_response_identity: Option>, } impl AppState { - /// Returns the partner registry, building it on first use. - /// - /// A failed build is not cached, so every caller sees its error. - /// - /// # Errors - /// - /// Returns the same errors as [`partner_registry`]. - pub(crate) fn partner_registry( - &self, - ) -> Result, Report> { - if let Some(registry) = self.partner_registry.get() { - return Ok(Arc::clone(registry)); - } - let registry = Arc::new(partner_registry(&self.settings.ec)?); - Ok(Arc::clone(self.partner_registry.get_or_init(|| registry))) + /// Returns the partner registry. + pub(crate) fn partner_registry(&self) -> Arc { + Arc::clone(&self.partner_registry) } } @@ -228,8 +222,9 @@ impl AppState { /// /// # Errors /// -/// Returns an error when settings, the auction orchestrator, or the integration -/// registry fail to initialise. +/// Returns an error when settings, the auction orchestrator, the integration +/// registry, the partner registry or an enabled identity provider fail to +/// initialise. pub(crate) fn build_state( stores: &RuntimeStoreConfig, ) -> Result, Report> { @@ -261,13 +256,30 @@ pub(crate) fn build_state_from_settings( let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + // A provider that cannot run must stop startup: skipping it would quietly + // hand its sources back to browser ingest. + let partner_registry = Arc::new(partner_registry(&settings.ec)?); + let page_identity = build_coordinator_with(&settings, Arc::clone(&partner_registry)) + .change_context(TrustedServerError::Configuration { + message: "identity provider configuration is invalid".to_owned(), + })? + .map(Arc::new); + let after_response_identity = + build_after_response_coordinator(&settings, Arc::clone(&partner_registry)) + .change_context(TrustedServerError::Configuration { + message: "identity provider configuration is invalid".to_owned(), + })? + .map(Arc::new); + Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, - partner_registry: OnceLock::new(), + partner_registry, + page_identity, + after_response_identity, })) } @@ -369,7 +381,7 @@ pub(crate) struct EcFinalizeState { /// Identity lookups dispatched for this navigation, collected before finalize. pub(crate) struct IdentityLookups { - pub(crate) coordinator: ProviderCoordinator, + pub(crate) coordinator: Arc, pub(crate) pending: PendingLookups, } @@ -377,8 +389,8 @@ pub(crate) struct IdentityLookups { /// has been sent. /// /// The router owns the request, so the request signals the providers read are -/// copied here. The coordinator is built and the lookups dispatched only after -/// the response, so neither delays the page. +/// copied here. The lookups are dispatched only after the response, so they +/// never delay the page. pub(crate) struct AfterResponseLookups { pub(crate) ec_id: String, pub(crate) client_ip: Option, @@ -442,15 +454,7 @@ fn prepare_after_response_lookups( req: &Request, ec: &EcRequestState, ) -> Option { - let has_after_response_provider = state - .settings - .ec - .identity_providers - .iter() - .any(|config| config.enabled && config.collect == CollectTiming::AfterResponse); - if !has_after_response_provider { - return None; - } + state.after_response_identity.as_ref()?; ec.kv_graph.as_ref()?; let ec_id = ec.ec_context.ec_value()?.to_owned(); let signals = ProviderRequestSignals::read(req, ec); @@ -463,46 +467,27 @@ fn prepare_after_response_lookups( /// Starts identity-provider lookups for a permitted publisher navigation. /// -/// Returns `None` when no provider is enabled, the request has no EC, or the -/// identity graph is unavailable. A provider configuration that cannot be -/// built is logged and disables lookups for this request. +/// Returns `None` when no provider is enabled, the request has no EC or one +/// minted on this request, or the identity graph is unavailable. async fn dispatch_identity_lookups( state: &AppState, services: &RuntimeServices, req: &Request, ec: &mut EcRequestState, ) -> Option { - if !state - .settings - .ec - .identity_providers - .iter() - .any(|config| config.enabled) - { + let coordinator = Arc::clone(state.page_identity.as_ref()?); + let graph = ec.kv_graph.as_ref()?; + // An EC minted on this request has no row yet and may never be issued, + // so it is not sent to vendors; a later navigation looks it up. + if ec.ec_context.ec_generated() { return None; } - let partners = match state.partner_registry() { - Ok(partners) => partners, - Err(report) => { - log::error!("Identity providers disabled for this request: {report:?}"); - return None; - } - }; - let coordinator = match build_coordinator_with(&state.settings, partners) { - Ok(Some(coordinator)) => coordinator, - Ok(None) => return None, - Err(report) => { - log::error!("Identity providers disabled for this request: {report:?}"); - return None; - } - }; - let graph = ec.kv_graph.as_ref()?; let ec_id = ec.ec_context.ec_value()?.to_owned(); if matches!(ec.ec_context.kv_snapshot(), EcKvSnapshot::NotRead) { let snapshot = graph.load_snapshot(&ec_id); ec.ec_context.set_kv_snapshot(snapshot); } - let transport = PlatformLookupTransport::new(services, state.settings.proxy.certificate_check); + let transport = PlatformLookupTransport::new(services); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); let signals = ProviderRequestSignals::read(req, ec); let now = trusted_server_core::ec::current_timestamp(); @@ -774,20 +759,19 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = state - .partner_registry() - .and_then(|registry| match handler { - NamedRouteHandler::AdminEcLookup => { - // Deliberately do not use an EC request-state graph: that - // copy is bot-gated, while operators use curl for this - // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); - handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) - } - NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), - _ => unreachable!("admin diagnostics should use early dispatch"), - }) - .unwrap_or_else(|error| http_error(&error)); + let registry = state.partner_registry(); + let response = match handler { + NamedRouteHandler::AdminEcLookup => { + // Deliberately do not use an EC request-state graph: that + // copy is bot-gated, while operators use curl for this + // authenticated diagnostic. + let kv = crate::maybe_identity_graph(&state.settings); + handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) + } + NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), + _ => unreachable!("admin diagnostics should use early dispatch"), + } + .unwrap_or_else(|error| http_error(&error)); return Ok(response); } @@ -856,7 +840,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = state.partner_registry()?; + let partner_registry = state.partner_registry(); handle_identify( &state.settings, &kv, @@ -870,7 +854,7 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = state.partner_registry()?; + let partner_registry = state.partner_registry(); let registry_ref = if partner_registry.is_empty() { None } else { @@ -894,7 +878,7 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = state.partner_registry()?; + let partner_registry = state.partner_registry(); let registry_ref = if partner_registry.is_empty() { None } else { @@ -941,7 +925,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = state.partner_registry()?; + let partner_registry = state.partner_registry(); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -1088,44 +1072,40 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match state.partner_registry() { - Ok(partner_registry) => { - let auction = AuctionDispatch { - integration_registry: &state.registry, - orchestrator: &state.orchestrator, - slots, - registry: Some(&*partner_registry), - }; - match handle_publisher_request( - &state.settings, - services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, - EdgeCacheHeader::SurrogateControl, + let partner_registry = state.partner_registry(); + let auction = AuctionDispatch { + integration_registry: &state.registry, + orchestrator: &state.orchestrator, + slots, + registry: Some(&*partner_registry), + }; + match handle_publisher_request( + &state.settings, + services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, + EdgeCacheHeader::SurrogateControl, + ) + .await + { + Ok(pub_response) => { + // Origin start succeeded on the sole publisher-page + // path. Authorize orphan recovery only for real-browser + // document navigations so named routes, integration + // proxies, and filter short circuits cannot rotate an + // identity. + ec.ec_context.set_recovery_eligible(is_publisher_navigation); + publisher_response_into_streaming_response( + pub_response, + &method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + services.clone(), ) .await - { - Ok(pub_response) => { - // Origin start succeeded on the sole publisher-page - // path. Authorize orphan recovery only for real-browser - // document navigations so named routes, integration - // proxies, and filter short circuits cannot rotate an - // identity. - ec.ec_context.set_recovery_eligible(is_publisher_navigation); - publisher_response_into_streaming_response( - pub_response, - &method, - Arc::clone(&state.settings), - state.registry.as_ref(), - Arc::clone(&state.orchestrator), - services.clone(), - ) - .await - } - Err(e) => Err(e), - } } Err(e) => Err(e), } @@ -1852,7 +1832,9 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, - partner_registry: std::sync::OnceLock::new(), + partner_registry: Arc::new(PartnerRegistry::empty()), + page_identity: None, + after_response_identity: None, }); TrustedServerApp::routes_for_state(&state) } @@ -2858,15 +2840,27 @@ mod tests { } #[test] - fn partner_registry_is_built_once_per_state() { - let state = build_state_from_settings(test_settings()).expect("should build test state"); + fn enabled_provider_with_unregistered_kind_fails_startup() { + let mut settings = test_settings(); + settings.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "Example IDs", + "source_domain": "ids.example.com" + })) + .expect("should parse partner"), + ); + settings.ec.identity_providers.push( + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider"), + ); - let first = state.partner_registry().expect("should build the registry"); - let second = state.partner_registry().expect("should reuse the registry"); + let result = build_state_from_settings(settings); assert!( - Arc::ptr_eq(&first, &second), - "should return the cached registry" + result.is_err(), + "should refuse to start with a provider this build cannot run" ); } @@ -3041,8 +3035,22 @@ mod tests { let mut settings = test_settings(); settings.ec.ec_store = Some("ec_identity_store".to_owned()); settings.ec.partners = vec![partner]; - settings.ec.identity_providers = vec![provider_config.clone()]; + // This build registers no kind, so the app state is built without the + // provider and the test supplies its registry and coordinator. let router = TrustedServerApp::routes_for_state(&app_state_for_settings(settings.clone())); + let partners = { + let mut with_provider = settings.ec.clone(); + with_provider.identity_providers = vec![provider_config.clone()]; + Arc::new( + PartnerRegistry::from_ec_with_kinds(&with_provider, |_| true) + .expect("should build registry"), + ) + }; + let app_state = AppState { + partner_registry: Arc::clone(&partners), + ..Arc::into_inner(app_state_for_settings(settings.clone())) + .expect("should own the app state") + }; let mut request = navigation_request("/article"); // Viceroy geolocates loopback to a US privacy state; a US Privacy // string without a sale opt-out lets the navigation mint an EC. @@ -3082,14 +3090,11 @@ mod tests { })); let coordinator = ProviderCoordinator::new( vec![RegisteredProvider::new(provider_config, provider)], - Arc::new( - PartnerRegistry::from_ec_with_kinds(&settings.ec, |_| true) - .expect("should build registry"), - ), + partners, Duration::from_millis(200), ) .expect("should build coordinator"); - let transport = PlatformLookupTransport::new(&state.services, false); + let transport = PlatformLookupTransport::new(&state.services); let consent = ConsentContext { jurisdiction: Jurisdiction::NonRegulated, ..Default::default() @@ -3112,10 +3117,10 @@ mod tests { let result = crate::apply_edgezero_ec_finalize( &settings, - None, + Some(&app_state), &mut state, Some(super::IdentityLookups { - coordinator, + coordinator: Arc::new(coordinator), pending, }), &mut response, diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 134889694b..cc8ba854e8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -38,7 +38,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; -use trusted_server_identity_providers::{build_after_response_coordinator, partner_registry}; +use trusted_server_identity_providers::partner_registry; mod app; mod backend; @@ -579,6 +579,7 @@ fn edgezero_main( run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); run_after_response_lookups( settings, + app_state.as_deref(), &partner_registry, &mut ec_state, after_response, @@ -614,6 +615,7 @@ fn edgezero_main( ); run_after_response_lookups( &settings, + None, &partner_registry, &mut ec_state, after_response, @@ -704,13 +706,12 @@ fn apply_edgezero_ec_finalize( pending, }) = identity { - let transport = - PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); + let transport = PlatformLookupTransport::new(&ec_state.services); let results = futures::executor::block_on(coordinator.collect(&transport, pending)); ec_state.ec_context.set_lookup_results(results); } let partner_registry = match state { - Some(state) => state.partner_registry()?, + Some(state) => state.partner_registry(), None => Arc::new(partner_registry(&settings.ec)?), }; let finalize_kv_graph = if ec_state.use_finalize_kv { @@ -765,6 +766,7 @@ fn run_edgezero_pull_sync_after_send( /// for. fn run_after_response_lookups( settings: &Settings, + state: Option<&AppState>, partner_registry: &Arc, ec_state: &mut EcFinalizeState, lookups: Option, @@ -782,14 +784,8 @@ fn run_after_response_lookups( log::debug!("After-response lookups skipped: EC changed during the request"); return; } - let coordinator = match build_after_response_coordinator(settings, Arc::clone(partner_registry)) - { - Ok(Some(coordinator)) => coordinator, - Ok(None) => return, - Err(report) => { - log::error!("After-response identity providers disabled for this request: {report:?}"); - return; - } + let Some(coordinator) = state.and_then(|state| state.after_response_identity.as_ref()) else { + return; }; let Some(graph) = maybe_identity_graph(settings) else { return; @@ -803,8 +799,7 @@ fn run_after_response_lookups( log::debug!("After-response lookups skipped: new EC or no live row"); return; } - let transport = - PlatformLookupTransport::new(&ec_state.services, settings.proxy.certificate_check); + let transport = PlatformLookupTransport::new(&ec_state.services); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); let now = current_timestamp(); let pending = futures::executor::block_on(coordinator.dispatch( diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9a35062fc7..a9f4e031d2 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -133,6 +133,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Spin")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 50fdd263f8..1f5a8932b8 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -17,7 +17,7 @@ use crate::ec::EcKvSnapshot; use crate::ec::current_timestamp; use crate::ec::eid_types::UnixSeconds; use crate::ec::kv::KvIdentityGraph; -use crate::ec::kv_types::MAX_UID_LENGTH; +use crate::ec::kv_types::{MAX_STORED_UID_LENGTH, MAX_UID_LENGTH}; use crate::ec::prebid_eids::parse_prebid_eids_cookie; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; @@ -537,7 +537,12 @@ pub(crate) fn resolve_auction_eids( } let now = UnixSeconds::new(current_timestamp()); - Some(entry.live_eids(registry, now).map(Eid::from).collect()) + Some( + entry + .live_eids(registry, ec_context.consent(), now) + .map(Eid::from) + .collect(), + ) } pub(crate) fn resolve_client_auction_eids( @@ -657,7 +662,9 @@ pub(crate) fn merge_auction_eids( } = eid; let mut kept = Vec::with_capacity(uids.len()); for uid in uids { - if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + // Stored provider UIDs may reach the stored limit; client UIDs + // were already capped at `MAX_UID_LENGTH` when parsed. + if uid.id.trim().is_empty() || uid.id.len() > MAX_STORED_UID_LENGTH { continue; } // A duplicate keeps the earlier UID's fields but fills its @@ -2168,6 +2175,26 @@ mod tests { assert_eq!(merged[1].uids[0].id, "u2", "should keep client uid"); } + #[test] + fn merge_auction_eids_keeps_a_stored_provider_uid_past_the_ingest_limit() { + let long = "p".repeat(MAX_UID_LENGTH + 100); + let resolved_eids = Some(vec![Eid::new( + "ids.example.com".to_string(), + vec![Uid { + id: long.clone(), + atype: Some(1), + ext: None, + }], + )]); + + let merged = merge_auction_eids(None, resolved_eids).expect("should merge EIDs"); + + assert_eq!( + merged[0].uids[0].id, long, + "should send a stored UID within the stored limit" + ); + } + #[test] fn merge_auction_eids_deduplicates_client_and_resolved_ids() { let client_eids = Some(vec![Eid::new( diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 44a592f1f3..cb98956130 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -344,10 +344,10 @@ fn with_placeholder_partner_tokens(ec: &Ec) -> Ec { ec } -/// Checks the identity-provider rules core owns (spec §12.1). +/// Checks the identity-provider rules core owns. /// -/// Unknown kinds and capability checks need the provider crate and are -/// enforced when the adapter builds the coordinator. +/// Unknown kinds and capability checks need the provider crate; the adapter +/// builds the coordinators at startup and fails to start on either. fn validate_identity_providers(ec: &Ec) -> Result<(), Report> { if ec.identity_providers.is_empty() { return Ok(()); diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index a0d8151a36..7b8493beb2 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -26,6 +26,7 @@ use serde_json::Value as JsonValue; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; +use crate::consent::ConsentContext; use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; @@ -508,9 +509,15 @@ fn iso_timestamp(unix_seconds: u64) -> Option { /// Derives the auction EID view for a valid entry through the same /// [`KvEntry::live_eids`] resolution the live auction uses, reporting why each /// stored ID was skipped. +/// +/// Vendor consent belongs to the visitor's request, which this diagnostic +/// does not have, so no source is withheld for it here. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { let now = UnixSeconds::new(current_timestamp()); - let eids = entry.live_eids(registry, now).map(Eid::from).collect(); + let eids = entry + .live_eids(registry, &ConsentContext::default(), now) + .map(Eid::from) + .collect(); let mut skipped = Vec::new(); for (source_domain, partner_uid) in &entry.ids { @@ -1442,7 +1449,7 @@ mod tests { #[test] fn eids_lookup_reports_configured_source_without_valid_uid() { - let oversized_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); + let oversized_uid = "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1); let cookie = eids_cookie_for(&serde_json::json!([{ "source": "bidstream.example", "uids": [ @@ -1596,7 +1603,11 @@ mod tests { let view_json = serde_json::to_value(&view.eids).expect("should serialize view eids"); let live: Vec = entry - .live_eids(®istry, UnixSeconds::new(current_timestamp())) + .live_eids( + ®istry, + &ConsentContext::default(), + UnixSeconds::new(current_timestamp()), + ) .map(Eid::from) .collect(); assert_eq!( diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index 59abf86484..e14de5eb18 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -149,7 +149,7 @@ mod tests { .collect(); let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); assert!( - registry.mark_provider_owned("ids.example.com"), + registry.mark_provider_owned("ids.example.com", None), "should mark the provider source as owned" ); registry diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 66a894dad8..756cc0f490 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -34,6 +34,7 @@ const REASON_INVALID_PARTNER_UID: &str = "invalid_partner_uid"; const REASON_INELIGIBLE: &str = "ineligible"; const REASON_KV_UNAVAILABLE: &str = "kv_unavailable"; const REASON_PROVIDER_OWNED: &str = "provider_owned"; +const REASON_ROW_FULL: &str = "row_full"; /// Maximum number of mappings allowed in a single batch request. const MAX_BATCH_SIZE: usize = 1000; @@ -278,6 +279,12 @@ fn process_mappings( Ok(UpsertResult::Written | UpsertResult::Unchanged) => { accepted += group.indexes.len(); } + Ok(UpsertResult::RowFull) => { + errors.extend(group.indexes.iter().map(|&index| MappingError { + index, + reason: REASON_ROW_FULL, + })); + } Ok(UpsertResult::NotFound | UpsertResult::ConsentWithdrawn) => { errors.extend(group.indexes.iter().map(|&index| MappingError { index, @@ -595,6 +602,27 @@ mod tests { ); } + #[test] + fn process_mappings_reports_a_full_row_and_continues() { + let writer = MockWriter::new(vec![Ok(UpsertResult::RowFull), Ok(UpsertResult::Written)]); + + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 1, "should accept the mapping after the full row"); + assert_eq!(errors.len(), 1, "should report only the full row"); + assert_eq!(errors[0].index, 0, "should report the full row's index"); + assert_eq!( + errors[0].reason, REASON_ROW_FULL, + "should give a full row its own reason" + ); + assert_eq!(writer.calls().len(), 2, "should continue past a full row"); + } + #[test] fn process_mappings_aborts_on_kv_unavailable() { let writer = MockWriter::new(vec![ diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 11f4c41855..c3f6a9a642 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -25,8 +25,8 @@ use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::generation::ec_hash; use crate::ec::kv_types::KvEntry; -use crate::ec::page_load::RowChange; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, @@ -34,7 +34,7 @@ use crate::ec::provider::{ }; use crate::ec::rate_limiter::RateLimiter; use crate::ec::registry::PartnerRegistry; -use crate::ec::write_plan::{WritePlanInput, plan_write}; + use crate::settings::IdentityProviderConfig; /// Coordinates server-side identity lookups for one navigation. @@ -64,12 +64,6 @@ pub trait IdentityCoordinator { transport: &T, pending: PendingLookups, ) -> LookupResults; - - /// Merges browser ingest and lookup results into the navigation's single - /// row change. - /// - /// Returns `None` when the row would not change, so no write is made. - fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; } /// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. @@ -148,25 +142,13 @@ pub struct RegisteredProvider

{ pub config: IdentityProviderConfig, /// The built provider. pub provider: P, - rate_limit_per_hour: Option, } impl

RegisteredProvider

{ /// Pairs a configuration entry with its built provider. #[must_use] pub fn new(config: IdentityProviderConfig, provider: P) -> Self { - Self { - config, - provider, - rate_limit_per_hour: None, - } - } - - /// Applies an hourly per-EC lookup limit (carried over from pull sync). - #[must_use] - pub fn with_rate_limit(mut self, per_hour: NonZeroU32) -> Self { - self.rate_limit_per_hour = Some(per_hour); - self + Self { config, provider } } } @@ -241,11 +223,7 @@ impl ProviderCoordinator

{ fn register( registered: RegisteredProvider

, ) -> Result, Report> { - let RegisteredProvider { - config, - provider, - rate_limit_per_hour, - } = registered; + let RegisteredProvider { config, provider } = registered; let sources = parse_sources(&config)?; let capable: HashSet<&SourceDomain> = provider .capabilities() @@ -269,7 +247,7 @@ impl ProviderCoordinator

{ .map(|domain| Box::from(domain.as_str())) .collect(), tcf_vendor_id: config.tcf_vendor_id, - rate_limit_per_hour, + rate_limit_per_hour: config.rate_limit_per_hour, }, provider, }) @@ -326,14 +304,17 @@ impl ProviderCoordinator

{ (due, input_blocked) } - /// Whether the provider's hourly per-EC limit is reached. Limiter errors - /// fail open, matching pull sync. + /// Whether the provider's hourly limit is reached. Limiter errors fail + /// open, matching pull sync. + /// + /// The bucket is the EC hash (derived from the client IP), not the full + /// EC ID, so ECs behind one IP share it and a client that clears cookies + /// cannot reset it. fn rate_limited(registered: &Registered

, lookup: &LookupContext<'_>) -> bool { let Some(limit) = registered.policy.rate_limit_per_hour else { return false; }; - let hash = lookup.ec_id.split('.').next().unwrap_or(lookup.ec_id); - let key = format!("identity:{}:{hash}", registered.id); + let key = format!("identity:{}:{}", registered.id, ec_hash(lookup.ec_id)); match lookup.rate_limiter.exceeded(&key, limit.get()) { Ok(exceeded) => exceeded, Err(report) => { @@ -534,8 +515,10 @@ impl IdentityCoordinator for ProviderCoordinator

{ // Only a due source without a live UID may take a generated // ID: a refresh-due source keeps its UID and goes to lookup. eids.retain(|eid| { - due.iter().any(|source| source.as_str() == eid.source) - && !Self::holds_live_uid(entry, &eid.source, lookup.now) + SourceDomain::try_from(eid.source.as_ref()).is_ok_and(|source| { + due.contains(&source) + && !Self::holds_live_uid(entry, source.as_str(), lookup.now) + }) }); if !eids.is_empty() { pending.generated.push(Tagged { @@ -552,12 +535,15 @@ impl IdentityCoordinator for ProviderCoordinator

{ let mut request = match registered.provider.build_request(&input) { Ok(request) => request, Err(report) => { - log::debug!( + log::warn!( "Identity lookup: provider={} request not built: {}", registered.id, report.current_context() ); - pending.early.push(skip(SkipReason::Input)); + pending.early.push(Tagged { + provider, + value: LookupStatus::RequestFailed, + }); continue; } }; @@ -628,10 +614,6 @@ impl IdentityCoordinator for ProviderCoordinator

{ } LookupResults::new(&ec_id, lookups) } - - fn plan_write(&self, input: &WritePlanInput<'_>) -> Option { - plan_write(input) - } } fn log_outcome(lookup: &ProviderLookup) { @@ -1178,10 +1160,10 @@ mod tests { #[test] fn rate_limited_provider_is_skipped() { - let coordinator = coordinator_with( - RegisteredProvider::new(provider_config(""), echo_provider(SourceInput::Anonymous)) - .with_rate_limit(NonZeroU32::new(1).expect("should be non-zero")), - ); + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + echo_provider(SourceInput::Anonymous), + )); let transport = CannedTransport::new(ok_reply); let results = run_with( @@ -1469,8 +1451,54 @@ mod tests { ); } - /// An echo provider that generates its ID instead of looking it up. - struct GeneratingProvider(EchoProvider); + /// An echo provider whose request can never be built. + struct UnbuildableProvider(EchoProvider); + + impl IdentityProvider for UnbuildableProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + } + + #[test] + fn a_request_the_provider_cannot_build_is_reported_as_a_failure() { + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + UnbuildableProvider(echo_provider(SourceInput::Anonymous)), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::RequestFailed, + "should report a provider bug instead of a skip" + ); + } + + /// An echo provider that generates its ID for the given source instead of + /// looking it up. + struct GeneratingProvider(EchoProvider, &'static str); impl IdentityProvider for GeneratingProvider { fn capabilities(&self) -> &ProviderCapabilities { @@ -1493,7 +1521,7 @@ mod tests { fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { Some(vec![ProviderEid { - source: Cow::Borrowed("ids.example.com"), + source: Cow::Borrowed(self.1), inserter: None, matcher: None, mm: None, @@ -1510,10 +1538,14 @@ mod tests { } fn generating_coordinator() -> ProviderCoordinator { + generating_coordinator_for("ids.example.com") + } + + fn generating_coordinator_for(source: &'static str) -> ProviderCoordinator { ProviderCoordinator::new( vec![RegisteredProvider::new( provider_config(""), - GeneratingProvider(echo_provider(SourceInput::Anonymous)), + GeneratingProvider(echo_provider(SourceInput::Anonymous), source), )], partners(), Duration::from_millis(200), @@ -1541,6 +1573,21 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send no lookup"); } + #[test] + fn a_generated_id_matches_its_due_source_after_normalizing() { + let coordinator = generating_coordinator_for("IDS.example.com"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID for a differently cased source" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + #[test] fn a_refresh_due_live_source_is_looked_up_not_regenerated() { let coordinator = generating_coordinator(); diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index c8e6578f71..e7348cda02 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -51,8 +51,6 @@ impl DropCounts { /// Why a provider was not called. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SkipReason { - /// The provider is disabled. - Disabled, /// Consent does not permit this provider. Consent, /// None of its sources is due. @@ -84,6 +82,8 @@ pub enum LookupStatus { Timeout, /// A non-success HTTP status. HttpError(http::StatusCode), + /// The provider could not build its request. + RequestFailed, /// The request could not be sent or the body exceeded the limit. TransportFailed, /// The body could not be parsed at all. @@ -101,9 +101,9 @@ impl LookupStatus { Self::NoMatch => "no_match", Self::Timeout => "timeout", Self::HttpError(_) => "http_error", + Self::RequestFailed => "request_error", Self::TransportFailed => "transport_error", Self::InvalidResponse => "invalid_response", - Self::Skipped(SkipReason::Disabled) => "skipped_disabled", Self::Skipped(SkipReason::Consent) => "skipped_consent", Self::Skipped(SkipReason::NotDue) => "skipped_not_due", Self::Skipped(SkipReason::Input) => "skipped_input", diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs index 9402d6aab2..668fbb5b91 100644 --- a/crates/trusted-server-core/src/ec/coordinator/transport.rs +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -136,17 +136,19 @@ impl core::error::Error for TransportError {} /// canonicalized, keeping the number of registered dynamic backends bounded. pub struct PlatformLookupTransport<'s> { services: &'s RuntimeServices, - certificate_check: bool, next: Cell, } impl<'s> PlatformLookupTransport<'s> { /// Borrows the request's runtime services. + /// + /// Lookups always verify the provider's certificate: they carry the EC ID + /// and the provider's credential, so `proxy.certificate_check` (meant for + /// development origins) does not apply to them. #[must_use] - pub fn new(services: &'s RuntimeServices, certificate_check: bool) -> Self { + pub fn new(services: &'s RuntimeServices) -> Self { Self { services, - certificate_check, next: Cell::new(0), } } @@ -191,7 +193,7 @@ impl LookupTransport for PlatformLookupTransport<'_> { host, port: url.port(), host_header_override: None, - certificate_check: self.certificate_check, + certificate_check: true, first_byte_timeout: timeout, between_bytes_timeout: timeout, discriminator: Some(format!("ts-identity-{sequence}")), @@ -349,7 +351,7 @@ mod tests { client.push_response(200, b"{\"a\":1}".to_vec()); client.push_response(204, Vec::new()); let services = build_services_with_http_client(client.clone()); - let transport = PlatformLookupTransport::new(&services, true); + let transport = PlatformLookupTransport::new(&services); let deadline = Deadline::after(Duration::from_millis(200)); let results = futures::executor::block_on(async { @@ -411,7 +413,7 @@ mod tests { client.push_response(200, Vec::new()); client.push_select_delay(Duration::from_millis(50)); let services = build_services_with_http_client(client.clone()); - let transport = PlatformLookupTransport::new(&services, true); + let transport = PlatformLookupTransport::new(&services); let results = gather_one(&transport, Deadline::after(Duration::from_millis(1))); @@ -427,7 +429,7 @@ mod tests { let client = Arc::new(StubHttpClient::new()); client.push_response(200, vec![b'x'; MAX_PROVIDER_RESPONSE_BYTES + 1]); let services = build_services_with_http_client(client.clone()); - let transport = PlatformLookupTransport::new(&services, true); + let transport = PlatformLookupTransport::new(&services); let results = gather_one(&transport, Deadline::after(Duration::from_millis(200))); @@ -445,7 +447,7 @@ mod tests { fn gather_with_no_requests_returns_nothing() { let client = Arc::new(StubHttpClient::new()); let services = build_services_with_http_client(client.clone()); - let transport = PlatformLookupTransport::new(&services, true); + let transport = PlatformLookupTransport::new(&services); let results = futures::executor::block_on( transport.gather(Vec::new(), Deadline::after(Duration::from_millis(1))), diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 367e049e55..9d9a63b175 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -321,7 +321,7 @@ mod tests { let partners = registry(); let provider = ProviderId::new("example-provider"); let claimed = claimed(); - let long = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); + let long = "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1); let mut expired = uid("u-old"); expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); diff --git a/crates/trusted-server-core/src/ec/eid_types.rs b/crates/trusted-server-core/src/ec/eid_types.rs index 9b1a647f59..9c9826e5d3 100644 --- a/crates/trusted-server-core/src/ec/eid_types.rs +++ b/crates/trusted-server-core/src/ec/eid_types.rs @@ -11,7 +11,7 @@ use serde::de::{self, Deserializer, Visitor}; use serde::{Deserialize, Serialize, Serializer}; use serde_json::value::RawValue; -use super::kv_types::MAX_UID_LENGTH; +use super::kv_types::MAX_STORED_UID_LENGTH; use super::partner::normalize_partner_source_domain; /// Maximum serialized size of an `ext` object, in bytes. @@ -27,7 +27,7 @@ pub enum EidValidationError { /// The UID is blank. #[display("UID is empty")] EmptyUid, - /// The UID exceeds [`MAX_UID_LENGTH`]. + /// The UID exceeds [`MAX_STORED_UID_LENGTH`]. #[display("UID is too long")] UidTooLong, /// The agent type is negative. @@ -170,7 +170,7 @@ impl<'de> Deserialize<'de> for SourceDomain { } } -/// A non-blank UID of at most [`MAX_UID_LENGTH`] bytes, kept verbatim. +/// A non-blank UID of at most [`MAX_STORED_UID_LENGTH`] bytes, kept verbatim. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] #[serde(try_from = "String")] pub struct UidValue(Box); @@ -186,7 +186,7 @@ impl UidValue { if value.trim().is_empty() { return Err(EidValidationError::EmptyUid); } - if value.len() > MAX_UID_LENGTH { + if value.len() > MAX_STORED_UID_LENGTH { return Err(EidValidationError::UidTooLong); } Ok(()) @@ -380,7 +380,12 @@ impl RawJson { if text.len() > MAX_EXT_BYTES || !text.trim_start().starts_with('{') { return Err(EidValidationError::InvalidExt); } - Ok(()) + // `RawValue` skips nested values without serde_json's recursion limit, + // so parse once here: an `ext` that cannot become a `Value` would make + // `to_value` panic on every serve of the row. + serde_json::from_str::(text) + .map(|_| ()) + .map_err(|_| EidValidationError::InvalidExt) } } @@ -440,7 +445,7 @@ mod tests { #[test] fn uid_value_rejects_blank_and_oversized() { assert!(UidValue::try_from(" ").is_err(), "should reject blank UID"); - let long = "a".repeat(MAX_UID_LENGTH + 1); + let long = "a".repeat(MAX_STORED_UID_LENGTH + 1); assert!( UidValue::try_from(long.as_str()).is_err(), "should reject oversized UID" @@ -535,6 +540,23 @@ mod tests { ); } + #[test] + fn raw_json_rejects_ext_nested_past_the_parse_limit() { + let depth = 150; + let text = format!("{}1{}", r#"{"a":"#.repeat(depth), "}".repeat(depth)); + let raw: Box = + serde_json::from_str(&text).expect("should read deep JSON as a raw value"); + assert!( + text.len() <= MAX_EXT_BYTES, + "should fit the size limit so only the depth is under test" + ); + + assert!( + RawJson::try_from(raw).is_err(), + "should reject an ext that cannot be parsed into a value" + ); + } + #[test] fn raw_json_compares_by_bytes() { let a = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 8bd39614a8..a19f51e008 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -7,6 +7,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use error_stack::{Report, ResultExt as _}; +use crate::consent::{ConsentContext, tcf_vendor_consent_denied}; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; @@ -102,15 +103,22 @@ pub fn source_views<'row>( impl KvEntry { /// Iterates live entries for registered, `bidstream_enabled` sources in /// source order, borrowing from the row. + /// + /// A source whose owning provider has a TCF vendor ID is skipped while + /// `consent` withholds that vendor's consent. pub fn live_eids<'row>( &'row self, registry: &'row PartnerRegistry, + consent: &'row ConsentContext, now: UnixSeconds, ) -> impl Iterator> + 'row { self.ids.iter().flat_map(move |(source, record)| { - let partner = registry - .get(source) - .filter(|partner| partner.bidstream_enabled); + let partner = registry.get(source).filter(|partner| { + partner.bidstream_enabled + && !partner + .owner_tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(consent, vendor)) + }); partner .into_iter() .flat_map(move |partner| source_views(source, record, partner.openrtb_atype, now)) @@ -175,6 +183,8 @@ mod tests { use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::redacted::Redacted; use crate::settings::EcPartner; + use crate::test_support::tests::consent_granting_vendors; + use std::num::NonZeroU16; fn make_test_partner(source_domain: &str) -> EcPartner { EcPartner { @@ -278,7 +288,7 @@ mod tests { ); let eids: Vec = entry - .live_eids(®istry, UnixSeconds::new(20)) + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(20)) .map(Eid::from) .collect(); @@ -292,6 +302,37 @@ mod tests { assert_eq!(ids, vec!["new"], "should drop expired uid"); } + #[test] + fn live_eids_withhold_a_source_whose_owner_lacks_vendor_consent() { + let mut registry = registry_with("ids.example.com", true); + assert!( + registry.mark_provider_owned("ids.example.com", NonZeroU16::new(9999)), + "should mark the source" + ); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[("p-1", Expiry::Never)]), + ); + let denied = consent_granting_vendors(vec![1]); + let granted = consent_granting_vendors(vec![9999]); + + assert_eq!( + entry + .live_eids(®istry, &denied, UnixSeconds::new(0)) + .count(), + 0, + "should withhold the provider's UIDs without its vendor consent" + ); + assert_eq!( + entry + .live_eids(®istry, &granted, UnixSeconds::new(0)) + .count(), + 1, + "should serve the provider's UIDs with its vendor consent" + ); + } + #[test] fn live_eids_respect_bidstream_flag() { let registry = registry_with("ids.example.com", false); @@ -300,7 +341,9 @@ mod tests { .ids .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); assert_eq!( - entry.live_eids(®istry, UnixSeconds::new(0)).count(), + entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .count(), 0, "should hide non-bidstream source" ); @@ -314,7 +357,7 @@ mod tests { .ids .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); let eids: Vec = entry - .live_eids(®istry, UnixSeconds::new(0)) + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) .map(Eid::from) .collect(); assert_eq!(eids[0].uids[0].id, "v", "should emit legacy uid"); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 2e28f92a23..d563c5b182 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -3,7 +3,10 @@ //! Centralizes post-routing EC behavior so all handlers get consistent cookie //! and KV semantics. Every browser-driven KV write goes through the //! navigation's single page-load write ([`KvIdentityGraph::commit_page_load`]), -//! so a request without a navigation permit never writes. +//! so a request without a navigation permit never writes. The one exception is +//! a consent withdrawal, whose tombstone is written on any request. + +use std::collections::BTreeSet; use edgezero_core::body::Body as EdgeBody; use http::Response; @@ -15,15 +18,15 @@ use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; use super::eid_types::UnixSeconds; -use super::generation::generate_ec_id; +use super::generation::{generate_ec_id, is_valid_ec_id}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; -use super::page_load::{Generation, PageLoadWrite, RowChange}; +use super::page_load::{PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; use super::write_plan::{WritePlanInput, plan_write}; -use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; +use super::{EcKvSnapshot, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -38,15 +41,15 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// Applies withdrawal handling, browser EID ingestion, provider lookup /// results, orphan recovery and cookie writes for new EC generation. /// -/// KV is written at most once, and only when the request carries a -/// navigation permit (see [`super::page_load`]) and the final response is not -/// a redirect. Without a permit a newly generated EC is not issued and a -/// withdrawal only strips EC headers: the EC cookie is kept so the next -/// navigation can write the tombstone. +/// The identity row is written at most once, and only when the request +/// carries a navigation permit (see [`super::page_load`]) and the final +/// response is not a redirect. Without a permit a newly generated EC is not +/// issued. /// -/// On consent withdrawal during a navigation, the EC identity-graph KV -/// tombstone is the authoritative revocation marker and the EC cookie is -/// expired once the tombstone is in place (or the row is proven absent). +/// Consent withdrawal is the one exception: on any request carrying the EC +/// cookie, the KV tombstone is written as the authoritative revocation marker +/// and the EC cookie is expired once every tombstone is in place (or the row +/// is proven absent). /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted /// from the request *before* routing consumes it. @@ -79,7 +82,6 @@ pub fn ec_finalize_response( ec_context, kv, registry, - permit, consent_withdrawn, response, ); @@ -237,10 +239,8 @@ fn commit_returning( ec_context.set_kv_snapshot(snapshot); } if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) { - if ec_context.recovery_eligible() { - confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); - } - return matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }); + return ec_context.recovery_eligible() + && confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); } if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Failed { .. }) { return false; @@ -395,12 +395,15 @@ fn recover_orphaned_ec( /// the identity stays intact and recovery is retried on a later navigation; /// - neither a read failure nor a failed existence check is a miss, and neither /// rotates. +/// +/// Returns whether the navigation's write succeeded or had nothing to change, +/// which is what allows the after-response follow-up write. fn confirm_then_recover_orphaned_ec( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: PageLoadWrite, response: &mut Response, -) { +) -> bool { let confirmed = commit.graph.load_snapshot(commit.ec_id); match confirmed { EcKvSnapshot::Present { .. } => { @@ -408,23 +411,28 @@ fn confirm_then_recover_orphaned_ec( // plan the navigation's write against it rather than rotating a // valid identity. ec_context.set_kv_snapshot(confirmed); - update_returning(commit, ec_context, permit); + update_returning(commit, ec_context, permit) } EcKvSnapshot::Missing { .. } => match commit.graph.key_exists_confirmed(commit.ec_id) { - Ok(false) => recover_orphaned_ec(commit, ec_context, permit, response), + Ok(false) => { + recover_orphaned_ec(commit, ec_context, permit, response); + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }) + } Ok(true) => { log::warn!( "Orphan EC recovery skipped: both point reads missed a row the store still \ lists; leaving the identity intact for a later navigation" ); + false } Err(err) => { log::warn!("Orphan EC recovery skipped: existence check failed: {err:?}"); + false } }, // A failed or not-read confirmation is not an authoritative miss: leave // the existing snapshot in place and do not rotate an unconfirmed miss. - EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => {} + EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => false, } } @@ -477,17 +485,21 @@ pub fn clear_ec_on_response(settings: &Settings, response: &mut Response, registry: &PartnerRegistry, - permit: Option, consent_withdrawn: bool, response: &mut Response, ) { @@ -496,112 +508,48 @@ fn finalize_unusable_consent( if !(consent_withdrawn && ec_context.cookie_was_present()) { return; } - let Some(permit) = permit else { - // Not a page-load navigation: serve nothing, keep the EC cookie so the - // next navigation can write the tombstone (spec §3.1). - return; - }; - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { - // No identity graph (or no valid active EC) holds a row to tombstone, - // so the cookie is the only identity state left to clear. + let Some(graph) = kv else { + // No identity graph holds a row to tombstone, so the cookie is the + // only identity state left to clear. expire_ec_cookie(settings, response); return; }; - if withdraw_active_ec(graph, ec_context, &ec_id, permit) { - expire_ec_cookie(settings, response); - } -} -/// Tombstones the active EC's row with the navigation's single write. -/// -/// Returns whether the withdrawal is complete: the row is now a tombstone or -/// is proven absent. A snapshot that is not a usable live row with a CAS -/// generation is re-read once; a miss is confirmed against the primary data -/// source so a stale read never clears the cookie of a live row. -fn withdraw_active_ec( - graph: &KvIdentityGraph, - ec_context: &mut EcContext, - ec_id: &str, - permit: PageLoadWrite, -) -> bool { - let usable = match ec_context.kv_snapshot().entry_for(ec_id) { - Some(entry) => { - !entry.consent.ok || Generation::of(ec_context.kv_snapshot(), ec_id).is_some() - } - None => false, - }; - if !usable { - let snapshot = graph.load_snapshot(ec_id); - ec_context.set_kv_snapshot(snapshot); - } - - let snapshot = ec_context.kv_snapshot(); - if let Some(entry) = snapshot.entry_for(ec_id) { - if !entry.consent.ok { - return true; - } - let Some(expected) = Generation::of(snapshot, ec_id) else { - log::warn!( - "EC withdrawal tombstone for '{}' skipped (no_generation); keeping the cookie", - log_id(ec_id) - ); - return false; - }; - return match graph.commit_page_load(permit, ec_id, RowChange::Tombstone { expected }) { - Ok(tombstone) => { - ec_context.set_kv_snapshot(present_snapshot(ec_id, tombstone)); - true - } - Err(report) => { - log::warn!( - "EC withdrawal tombstone failed for '{}': {}; keeping the cookie", - log_id(ec_id), - report.current_context() - ); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }); - false - } + let mut complete = true; + for ec_id in withdrawal_ec_ids(ec_context) { + // Only the active ID has a snapshot in the context to start from and + // to correct, so post-send work sees the tombstone just written. + let initial = if ec_context.kv_snapshot().belongs_to(&ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead }; - } - if !matches!(snapshot, EcKvSnapshot::Missing { .. }) { - return false; - } - // A still-valid completion marker proves an earlier withdrawal finished, - // so the root existence check can be skipped. - match graph.withdrawal_marker_exists(ec_id, checked_current_timestamp()) { - Ok(true) => return true, - Ok(false) => {} - Err(err) => log::warn!( - "EC withdrawal completion marker lookup failed for '{}': {err:?}", - log_id(ec_id) - ), - } - match graph.key_exists_confirmed(ec_id) { - Ok(false) => true, - Ok(true) => { + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, initial); + if matches!(outcome, EcKvSnapshot::Failed { .. }) { log::warn!( - "EC withdrawal for '{}': point read missed a row the store still lists; \ - keeping the cookie for a later navigation", - log_id(ec_id) + "EC withdrawal tombstone for '{}' failed; keeping the cookie so the next \ + request retries", + log_id(&ec_id) ); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }); - false + complete = false; } - Err(err) => { - log::warn!( - "EC withdrawal for '{}': existence check failed: {err:?}", - log_id(ec_id) - ); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }); - false + if ec_context.ec_value() == Some(ec_id.as_str()) { + ec_context.set_kv_snapshot(outcome); } } + if complete { + expire_ec_cookie(settings, response); + } +} + +/// The EC IDs a withdrawal applies to: the cookie's and the active one. +fn withdrawal_ec_ids(ec_context: &EcContext) -> BTreeSet { + [ec_context.existing_cookie_ec_id(), ec_context.ec_value()] + .into_iter() + .flatten() + .filter(|ec_id| is_valid_ec_id(ec_id)) + .map(str::to_owned) + .collect() } #[cfg(test)] @@ -1651,7 +1599,7 @@ mod tests { let mut registry = PartnerRegistry::from_config(&[make_partner("id5-sync.com")]) .expect("should build registry"); assert!( - registry.mark_provider_owned("id5-sync.com"), + registry.mark_provider_owned("id5-sync.com", None), "should mark the provider source" ); let mut response = empty_response(); @@ -1696,7 +1644,7 @@ mod tests { let mut registry = PartnerRegistry::from_config(&[make_partner("sharedid.org")]) .expect("should build registry"); assert!( - registry.mark_provider_owned("sharedid.org"), + registry.mark_provider_owned("sharedid.org", None), "should mark the provider source" ); let mut response = empty_response(); @@ -2250,7 +2198,7 @@ mod tests { } #[test] - fn finalize_withdrawal_tombstones_only_the_active_id_once() { + fn finalize_withdrawal_tombstones_the_active_and_cookie_ids_once() { let settings = create_test_settings(); let active_ec = sample_ec_id("activ3"); let cookie_ec = sample_ec_id("cook3e"); @@ -2293,10 +2241,10 @@ mod tests { .get(&active_ec) .expect("should read active row") .expect("should retain active tombstone"); - let (cookie_row, cookie_generation) = graph + let (cookie_tombstone, cookie_generation) = graph .get(&cookie_ec) .expect("should read cookie row") - .expect("should retain cookie row"); + .expect("should retain cookie tombstone"); assert!( !active_tombstone.consent.ok, "active row should be withdrawn" @@ -2306,8 +2254,8 @@ mod tests { "active IDs should be cleared" ); assert!( - cookie_row.consent.ok, - "the navigation's single write should tombstone only the active row" + !cookie_tombstone.consent.ok, + "the cookie's row should be withdrawn too" ); ec_context.set_navigation_permit(NavigationPermit::for_test()); @@ -2335,10 +2283,10 @@ mod tests { graph .get(&cookie_ec) .expect("should read cookie row") - .expect("should retain cookie row") + .expect("should retain cookie tombstone") .1, cookie_generation, - "repeated finalization should not rewrite the cookie row" + "repeated finalization should not rewrite the cookie tombstone" ); } @@ -2741,22 +2689,38 @@ mod tests { ); } + fn withdrawing_context(ec_id: &str) -> EcContext { + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + make_context_with_consent(Some(ec_id), Some(ec_id), true, false, consent) + } + + fn assert_withdrawn(graph: &KvIdentityGraph, ec_id: &str, response: &Response) { + let (entry, _) = graph + .get(ec_id) + .expect("should read") + .expect("should keep the tombstone"); + assert!(!entry.consent.ok, "should tombstone the row"); + assert!( + get_header_str(response, "set-cookie") + .is_some_and(|cookie| cookie.contains("Max-Age=0")), + "should expire the EC cookie" + ); + } + #[test] - fn withdrawal_without_permit_keeps_row_and_cookie() { + fn withdrawal_without_permit_tombstones_row_and_expires_cookie() { let settings = create_test_settings(); let ec_id = sample_ec_id("wd0001"); let graph = KvIdentityGraph::in_memory("finalize"); graph .create(&ec_id, &live_entry()) .expect("should seed row"); - let consent = ConsentContext { - jurisdiction: Jurisdiction::UsState("CA".to_owned()), - gpc: true, - source: ConsentSource::Cookie, - ..Default::default() - }; - let mut ec_context = - make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + let mut ec_context = withdrawing_context(&ec_id); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); @@ -2770,30 +2734,111 @@ mod tests { &mut response, ); - let (entry, _) = graph - .get(&ec_id) - .expect("should read") - .expect("should exist"); - assert!( - entry.consent.ok, - "should not tombstone outside a navigation" - ); - assert!( - get_header(&response, "set-cookie").is_none(), - "should keep the EC cookie" - ); + assert_withdrawn(&graph, &ec_id, &response); assert!( get_header(&response, "x-ts-ec").is_none(), "should still strip EC response headers" ); } + #[test] + fn withdrawal_on_a_redirect_tombstones_row_and_expires_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0002"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + } + + #[test] + fn withdrawal_retries_after_a_concurrent_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0003"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + graph + .upsert_partner_id(&ec_id, "partner.example.com", "late-uid") + .expect("should write concurrently after the snapshot"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; entry } + #[test] + fn finalize_withholds_after_response_lookups_when_the_confirmed_write_fails() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after3"); + let graph = KvIdentityGraph::write_failing_with( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ); + // The preload missed a row the confirmation read then finds, and the + // navigation's write against it fails. + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + true, + ); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow a follow-up write after the page-load write failed" + ); + } + #[test] fn finalize_allows_after_response_lookups_on_a_live_row() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 18346249c7..1c217e09e2 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -11,6 +11,7 @@ use url::Url; use super::auth::authenticate_bearer; use super::consent::ec_consent_granted; +use crate::consent::tcf_vendor_consent_denied; use crate::error::TrustedServerError; use crate::openrtb::Eid; use crate::settings::Settings; @@ -97,11 +98,17 @@ pub fn handle_identify( // write back with the live-entry TTL. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { - // Extract only this partner's first live entry. + // Extract only this partner's first live entry. A + // provider-owned source is withheld while its owner lacks TCF + // vendor consent, as in the bidstream. let now = UnixSeconds::new(current_timestamp()); + let vendor_denied = partner + .owner_tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(ec_context.consent(), vendor)); eid = entry .ids .get(&partner.source_domain) + .filter(|_| !vendor_denied) .and_then(|record| { source_views(&partner.source_domain, record, partner.openrtb_atype, now) .next() @@ -341,7 +348,8 @@ mod tests { use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{consent_granting_vendors, create_test_settings}; + use std::num::NonZeroU16; const VALID_API_TOKEN: &str = "identify-test-token-32-bytes-min"; @@ -778,10 +786,24 @@ mod tests { } fn identify_body_for_record(record: KvPartnerId) -> serde_json::Value { + let registry = + PartnerRegistry::from_config(&[make_test_partner("ssp.example.com", VALID_API_TOKEN)]) + .expect("should build registry"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..ConsentContext::default() + }; + identify_body_with(record, ®istry, consent) + } + + fn identify_body_with( + record: KvPartnerId, + registry: &PartnerRegistry, + consent: ConsentContext, + ) -> serde_json::Value { let settings = create_test_settings(); let kv = KvIdentityGraph::in_memory("identify_store"); - let partners = vec![make_test_partner("ssp.example.com", VALID_API_TOKEN)]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let ec_id = format!("{}.ABC123", "a".repeat(64)); let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); entry.ids.insert("ssp.example.com".to_owned(), record); @@ -792,9 +814,9 @@ mod tests { .header("authorization", format!("Bearer {VALID_API_TOKEN}")) .body(EdgeBody::empty()) .expect("should build test request"); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = EcContext::new_for_test(Some(ec_id), consent); - let response = handle_identify(&settings, &kv, ®istry, &req, &ec_context) + let response = handle_identify(&settings, &kv, registry, &req, &ec_context) .expect("should construct identify response"); assert_eq!(response.status(), StatusCode::OK, "should return 200"); @@ -804,6 +826,30 @@ mod tests { .expect("should decode identify response JSON") } + #[test] + fn handle_identify_withholds_a_source_whose_owner_lacks_vendor_consent() { + let mut registry = + PartnerRegistry::from_config(&[make_test_partner("ssp.example.com", VALID_API_TOKEN)]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("ssp.example.com", NonZeroU16::new(9999)), + "should mark the source provider-owned" + ); + let record = || provider_record("live", Expiry::Never); + + let denied = identify_body_with(record(), ®istry, consent_granting_vendors(vec![1])); + let granted = identify_body_with(record(), ®istry, consent_granting_vendors(vec![9999])); + + assert!( + denied.get("uid").is_none(), + "should withhold the UID without the owner's vendor consent" + ); + assert_eq!( + granted["uid"], "live", + "should return the UID with the owner's vendor consent" + ); + } + #[test] fn handle_identify_omits_expired_provider_uid() { let body = diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0dc976387..3d2c40f10c 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -23,7 +23,7 @@ use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; use super::page_load::{AfterResponseWrite, Generation, KvWriteError, PageLoadWrite, RowChange}; -use super::{EcKvSnapshot, current_timestamp, log_id}; +use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. const MAX_CAS_RETRIES: u32 = 5; @@ -60,6 +60,17 @@ pub enum UpsertResult { ConsentWithdrawn, /// The partner ID already had the requested UID, so no write was needed. Unchanged, + /// The row with this partner ID would exceed [`MAX_ENTRY_BYTES`], so + /// nothing was written. Retrying gives the same result. + RowFull, +} + +/// A serialized entry, or the size that kept it from being written. +enum SerializedEntry { + /// Body and metadata ready to write. + Fits { body: String, meta: String }, + /// The body is this many bytes, over [`MAX_ENTRY_BYTES`]. + TooLarge(usize), } /// Partner UID update to apply to a KV identity graph entry. @@ -161,10 +172,30 @@ impl KvIdentityGraph { } /// Serializes an entry body and metadata for insertion. + /// + /// A body over [`MAX_ENTRY_BYTES`] is an error; callers that report a + /// full row separately use [`Self::serialize_entry_unless_full`]. fn serialize_entry( entry: &KvEntry, store_name: &str, ) -> Result<(String, String), Report> { + match Self::serialize_entry_unless_full(entry, store_name)? { + SerializedEntry::Fits { body, meta } => Ok((body, meta)), + SerializedEntry::TooLarge(bytes) => Err(Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Refusing to write KV entry of {bytes} bytes (limit {MAX_ENTRY_BYTES})" + ), + })), + } + } + + /// Serializes an entry body and metadata, reporting an oversized body as + /// [`SerializedEntry::TooLarge`] rather than an error. + fn serialize_entry_unless_full( + entry: &KvEntry, + store_name: &str, + ) -> Result> { entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -177,13 +208,7 @@ impl KvIdentityGraph { message: "Failed to serialize KV entry body".to_owned(), })?; if body.len() > MAX_ENTRY_BYTES { - return Err(Report::new(TrustedServerError::KvStore { - store_name: store_name.to_owned(), - message: format!( - "Refusing to write KV entry of {} bytes (limit {MAX_ENTRY_BYTES})", - body.len() - ), - })); + return Ok(SerializedEntry::TooLarge(body.len())); } let meta = KvMetadata::from_entry(entry); let meta_str = @@ -191,7 +216,10 @@ impl KvIdentityGraph { store_name: store_name.to_owned(), message: "Failed to serialize KV entry metadata".to_owned(), })?; - Ok((body, meta_str)) + Ok(SerializedEntry::Fits { + body, + meta: meta_str, + }) } /// Reads the raw stored body, metadata, and generation for an EC ID key. @@ -354,13 +382,9 @@ impl KvIdentityGraph { /// Applies the navigation's single row change in one attempt. /// - /// The `permit` is consumed, so a navigation cannot write twice. A - /// successful tombstone write also records the existing - /// withdrawal-completion marker: that second insert is the one deliberate - /// exception to the one-write-per-page-load rule. It predates the rule and - /// is kept because later withdrawals rely on it to skip the strongly - /// consistent existence check of an already-withdrawn row; it never changes - /// the identity row itself. + /// The `permit` is consumed, so a navigation cannot write twice. Consent + /// withdrawals do not go through here: their tombstone is a privacy write + /// on any request ([`Self::tombstone_existing_from_snapshot`]). /// /// # Errors /// @@ -375,29 +399,13 @@ impl KvIdentityGraph { change: RowChange, ) -> Result> { permit.spend(); - let is_tombstone = matches!(change, RowChange::Tombstone { .. }); - let (entry, ttl, mode) = match change { - RowChange::Create(entry) => (entry, ENTRY_TTL, EcKvWriteMode::Add), - RowChange::Update { entry, expected } => ( - entry, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(expected.get()), - ), - RowChange::Tombstone { expected } => ( - KvEntry::tombstone(current_timestamp()), - TOMBSTONE_TTL, - EcKvWriteMode::IfGenerationMatch(expected.get()), - ), + let (entry, mode) = match change { + RowChange::Create(entry) => (entry, EcKvWriteMode::Add), + RowChange::Update { entry, expected } => { + (entry, EcKvWriteMode::IfGenerationMatch(expected.get())) + } }; - self.write_row(ec_id, &entry, ttl, mode)?; - if is_tombstone { - // Deliberate exception to the one-write-per-page-load rule: - // the withdrawal-completion marker is a separate key that - // predates the rule. Later withdrawals use it to avoid a - // strongly consistent existence check on a row already - // withdrawn, and it never touches the identity row. - self.record_withdrawal_completion(ec_id, entry.consent.updated); - } + self.write_row(ec_id, &entry, ENTRY_TTL, mode)?; Ok(entry) } @@ -459,8 +467,9 @@ impl KvIdentityGraph { /// - **Existing tombstone** (`consent.ok = false`) — CAS overwrite with /// the new entry. Retries up to [`MAX_CAS_RETRIES`] on conflict. /// - /// This method is reserved for explicit same-key revival. Production EC - /// generation uses [`Self::create_if_absent`] with a freshly generated ID. + /// This method is reserved for explicit same-key revival. A freshly + /// generated EC's row is created by the navigation's page-load write + /// ([`Self::commit_page_load`]). /// /// # Errors /// @@ -635,11 +644,29 @@ impl KvIdentityGraph { if !apply_partner_id_updates(&mut entry, updates) { return current; } - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), + let (body, meta_str) = + match Self::serialize_entry_unless_full(&entry, self.store_name()) { + Ok(SerializedEntry::Fits { body, meta }) => (body, meta), + Ok(SerializedEntry::TooLarge(bytes)) => { + log::warn!( + "snapshot partner upsert for '{}' skipped: row would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES})", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + Err(err) => { + log::warn!( + "snapshot partner upsert for '{}' could not serialize the row: {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } }; - }; match self.write_entry( ec_id, &body, @@ -836,7 +863,18 @@ impl KvIdentityGraph { super::kv_types::KvPartnerId::single(uid.to_owned()), ); - let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; + let (body, meta_str) = + match Self::serialize_entry_unless_full(&entry, self.store_name())? { + SerializedEntry::Fits { body, meta } => (body, meta), + SerializedEntry::TooLarge(bytes) => { + log::warn!( + "upsert_partner_id_if_exists: row for '{}' would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES}); partner '{partner_id}' not written", + log_id(ec_id), + ); + return Ok(UpsertResult::RowFull); + } + }; match self.write_entry( ec_id, @@ -1074,7 +1112,6 @@ impl KvIdentityGraph { self.overwrite_withdrawal_tombstone(ec_id).map(Some) } - #[cfg(test)] fn overwrite_withdrawal_tombstone( &self, ec_id: &str, @@ -1098,6 +1135,199 @@ impl KvIdentityGraph { }) } + /// Writes a withdrawal tombstone only when an existing row can be confirmed. + /// + /// This is a privacy write outside the one-write-per-page-load rule: it + /// runs on any finalized request that carries an explicit withdrawal and + /// an EC cookie, whatever the route or response status. + /// + /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` + /// cookie must not mint a row. But a *point read* cannot prove absence on + /// an eventually-consistent store, and dropping a withdrawal is worse than + /// a redundant read, so absence is established in two stages: + /// + /// 1. Any snapshot that is not a usable `Present` for this EC ID — a + /// publisher preload that read `Missing`, a read that `Failed`, or one + /// lacking a CAS generation — is re-read. + /// 2. A re-read that still reports the row absent is checked against + /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads + /// the primary data source. + /// + /// A tombstone is idempotent, so a CAS conflict re-reads and retries up to + /// [`MAX_CAS_RETRIES`] times. Resolving the initial snapshot happens + /// outside the retry counter. + pub(crate) fn tombstone_existing_from_snapshot( + &self, + ec_id: &str, + snapshot: EcKvSnapshot, + ) -> EcKvSnapshot { + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + _ => self.load_snapshot(ec_id), + }; + + for _attempt in 0..MAX_CAS_RETRIES { + let generation = match current { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return current, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(generation), + .. + } if snapshot_id == ec_id => generation, + // A missing row (including one that disappeared mid-retry) is + // only a no-op once absence is proven against the primary data + // source. + EcKvSnapshot::Missing { + ec_id: ref snapshot_id, + } if snapshot_id == ec_id => { + return self.tombstone_unproven_missing( + ec_id, + current, + checked_current_timestamp(), + ); + } + // A refreshed read that failed (or any other unusable state) + // fails closed rather than silently dropping the withdrawal. + _ => { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + }; + + let tombstone = KvEntry::tombstone(current_timestamp()); + let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + }; + match self.write_entry( + ec_id, + &body, + &meta_str, + TOMBSTONE_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) { + Ok(EcKvWriteOutcome::Written) => { + self.record_withdrawal_completion(ec_id, tombstone.consent.updated); + return EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + }; + } + Ok(EcKvWriteOutcome::PreconditionFailed) => { + current = self.load_snapshot(ec_id); + } + Err(err) => { + log::warn!( + "conditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + } + } + + log::warn!( + "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ + identity-graph row may still be live with consent granted", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + + /// Resolves a withdrawal whose row read as missing. + /// + /// A still-valid completion marker or a confirmed absent root means + /// there is nothing to withdraw, and a forged cookie must not mint a row. A + /// key that provably exists is tombstoned unconditionally because no CAS + /// generation is available after a missed read. Marker-check failure falls + /// back to that privacy write; root-existence failure leaves withdrawal + /// unresolved rather than silently dropped. + fn tombstone_unproven_missing( + &self, + ec_id: &str, + missing: EcKvSnapshot, + now: Option, + ) -> EcKvSnapshot { + match self.withdrawal_marker_exists(ec_id, now) { + Ok(true) => { + log::debug!( + "withdrawal tombstone for '{}': completion marker already exists", + log_id(ec_id) + ); + return missing; + } + Ok(false) => {} + Err(err) => { + // Marker failure must not weaken withdrawal. Fall back to the + // existing root existence check and unconditional privacy write. + log::warn!( + "withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } + + match self.key_exists_confirmed(ec_id) { + Ok(false) => missing, + Ok(true) => { + log::warn!( + "withdrawal tombstone for '{}': point read missed a row the store still \ + lists; writing an unconditional tombstone", + log_id(ec_id) + ); + match self.overwrite_withdrawal_tombstone(ec_id) { + Ok(tombstone) => { + self.record_withdrawal_completion(ec_id, tombstone.consent.updated); + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + } + } + Err(err) => { + log::warn!( + "unconditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + Err(err) => { + log::warn!( + "withdrawal tombstone for '{}': existence check failed, cannot confirm \ + absence: {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -1170,7 +1400,20 @@ impl KvIdentityGraph { network.cluster_size = Some(cluster_size); updated_entry.network = Some(network); - let (body, meta_str) = Self::serialize_entry(&updated_entry, self.store_name())?; + let (body, meta_str) = + match Self::serialize_entry_unless_full(&updated_entry, self.store_name())? { + SerializedEntry::Fits { body, meta } => (body, meta), + SerializedEntry::TooLarge(bytes) => { + // The computed value is still valid; only the cache write + // is skipped. + log::warn!( + "evaluate_cluster: row for '{}' would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES}); cluster_size not stored", + log_id(ec_id) + ); + return Ok(Some(cluster_size)); + } + }; match self.write_entry( ec_id, @@ -1225,6 +1468,16 @@ impl KvIdentityGraph { )) } + /// A graph holding `entry` at `ec_id` whose reads succeed and whose + /// writes all fail. + pub(crate) fn write_failing_with(ec_id: &str, entry: &KvEntry) -> Self { + let store = super::kv_backend::test_support::WriteFailingEcKv::new(); + let (body, meta) = Self::serialize_entry(entry, store.store_name()) + .expect("should serialize the seed row"); + store.seed(ec_id, &body, &meta); + Self::new(store) + } + /// Test helper: a graph backed by an in-memory store with generation /// tracking. pub(crate) fn in_memory(store_name: impl Into) -> Self { @@ -1271,9 +1524,8 @@ mod tests { use super::*; use crate::ec::checked_current_timestamp; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; - use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_backend::test_support::{InMemoryEcKv, WriteFailingEcKv}; use crate::ec::kv_types::{KvEid, KvUid}; - use crate::ec::page_load::{Generation, NavigationPermit}; fn snapshot_ec_id() -> String { format!("{}.ABC123", "a".repeat(64)) @@ -1314,7 +1566,7 @@ mod tests { entry.ids.insert( "ssp_x".to_owned(), crate::ec::kv_types::KvPartnerId::single( - "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), ), ); entry.ids.insert( @@ -1354,7 +1606,7 @@ mod tests { entry.ids.insert( "ssp_x".to_owned(), crate::ec::kv_types::KvPartnerId::single( - "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), ), ); @@ -1996,6 +2248,46 @@ mod tests { ); } + #[test] + fn upsert_partner_id_if_exists_reports_a_full_row_without_writing() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = live_entry(); + let uid = "u".repeat(500); + let mut partner = 0; + while serde_json::to_string(&entry) + .expect("should serialize entry") + .len() + < MAX_ENTRY_BYTES - 600 + { + entry.ids.insert( + format!("ssp{partner}.example.com"), + crate::ec::kv_types::KvPartnerId::single(uid.clone()), + ); + partner += 1; + } + kv.create(&ec_id, &entry) + .expect("should create a near-full row"); + + let result = kv + .upsert_partner_id_if_exists(&ec_id, "full.example.com", &uid) + .expect("should report a full row as a result, not an error"); + + assert_eq!( + result, + UpsertResult::RowFull, + "should refuse a partner ID that would overflow the row" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read the row") + .expect("should keep the row"); + assert!( + !stored.ids.contains_key("full.example.com"), + "should not write the refused partner ID" + ); + } + #[test] fn upsert_partner_id_if_exists_writes_and_detects_unchanged() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -2343,53 +2635,6 @@ mod tests { } } - /// [`EcKvStore`] whose reads succeed but every write fails, simulating a - /// store that becomes unwritable mid-request. - struct WriteFailingEcKv { - inner: InMemoryEcKv, - } - - impl WriteFailingEcKv { - fn new() -> Self { - Self { - inner: InMemoryEcKv::new("write-failing-store"), - } - } - } - - impl EcKvStore for WriteFailingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - _key: &str, - _write: EcKvWrite<'_>, - ) -> Result> { - Err(Report::new(TrustedServerError::KvStore { - store_name: self.inner.store_name().to_owned(), - message: "write failing test store".to_owned(), - })) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn snapshot_upsert_with_generation_writes_without_reading() { let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -3123,13 +3368,13 @@ mod tests { } fn tombstone_via_page_load(kv: &KvIdentityGraph, ec_id: &str) { - let snapshot = kv.load_snapshot(ec_id); - let expected = Generation::of(&snapshot, ec_id).expect("should read generation"); - let permit = NavigationPermit::for_test() - .confirm(http::StatusCode::OK) - .expect("should confirm permit"); - kv.commit_page_load(permit, ec_id, RowChange::Tombstone { expected }) - .expect("should write tombstone"); + let outcome = kv.tombstone_existing_from_snapshot(ec_id, EcKvSnapshot::NotRead); + assert!( + outcome + .entry_for(ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should write tombstone" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index bfe74ca6f3..70a999ffa0 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -178,6 +178,68 @@ pub(crate) mod test_support { use super::*; + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a + /// store that becomes unwritable mid-request. + pub(crate) struct WriteFailingEcKv { + inner: InMemoryEcKv, + } + + impl WriteFailingEcKv { + pub(crate) fn new() -> Self { + Self { + inner: InMemoryEcKv::new("write-failing-store"), + } + } + + /// Stores a row directly, bypassing the failing write path. + pub(crate) fn seed(&self, key: &str, body: &str, metadata: &str) { + self.inner + .insert( + key, + EcKvWrite { + body, + metadata, + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::Overwrite, + }, + ) + .expect("should seed the in-memory store"); + } + } + + impl EcKvStore for WriteFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + _key: &str, + _write: EcKvWrite<'_>, + ) -> Result> { + Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "write failing test store".to_owned(), + })) + } + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// [`EcKvStore`] wrapper that counts `lookup` calls through a shared counter /// so tests can prove exactly how many reads a flow performs. pub(crate) struct CountingEcKv { diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index b24e91e6c0..f13b258dfc 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -36,10 +36,18 @@ pub const MAX_SEEN_DOMAINS: usize = 50; pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// Maximum allowed length (in bytes) for a partner UID across all sync -/// mechanisms (pixel, batch, pull) and identity providers. Sized for -/// `LiveRamp` envelopes (about 700 bytes). Defined centrally to ensure -/// consistent validation. -pub const MAX_UID_LENGTH: usize = 1024; +/// mechanisms (pixel, batch, pull) and browser ingest. Defined centrally to +/// ensure consistent validation. +/// +/// Builds before identity providers reject a row holding a longer UID, so +/// these writers stay at this limit until rollback past them is no longer +/// supported; only provider writes use [`MAX_STORED_UID_LENGTH`]. +pub const MAX_UID_LENGTH: usize = 512; + +/// Maximum length (in bytes) of a UID a row may hold: the limit for identity +/// provider writes and for reading stored rows. Sized for `LiveRamp` envelopes +/// (about 700 bytes). +pub const MAX_STORED_UID_LENGTH: usize = 1024; /// How long a provider opt-out keeps that provider from being called again, /// in seconds. Matches the longest lifetime a provider UID may have. @@ -48,7 +56,7 @@ pub const OPT_OUT_MEMORY_SECONDS: u64 = 30 * 24 * 60 * 60; /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). -/// **KV value:** JSON-serialized `KvEntry` (max ~5KB). +/// **KV value:** JSON-serialized `KvEntry` (at most [`MAX_ENTRY_BYTES`]). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvEntry { /// Schema version — always [`SCHEMA_VERSION`]. @@ -112,7 +120,8 @@ pub struct KvGeo { pub dma: Option, } -/// Maximum serialized size of a KV entry body written by a page load. +/// Maximum serialized size of a KV entry body. Every writer refuses a larger +/// row; only the page-load planner trims entries to fit. pub const MAX_ENTRY_BYTES: usize = 16 * 1024; /// A partner ID record within a KV entry, keyed by source domain. @@ -196,27 +205,32 @@ impl KvPartnerId { if self.eids.is_empty() && !self.uid.is_empty() { return false; } - let before = self.clone(); + let mut changed = false; for eid in &mut self.eids { + let held = eid.uids.len(); eid.uids.retain(|uid| uid.is_live(now)); + changed |= eid.uids.len() != held; } self.eids.retain(|eid| !eid.uids.is_empty()); - self.uid = self + let first = self .eids .first() .and_then(|eid| eid.uids.first()) - .map(|uid| uid.id().as_str().to_owned()) - .unwrap_or_default(); - *self != before + .map_or("", |uid| uid.id().as_str()); + if self.uid != first { + self.uid = first.to_owned(); + changed = true; + } + changed } /// Drops entries left without UIDs and resyncs `uid`. /// /// Returns whether the record still holds a UID within - /// [`MAX_UID_LENGTH`]. + /// [`MAX_STORED_UID_LENGTH`]. fn retain_readable(&mut self) -> bool { if self.eids.is_empty() { - return self.uid.len() <= MAX_UID_LENGTH; + return self.uid.len() <= MAX_STORED_UID_LENGTH; } self.eids.retain(|eid| !eid.uids.is_empty()); self.uid = self @@ -280,7 +294,7 @@ struct StoredId<'a> { } /// Deserializes stored UIDs, skipping any whose `id` exceeds -/// [`MAX_UID_LENGTH`]. +/// [`MAX_STORED_UID_LENGTH`]. /// /// A row written under a higher limit then still loads; an entry left /// without UIDs is removed by [`KvEntry::drop_unreadable_ids`]. Any other @@ -292,7 +306,7 @@ fn deserialize_readable_uids<'de, D: Deserializer<'de>>( let mut uids = Vec::with_capacity(raw.len()); for value in &raw { let oversized = serde_json::from_str::>(value.get()) - .is_ok_and(|stored| stored.id.len() > MAX_UID_LENGTH); + .is_ok_and(|stored| stored.id.len() > MAX_STORED_UID_LENGTH); if oversized { continue; } @@ -628,7 +642,7 @@ impl KvEntry { /// Removes partner records holding no UID this build can serve. /// /// Lets this build load rows written by one with a higher - /// [`MAX_UID_LENGTH`] instead of rejecting the whole row. Returns the + /// [`MAX_STORED_UID_LENGTH`] instead of rejecting the whole row. Returns the /// number of records removed. pub fn drop_unreadable_ids(&mut self) -> usize { let before = self.ids.len(); @@ -779,9 +793,9 @@ impl KvEntry { } for (partner_id, partner_uid) in &self.ids { - if partner_uid.uid.len() > MAX_UID_LENGTH { + if partner_uid.uid.len() > MAX_STORED_UID_LENGTH { return Err(format!( - "partner ID '{partner_id}' exceeds MAX_UID_LENGTH ({})", + "partner ID '{partner_id}' exceeds MAX_STORED_UID_LENGTH ({})", partner_uid.uid.len() )); } @@ -1153,14 +1167,14 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - KvPartnerId::single("x".repeat(MAX_UID_LENGTH + 1)), + KvPartnerId::single("x".repeat(MAX_STORED_UID_LENGTH + 1)), ); let err = entry .validate() .expect_err("should reject oversized partner UIDs"); assert!( - err.contains("MAX_UID_LENGTH"), + err.contains("MAX_STORED_UID_LENGTH"), "should describe the UID length validation failure" ); } @@ -1719,7 +1733,7 @@ mod tests { let mut entry = KvEntry::minimal("ids.example.com", "kept", 1); entry.ids.insert( "long.example.com".to_owned(), - KvPartnerId::single("x".repeat(MAX_UID_LENGTH + 1)), + KvPartnerId::single("x".repeat(MAX_STORED_UID_LENGTH + 1)), ); let dropped = entry.drop_unreadable_ids(); @@ -1735,7 +1749,7 @@ mod tests { #[test] fn deserialize_drops_oversized_provider_uids() { - let long = "x".repeat(MAX_UID_LENGTH + 1); + let long = "x".repeat(MAX_STORED_UID_LENGTH + 1); let json = serde_json::json!({ "v": SCHEMA_VERSION, "created": 1, diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 84c3147393..8b608df948 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -118,9 +118,6 @@ pub enum EidSyncSource { /// Admitted `GET /_ts/page-bids` SPA navigation. #[display("page_bids")] PageBids, - /// Request that generated a new EC identity during finalization. - #[display("new_ec")] - NewEc, } /// Request-scoped view of one EC identity-graph lookup. @@ -634,6 +631,11 @@ impl EcContext { self.navigation_permit = PermitSlot(Some(permit)); } + /// Whether this request still holds its page-load write permit. + pub(crate) fn has_navigation_permit(&self) -> bool { + self.navigation_permit.0.is_some() + } + /// Takes the navigation permit; returns `None` after the first call. pub(crate) fn take_navigation_permit(&mut self) -> Option { self.navigation_permit.0.take() diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs index 5eddc555d5..f530127bcd 100644 --- a/crates/trusted-server-core/src/ec/page_load.rs +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -165,11 +165,6 @@ pub enum RowChange { /// Generation the row must still have. expected: Generation, }, - /// Replace a live row with a withdrawal tombstone. - Tombstone { - /// Generation the row must still have. - expected: Generation, - }, } /// Why the single write did not land. @@ -367,26 +362,4 @@ mod tests { "should report conflict" ); } - - #[test] - fn commit_tombstone_marks_consent_withdrawn() { - let graph = KvIdentityGraph::in_memory("page_load"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - graph - .create(&ec_id, &live_entry()) - .expect("should seed row"); - let expected = - Generation::of(&graph.load_snapshot(&ec_id), &ec_id).expect("should read generation"); - - let committed = graph - .commit_page_load(permit(), &ec_id, RowChange::Tombstone { expected }) - .expect("should write tombstone"); - - assert!(!committed.consent.ok, "should return tombstone"); - let (stored, _) = graph - .get(&ec_id) - .expect("should read") - .expect("should exist"); - assert!(!stored.consent.ok, "should persist tombstone"); - } } diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index e0e50a9841..9f479fa296 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -25,14 +25,16 @@ use crate::consent::ConsentContext; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::kv_types::{KvEid, KvEntry}; use crate::ec::registry::PartnerRegistry; -use crate::settings::{IdentityProviderConfig, ProviderSecrets}; +use crate::error::TrustedServerError; +use crate::settings::{Ec, IdentityProviderConfig, ProviderSecrets}; /// Builds lookup requests and parses lookup responses for one identity provider. /// /// Implementations must not perform I/O or read the clock; the coordinator /// executes requests and applies consent, transport, validation, expiry and -/// persistence policy. -pub trait IdentityProvider { +/// persistence policy. Coordinators are built once and shared for the +/// application's lifetime, so implementations are `Send + Sync`. +pub trait IdentityProvider: Send + Sync { /// Describes which sources and EID shapes this provider can return. fn capabilities(&self) -> &ProviderCapabilities; @@ -477,11 +479,12 @@ pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S /// Checks the configuration rules core owns for every enabled provider /// (spec §12.1): the endpoint is an `https` URL whose host is in -/// `allowed_domains`, every source has an `[[ec.partners]]` entry, and no -/// source is claimed by two enabled providers. +/// `allowed_domains`, it claims at least one source, every source has an +/// `[[ec.partners]]` entry, no source is claimed by two enabled providers, and +/// every enabled provider has a non-empty id of its own. /// -/// Unknown kinds and capability checks need the provider crate and are -/// enforced when the coordinator is built. +/// Unknown kinds and capability checks need the provider crate; the adapter +/// builds the coordinators at startup and fails to start on either. /// /// # Errors /// @@ -492,7 +495,18 @@ pub fn validate_enabled_configs<'a>( partners: &PartnerRegistry, ) -> Result<(), Report> { let mut claimed: HashSet = HashSet::new(); + let mut ids: HashSet<&str> = HashSet::new(); for config in configs.into_iter().filter(|config| config.enabled) { + // The id keys UID ownership, opt-outs and the rate-limit bucket in + // the row, so two providers sharing one would read, refresh and clear + // each other's IDs. + let id = config.id.trim(); + if id.is_empty() || !ids.insert(id) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "provider id `{}` is empty or used by another enabled provider", + config.id + ))); + } let endpoint = Url::parse(&config.endpoint) .change_context(ProviderConfigError::Invalid) .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; @@ -502,7 +516,12 @@ pub fn validate_enabled_configs<'a>( config.id ))); } - for source in parse_sources(config)? { + let sources = parse_sources(config)?; + if sources.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach(format!("provider `{}` claims no sources", config.id))); + } + for source in sources { if partners.find_by_source_domain(source.as_str()).is_none() { return Err(Report::new(ProviderConfigError::Invalid).attach(format!( "source `{source}` of provider `{}` has no [[ec.partners]] entry", @@ -520,6 +539,28 @@ pub fn validate_enabled_configs<'a>( Ok(()) } +/// Fails when `[ec]` enables an identity provider on an adapter that has no +/// provider implementations. +/// +/// Ignoring the provider would leave its sources open to browser ingest with +/// no sign that it never runs. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming the first enabled +/// provider. +pub fn reject_enabled_providers(ec: &Ec, adapter: &str) -> Result<(), Report> { + match ec.identity_providers.iter().find(|config| config.enabled) { + Some(config) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "identity provider `{}` is enabled, but the {adapter} adapter does not run identity providers", + config.id + ), + })), + None => Ok(()), + } +} + #[cfg(test)] mod tests { use super::*; @@ -751,6 +792,65 @@ mod tests { assert_invalid(&error); } + #[test] + fn validate_enabled_configs_rejects_a_provider_without_sources() { + let mut empty = config(&json!({})); + empty.sources = Vec::new(); + + let error = validate_enabled_configs([&empty], &partners()) + .expect_err("should reject a provider that claims no sources"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_shared_or_empty_id() { + let registry = PartnerRegistry::from_config(&[ + serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + serde_json::from_value(json!({ + "name": "other", "source_domain": "other.example.com" + })) + .expect("should build partner"), + ]) + .expect("should build registry"); + let first = config(&json!({})); + let mut shared = config(&json!({})); + shared.sources = vec!["other.example.com".to_owned()]; + let mut empty = config(&json!({})); + empty.id = " ".to_owned(); + + let error = validate_enabled_configs([&first, &shared], ®istry) + .expect_err("should reject a shared provider id"); + assert_invalid(&error); + let error = validate_enabled_configs([&empty], ®istry) + .expect_err("should reject an empty provider id"); + assert_invalid(&error); + } + + #[test] + fn reject_enabled_providers_fails_only_for_an_enabled_provider() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + let mut disabled = config(&json!({})); + disabled.enabled = false; + ec.identity_providers = vec![disabled]; + reject_enabled_providers(&ec, "Axum").expect("should accept a disabled provider"); + + ec.identity_providers.push(config(&json!({}))); + let error = + reject_enabled_providers(&ec, "Axum").expect_err("should reject an enabled provider"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::Configuration { .. } + ), + "should report a configuration error" + ); + } + #[test] fn validate_enabled_configs_ignores_disabled_providers() { let first = config(&json!({})); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 4fac813dd5..ead979b312 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -561,6 +561,7 @@ mod tests { pull_sync_rate_limit: 20, ts_pull_token: Some(Redacted::new("token".to_owned())), provider_owned: false, + owner_tcf_vendor_id: None, } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 435e164b80..c82f6c0bd7 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -5,6 +5,7 @@ //! lookup by source domain and API key hash. use std::collections::{HashMap, HashSet}; +use std::num::NonZeroU16; use error_stack::{Report, ResultExt as _}; @@ -50,6 +51,9 @@ pub struct PartnerConfig { /// Whether an enabled identity provider claims this source. Browser /// ingest skips it and batch sync rejects it. pub provider_owned: bool, + /// TCF vendor ID of the provider that owns this source, when configured. + /// Its UIDs leave the server only while that vendor has consent. + pub owner_tcf_vendor_id: Option, } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -210,10 +214,10 @@ impl PartnerRegistry { /// Builds the registry from `[ec]` settings without provider ownership. /// - /// For adapters with no identity-provider implementations: no source is - /// provider-owned, so browser-supplied IDs are ingested for every source. - /// Kind registration is unknown here, so no unregistered-kind warning is - /// logged; use [`Self::from_ec_with_kinds`] where kinds are known. + /// For adapters with no identity-provider implementations, which refuse + /// to start with an enabled provider: no source is provider-owned, so + /// browser-supplied IDs are ingested for every source. Use + /// [`Self::from_ec_with_kinds`] where kinds are known. /// /// # Errors /// @@ -226,12 +230,12 @@ impl PartnerRegistry { /// by an enabled identity provider whose kind is registered as /// provider-owned. /// - /// An enabled provider with an unregistered kind never runs, so its - /// sources stay open to browser ingest and a warning is logged. - /// /// # Errors /// - /// Returns the same errors as [`Self::from_config`]. + /// Returns the same errors as [`Self::from_config`], and + /// [`TrustedServerError::Configuration`] for an enabled provider whose + /// kind is not registered: it could never run, and its sources would + /// silently go back to browser ingest. pub fn from_ec_with_kinds( ec: &Ec, is_registered_kind: impl Fn(&str) -> bool, @@ -239,15 +243,16 @@ impl PartnerRegistry { let mut registry = Self::from_config(&ec.partners)?; for provider in ec.identity_providers.iter().filter(|p| p.enabled) { if !is_registered_kind(provider.kind()) { - log::warn!( - "Identity provider '{}' has unregistered kind '{}'; its sources stay browser-ingested", - provider.id, - provider.kind() - ); - continue; + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "identity provider `{}` has kind `{}`, which this build does not include", + provider.id, + provider.kind() + ), + })); } for source in &provider.sources { - registry.mark_provider_owned(source); + registry.mark_provider_owned(source, provider.tcf_vendor_id); } } Ok(registry) @@ -282,16 +287,18 @@ impl PartnerRegistry { self.get(domain) } - /// Marks `source` as owned by an enabled identity provider. + /// Marks `source` as owned by an enabled identity provider, recording the + /// provider's TCF vendor ID, if any. /// /// Returns `false` when the source has no `[[ec.partners]]` entry. - pub fn mark_provider_owned(&mut self, source: &str) -> bool { + pub fn mark_provider_owned(&mut self, source: &str, tcf_vendor_id: Option) -> bool { let Ok(normalized) = normalize_partner_source_domain(source) else { return false; }; match self.by_source_domain.get_mut(&normalized) { Some(partner) => { partner.provider_owned = true; + partner.owner_tcf_vendor_id = tcf_vendor_id; true } None => false, @@ -381,6 +388,7 @@ fn build_partner_config( pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), provider_owned: false, + owner_tcf_vendor_id: None, } } @@ -543,7 +551,7 @@ mod tests { } #[test] - fn from_ec_skips_providers_with_unregistered_kinds() { + fn from_ec_with_kinds_rejects_an_unregistered_kind() { let mut ec = crate::test_support::tests::create_test_settings().ec; ec.partners = vec![make_partner( "ids.example.com", @@ -556,13 +564,16 @@ mod tests { .expect("should parse enabled provider"), ]; - let with_kinds = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") - .expect("should build registry"); + let error = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect_err("should reject an enabled provider with an unregistered kind"); let plain = PartnerRegistry::from_ec(&ec).expect("should build registry"); assert!( - !with_kinds.is_provider_owned("ids.example.com"), - "should not claim sources for an unregistered kind" + matches!( + error.current_context(), + TrustedServerError::Configuration { .. } + ), + "should report a configuration error" ); assert!( !plain.is_provider_owned("ids.example.com"), @@ -715,7 +726,7 @@ mod tests { let mut registry = PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); - registry.mark_provider_owned("pull.example.com"); + registry.mark_provider_owned("pull.example.com", None); assert!( registry.pull_enabled_partners().is_empty(), diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 3b543e2a56..66d6f39b15 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -60,7 +60,16 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } }; - let mut entry = base.clone(); + // Prune first so an expired record frees its slot for browser ingest. + // Pruning alone never triggers a write; it rides along with a real change. + let mut entry = base; + for record in entry.ids.values_mut() { + record.prune_expired(input.now); + } + entry.prune_opt_outs(input.now); + entry.ids.retain(|_, record| !record.uid.is_empty()); + let pruned = entry.clone(); + let lookups_apply = input.lookups.is_for(input.ec_id); apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); let written = if lookups_apply { @@ -74,18 +83,13 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } Vec::new() }; - // Pruning alone never triggers a write; it rides along with a real change. - let changed = entry != base; - for record in entry.ids.values_mut() { - record.prune_expired(input.now); - } - entry.prune_opt_outs(input.now); - entry.ids.retain(|_, record| !record.uid.is_empty()); enforce_size_cap(&mut entry, &written); + // Compared after the cap, which may trim the only real change away. + let changed = entry != pruned; match expected { None => Some(RowChange::Create(entry)), - Some(_) if !changed || entry == base => None, + Some(_) if !changed => None, Some(expected) => Some(RowChange::Update { entry, expected }), } } @@ -236,7 +240,10 @@ mod tests { ]) .expect("should build registry"); for source in owned { - assert!(registry.mark_provider_owned(source), "should mark {source}"); + assert!( + registry.mark_provider_owned(source, None), + "should mark {source}" + ); } registry } @@ -651,6 +658,45 @@ mod tests { ); } + #[test] + fn browser_ingest_fills_a_slot_freed_by_an_expired_record() { + // The source is not provider-owned (its provider was disabled), so the + // browser may fill it once the expired provider record is pruned. + let registry = registry(&[]); + let ingest = [PartnerIdUpdate::new("ids.example.com", "browser-1")]; + let lookups = LookupResults::default(); + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + let Some(RowChange::Update { entry, .. }) = change else { + panic!("should plan an update that fills the freed slot"); + }; + assert_eq!( + entry + .ids + .get("ids.example.com") + .map(|record| record.uid.as_str()), + Some("browser-1"), + "should store the browser UID in place of the expired record" + ); + } + fn provider_entry(ids: &[&str]) -> KvEntry { let mut entry = live_entry(); let lookups = found("ids.example.com", ids); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 2a68d2ab8f..c02070a0ce 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3448,6 +3448,7 @@ fn request_head_snapshot(req: &Request) -> Request { struct EcSnapshotPreloadInput { is_navigation: bool, is_get: bool, + has_navigation_permit: bool, has_ec_id: bool, has_kv: bool, marker_valid: bool, @@ -3465,6 +3466,14 @@ fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { return false; } let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; + // Without a permit (no Fetch Metadata, or a prefetch) the navigation + // cannot write, so browser EID persistence no longer needs the row; the + // auction, a withdrawal and pull sync (unless the marker proves it + // complete) still do. + if !input.has_navigation_permit { + return eligible + && (input.auction_needs_row || input.privacy_needs_row || !input.marker_valid); + } let marker_can_skip = input.marker_valid && !input.auction_needs_row && !input.eid_cookie_may_need_persistence @@ -4963,9 +4972,15 @@ pub async fn handle_publisher_request( // explicit withdrawal `ec_allowed` is false, so auction dispatch forwards no // EC while the origin-overlapped snapshot read still happens for the active // ID, keeping the withdrawal CAS off the post-origin latency path. + // + // An EC minted on this request has no row until finalize creates it, and + // that create can still fail or be skipped (a redirect), so it is not sent + // to bidders until a later request proves it was issued. let active_ec_id_owned = ec_context.ec_value().map(str::to_owned); let active_ec_id = active_ec_id_owned.as_deref(); - let ec_id_owned = active_ec_id_owned.clone().filter(|_| ec_allowed); + let ec_id_owned = active_ec_id_owned + .clone() + .filter(|_| ec_allowed && !ec_context.ec_generated()); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; if let Some(registry) = auction.registry { @@ -5305,6 +5320,7 @@ pub async fn handle_publisher_request( let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { is_navigation, is_get, + has_navigation_permit: ec_context.has_navigation_permit(), has_ec_id: active_ec_id.is_some(), has_kv: kv.is_some(), marker_valid: ec_context.pull_sync_marker().is_valid(), @@ -5362,18 +5378,9 @@ pub async fn handle_publisher_request( ); } if should_preload_ec && let (Some(graph), Some(active_ec_id)) = (kv, active_ec_id) { - let refreshed = graph.load_snapshot(active_ec_id); - // Never downgrade an in-request Add-confirmed Present snapshot: a - // freshly created row can read back Missing/Failed on an - // eventually-consistent store, and rotating or suppressing that - // just-generated identity would fragment it. Adopt the refresh only - // when it keeps or upgrades to a Present row (the intended - // generation-refresh) — otherwise retain the confirmed entry. - let keep_present = ec_context.kv_snapshot().entry_for(active_ec_id).is_some() - && refreshed.entry_for(active_ec_id).is_none(); - if !keep_present { - ec_context.set_kv_snapshot(refreshed); - } + // A new EC's row is created only in finalize, so no snapshot read + // before this point can be an in-request confirmed row to protect. + ec_context.set_kv_snapshot(graph.load_snapshot(active_ec_id)); } // Dispatch SSP bid requests while req still has the original client headers @@ -8396,6 +8403,7 @@ mod tests { let baseline = EcSnapshotPreloadInput { is_navigation: true, is_get: true, + has_navigation_permit: true, has_ec_id: true, has_kv: true, marker_valid: false, @@ -8430,11 +8438,54 @@ mod tests { ); } + #[test] + fn ec_snapshot_preload_without_a_permit_ignores_browser_eid_persistence() { + let no_permit = EcSnapshotPreloadInput { + is_navigation: true, + is_get: true, + has_navigation_permit: false, + has_ec_id: true, + has_kv: true, + marker_valid: true, + auction_needs_row: false, + eid_cookie_may_need_persistence: true, + privacy_needs_row: false, + snapshot_already_read: false, + }; + + assert!( + !should_preload_ec_snapshot(&no_permit), + "should not read a row the navigation can never write" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + auction_needs_row: true, + ..no_permit + }), + "should read the row for the auction's stored EIDs" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + privacy_needs_row: true, + ..no_permit + }), + "should read the row for a withdrawal" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + marker_valid: false, + ..no_permit + }), + "should read the row for pull sync while the marker is not valid" + ); + } + #[test] fn valid_marker_skips_only_when_no_other_consumer_needs_snapshot() { let marker_only = EcSnapshotPreloadInput { is_navigation: true, is_get: true, + has_navigation_permit: true, has_ec_id: true, has_kv: true, marker_valid: true, @@ -8706,6 +8757,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert!( ec_context.ec_allowed() && ec_context.ec_value().is_some(), "test precondition: an active, consent-allowed EC must exist" @@ -8916,6 +8968,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), ); @@ -13677,6 +13730,8 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); let response = handle_publisher_request( &settings, &services, @@ -16307,6 +16362,74 @@ mod tests { } } + #[tokio::test] + async fn publisher_auction_withholds_an_ec_minted_on_this_request() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings).auction.providers = + crate::auction_config_types::AuctionConfig::legacy_provider_map(&[ + SCHEDULING_PROVIDER, + ]); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let graph = KvIdentityGraph::in_memory("minted-ec-store"); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let partners = PartnerRegistry::empty(); + queue_shareable_html(&stub); + let minted = format!("{}.Mint01", "c".repeat(64)); + let mut ec_context = EcContext::new_for_test(None, scheduling_consent()); + ec_context.replace_with_generated( + minted.clone(), + crate::ec::EcKvSnapshot::Missing { + ec_id: minted.clone(), + }, + ); + let captured = Arc::new(Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { + captured: Arc::clone(&captured), + http: Arc::clone(&stub), + lookups: Arc::new(AtomicUsize::new(0)), + })); + let orchestrator = Arc::new(orchestrator); + + let response = handle_publisher_request( + &settings, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: Some(&partners), + }, + cookie_policy_request(&[b"ab_bucket=A"]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should serve a first visit"); + let response = finalize_test_publisher_response( + response, + &settings, + &services, + ®istry, + orchestrator, + Finalizer::Buffered, + ) + .await; + let body = body_of(response).await; + + assert!(!body.is_empty(), "should render the page"); + let captured = captured.lock().expect("should lock captured auction"); + let auction = captured.as_ref().expect("should dispatch the auction"); + assert_eq!( + auction.request.user.id, None, + "should not send an EC whose row does not exist yet" + ); + } + #[tokio::test] async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { @@ -20070,11 +20193,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } fn active_ec_context() -> EcContext { - EcContext::new_for_test_with_ip( + let mut ec_context = EcContext::new_for_test_with_ip( Some(format!("{}.CkId01", "b".repeat(64))), scheduling_consent(), Some("203.0.113.7".to_owned()), - ) + ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); + ec_context } #[tokio::test] diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 148d774624..f623aa1852 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -608,7 +608,8 @@ pub struct IdentityProviderConfig { /// When the provider's lookup runs on a navigation. #[serde(default)] pub collect: CollectTiming, - /// Most lookups per EC per hour, when set; unset means unlimited. + /// Most lookups per EC hash per hour, when set; unset means unlimited. + /// ECs behind one client IP share the budget. /// /// The Fastly limiter enforces this as a per-minute budget of at least /// one, so values below 60 behave like 60 per hour. @@ -703,10 +704,20 @@ pub struct Ec { /// Shared time budget, in milliseconds, for all provider lookups on one /// navigation. + /// + /// `Ec` denies unknown fields, so this and the other identity fields are + /// skipped from serialization at their defaults: a config blob without + /// providers then still loads in a runtime built before them. #[serde( default = "Ec::default_identity_budget_ms", + skip_serializing_if = "Ec::is_default_identity_budget_ms", deserialize_with = "from_value_or_str" )] + #[validate(range( + min = 1, + max = 1000, + message = "must be between 1 and 1000 milliseconds" + ))] pub identity_budget_ms: u64, /// Time budget, in milliseconds, for lookups of providers with @@ -714,12 +725,22 @@ pub struct Ec { /// response has been sent. #[serde( default = "Ec::default_identity_after_response_budget_ms", + skip_serializing_if = "Ec::is_default_identity_after_response_budget_ms", deserialize_with = "from_value_or_str" )] + #[validate(range( + min = 1, + max = 5000, + message = "must be between 1 and 5000 milliseconds" + ))] pub identity_after_response_budget_ms: u64, /// Server-side identity providers. - #[serde(default, deserialize_with = "vec_from_seq_or_map")] + #[serde( + default, + skip_serializing_if = "Vec::is_empty", + deserialize_with = "vec_from_seq_or_map" + )] pub identity_providers: Vec, } @@ -763,6 +784,16 @@ impl Ec { 2000 } + #[allow(clippy::trivially_copy_pass_by_ref)] + fn is_default_identity_budget_ms(value: &u64) -> bool { + *value == Self::default_identity_budget_ms() + } + + #[allow(clippy::trivially_copy_pass_by_ref)] + fn is_default_identity_after_response_budget_ms(value: &u64) -> bool { + *value == Self::default_identity_after_response_budget_ms() + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -4422,6 +4453,53 @@ mod tests { ); } + /// `Ec` denies unknown fields, so a binary built before identity providers + /// existed must still accept a blob that configures none. + #[test] + fn default_identity_settings_are_omitted_for_rollback() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + + let serialized = serde_json::to_value(&settings.ec).expect("should serialize ec settings"); + + for field in [ + "identity_budget_ms", + "identity_after_response_budget_ms", + "identity_providers", + ] { + assert!( + serialized.get(field).is_none(), + "should omit default {field} for older binaries: {serialized}" + ); + } + } + + #[test] + fn identity_budgets_are_range_checked() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + for (page, after_response, invalid) in [ + (1, 1, false), + (1000, 5000, false), + (0, 2000, true), + (1001, 2000, true), + (200, 0, true), + (200, 5001, true), + ] { + let ec = Ec { + identity_budget_ms: page, + identity_after_response_budget_ms: after_response, + ..settings.ec.clone() + }; + + assert_eq!( + ec.validate().is_err(), + invalid, + "should enforce identity budget bounds for {page} and {after_response}" + ); + } + } + /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. /// That only holds while the flag is skipped during serialization. diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37d54614c0..187b22a276 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -7,6 +7,35 @@ pub mod tests { use crate::consent::types::{ConsentContext, ConsentSource, TcfConsent}; use crate::settings::Settings; + /// GDPR consent with TCF Purposes 1 and 4 and the given vendor consents. + #[must_use] + pub fn consent_granting_vendors(vendor_consents: Vec) -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents, + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + #[must_use] pub fn crate_test_settings_str() -> String { r#" diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 831ed20729..1ff50dedf8 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -15,7 +15,6 @@ workspace = true [dependencies] error-stack = { workspace = true } -log = { workspace = true } trusted-server-core = { workspace = true } [dev-dependencies] diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 17f083be0f..23197150ff 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -29,12 +29,13 @@ pub fn is_registered_kind(_kind: &str) -> bool { false } -/// Builds the partner registry, marking only sources claimed by enabled -/// providers whose kind is registered as provider-owned. +/// Builds the partner registry, marking the sources claimed by enabled +/// providers as provider-owned. /// /// # Errors /// -/// Returns the same errors as [`PartnerRegistry::from_ec`]. +/// Returns the same errors as [`PartnerRegistry::from_ec_with_kinds`], +/// including for an enabled provider whose kind is not registered. pub fn partner_registry(ec: &Ec) -> Result> { PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) } @@ -77,9 +78,6 @@ pub fn build_coordinator( /// Builds the page-phase coordinator against an already built partner /// registry. /// -/// Enabled providers with an unregistered kind are skipped; building the -/// registry with [`partner_registry`] already warned about them. -/// /// # Errors /// /// Returns [`ProviderConfigError`] when a provider cannot be built or fails @@ -131,17 +129,10 @@ fn build_phase_coordinator( .ec .identity_providers .iter() - .filter(|config| { - config.enabled && config.collect == timing && is_registered_kind(config.kind()) - }) + .filter(|config| config.enabled && config.collect == timing) .map(|config| { - build(config, &config.secrets).map(|provider| { - let registered = RegisteredProvider::new(config.clone(), provider); - match config.rate_limit_per_hour { - Some(per_hour) => registered.with_rate_limit(per_hour), - None => registered, - } - }) + build(config, &config.secrets) + .map(|provider| RegisteredProvider::new(config.clone(), provider)) }) .collect::, Report>>()?; if registered.is_empty() { @@ -208,7 +199,7 @@ mod tests { } #[test] - fn enabled_unknown_kind_is_skipped_and_claims_no_sources() { + fn enabled_unknown_kind_fails_the_build() { let mut settings = create_test_settings(); settings.ec.identity_providers = vec![ toml::from_str( @@ -217,14 +208,11 @@ mod tests { .expect("should parse"), ]; - let coordinator = - build_coordinator(&settings).expect("should skip unregistered provider kinds"); - let registry = partner_registry(&settings.ec).expect("should build registry"); + let result = build_coordinator(&settings); - assert!(coordinator.is_none(), "should build no coordinator"); assert!( - !registry.is_provider_owned("ids.example.com"), - "should leave unregistered provider sources to browser ingest" + result.is_err(), + "should fail instead of skipping an enabled provider of an unknown kind" ); } diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 99059adba0..1ddd0ab4ae 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -363,8 +363,10 @@ Server-to-server batch sync endpoint for writing EC ID to partner UID mappings. **Contract:** The JSON body is limited to 2 MiB and 1,000 mappings. The per-partner, per-minute limit is `ec.partners[].batch_rate_limit`. Responses are `200` when every mapping is accepted or `207` when at least one is rejected; -item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, or -`kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; +item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, +`provider_owned` (the source belongs to a server-side identity provider), +`row_full` (the EC's row has no room for the UID; retrying gives the same +result), or `kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; rate exhaustion returns `429`; an oversized body returns `413`; too many mappings returns `400`. The endpoint emits JSON but defines no dedicated cache or CORS headers. It is Fastly-only and requires the EC KV store. diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index fb02fe2e68..c4b2c5c2b4 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -172,7 +172,8 @@ Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin currently accept at most one provider in an enabled auction. Every adapter runs target-aware fan-out and backend-name checks at startup. Auction collection uses an absolute cutoff for network waiting across headers and complete response bodies. -A nonblocking snapshot retains complete results after a slow origin. Fastly +A final snapshot retains complete results after a slow origin (on Fastly it may +wait up to 1 ms per partially received body). Fastly abandons pending backend requests because its SDK cannot cancel them; other adapters cancel their owned operations at cutoff. See [Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index c377157ce5..cd40f2a94d 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -828,15 +828,15 @@ Until that callback lands, startup remains the mandatory target-aware gate. Auction requests start at dispatch and progress while the publisher origin loads. Response collection waits for headers and complete upstream bodies only until the -original absolute deadline. At expiry, one nonblocking snapshot preserves complete -responses already available; unfinished providers become timeouts. A slow origin +original absolute deadline. At expiry, one final snapshot preserves complete +responses already available; unfinished providers become timeouts. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and keeps a response whose headers arrived by the cutoff. A slow origin never restarts the budget or causes an already-buffered bid to be discarded. Fastly has no upstream arrival timestamp, so this is a bound on network waiting, not bid eligibility based on an exact server-side arrival time. A mediator starts only with remaining budget and uses the earlier of the auction cutoff and its own logical timeout. -Provider bids that are already complete at the final nonblocking snapshot remain +Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 50fa54276f..acb84abaa8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -757,8 +757,8 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. | `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | | `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | | `partners` | Array | No | Static partner registry entries | -| `identity_budget_ms` | Integer | No | Shared time budget for identity provider lookups (default 200) | -| `identity_after_response_budget_ms` | Integer | No | Time budget for `collect = "after_response"` lookups (default 2000) | +| `identity_budget_ms` | Integer | No | Shared identity lookup budget, 1–1000 ms (default 200) | +| `identity_after_response_budget_ms` | Integer | No | After-response lookup budget, 1–5000 ms (default 2000) | | `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying @@ -800,9 +800,20 @@ share the `identity_budget_ms` budget, and are collected before the response is finalized. Results go into the navigation's single EC row write. A lookup that times out or fails leaves the row unchanged. -Lookups run only for an existing or newly issued EC, only when consent allows -EID persistence, and, when `tcf_vendor_id` is set and TCF applies to the -request, only with TCF vendor consent for that vendor. +Lookups run only for an EC issued on an earlier request (an EC minted on this +navigation is looked up on the next one), only when consent allows EID +persistence, and, when `tcf_vendor_id` is set and TCF applies to the request, +only with TCF vendor consent for that vendor. The same vendor consent gates +egress: while it is withheld, the provider's stored UIDs are left out of +auction `user.eids`, `x-ts-eids` and `/_ts/api/v1/identify`. Without +`tcf_vendor_id`, a provider is gated by the EID-persistence purposes alone, so +set it for any provider that has a TCF vendor ID. + +Identity providers run only on Fastly. The Axum, Cloudflare and Spin adapters +refuse to start when an enabled provider is configured. On Fastly, an enabled +provider whose `kind` this build does not include, or whose `settings` its kind +rejects, also fails startup. Lookups always verify the provider's TLS +certificate, whatever `proxy.certificate_check` says. A provider owns its `sources`: its results replace whatever the row holds for them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs @@ -816,14 +827,14 @@ again until the 30 days pass. | Field | Type | Required | Description | | --------------------- | ------- | -------- | ----------------------------------------------------------------------- | | `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | -| `kind` | String | Yes | Provider implementation built into this release | +| `kind` | String | Yes | Provider implementation; must be one this build includes | | `enabled` | Boolean | No | Whether the provider runs (default `true`) | | `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | | `allowed_domains` | Array | Yes | Hosts the provider may call | | `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | | `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | | `collect` | String | No | `"page"` (default) or `"after_response"`; see below | -| `rate_limit_per_hour` | Integer | No | Most lookups per EC per hour; unset means unlimited. See the note below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | | `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | | `settings` | Table | No | Provider-specific settings, parsed by the provider kind | @@ -833,6 +844,10 @@ again until the 30 days pass. [ec] identity_budget_ms = 200 +[[ec.partners]] +name = "Example IDs" +source_domain = "ids.example.com" + [[ec.identity_providers]] id = "example-provider" kind = "example-kind" @@ -845,19 +860,33 @@ token = "example_provider_token" ``` `collect = "after_response"` lookups run after the response is sent, within -`identity_after_response_budget_ms`, and only for an EC whose row already -existed before the navigation. Their results reach the row in a follow-up write. +`identity_after_response_budget_ms`, and only when `ec_store` is configured, +the response is not a redirect, the navigation's own write succeeded (or had +nothing to change), and the EC was not minted on this navigation. Their results +reach the row in a follow-up write. ::: warning Rate limit granularity -On Fastly, `rate_limit_per_hour` is enforced as a per-minute budget rounded up -to at least one lookup per minute, so any value below 60 behaves like 60 per -hour. Values of 60 and above are enforced as `ceil(value / 60)` per minute. +`rate_limit_per_hour` is counted per EC hash, which is derived from the client +IP, so ECs behind one IP (a NAT or carrier gateway) share the budget. On +Fastly it is enforced as a per-minute budget rounded up to at least one lookup +per minute, so any value below 60 behaves like 60 per hour. Values of 60 and +above are enforced as `ceil(value / 60)` per minute. ::: **Validation**: configuration validation fails when an enabled provider's -`endpoint` is not HTTPS or its host is not in `allowed_domains`, when a source -has no `[[ec.partners]]` entry, or when two enabled providers claim the same -source. Deploy validation also rejects an empty `secrets.token` key name. +`endpoint` is not HTTPS or its host is not in `allowed_domains`, when it claims +no sources, when a source has no `[[ec.partners]]` entry, when two enabled +providers claim the same source, or when its `id` is empty or shared with +another enabled provider. `identity_budget_ms` must be 1–1000 and +`identity_after_response_budget_ms` 1–5000. Deploy validation also rejects an +empty `secrets.token` key name. + +::: warning Rollback +A config blob that configures identity providers, or sets either identity +budget to a non-default value, cannot be loaded by a build from before +identity providers. Roll the runtime back only with a blob that leaves them +out. +::: ### Field Details @@ -2519,11 +2548,11 @@ push as proof that a Cloudflare or Spin multi-provider plan can start. ### Deadline behavior Auction response collection waits only until an absolute deadline measured from -dispatch, including headers and complete response bodies. One nonblocking snapshot +dispatch, including headers and complete response bodies. One final snapshot retains complete available responses after a slow origin; unfinished providers become timeouts and mediation cannot start with an exhausted budget. Backend -timeout rounding does not extend network waiting. -Provider bids that are already complete at the final nonblocking snapshot remain +timeout rounding does not extend network waiting. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and keeps a response whose headers arrived by the cutoff. +Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 527268a57f..a3e8176a97 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -42,7 +42,7 @@ maps to a stable base. ### Request Lifecycle -Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. +Every request passes through four phases. A new EC is minted, and its row created, only on a top-level publisher navigation: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` token in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, and a prefetch or prerender (`Sec-Purpose: prefetch;prerender`) never mints or writes. Integration proxies and `/auction` use an existing EC but never mint one, and read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. ```mermaid sequenceDiagram @@ -133,7 +133,7 @@ Partner identities flow into the KV identity graph through three channels. Each ```mermaid flowchart LR subgraph Browser-initiated - Prebid["Prebid EIDs
/auction body + ts-eids + sharedId
Response-time ingestion"] + Prebid["Prebid EIDs
ts-eids + sharedId
Navigation-time ingestion"] end subgraph Server-initiated @@ -148,7 +148,7 @@ flowchart LR ### Prebid EID Flow -Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a top-level document navigation, the only request that writes KV. It collects matched partner UIDs from two request-local sources, applied in this order so that, when both carry the same partner, the later source's UID is the candidate offered to KV: +Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a top-level document navigation, the only request that writes browser-resolved IDs to KV. It collects matched partner UIDs from two request-local sources, applied in this order so that, when both carry the same partner, the later source's UID is the candidate offered to KV: 1. the `ts-eids` cookie, 2. the `sharedId` cookie. @@ -184,7 +184,7 @@ The `sharedId` cookie follows a similar path but is written directly by Prebid's Cookie persistence runs only on top-level document navigations (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`). `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. -Each navigation attempts at most one conditional write. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. +Each navigation makes at most one conditional page-load write, which carries browser ingest and page-phase identity-provider results together. If another writer wins, the navigation's changes are dropped without a re-read; a later navigation offers them again. Two writes sit outside that rule: a consent withdrawal's tombstone is written on any request that carries the EC cookie, with bounded retries, and `collect = "after_response"` identity providers make one follow-up write after the response is sent. Batch sync and pull sync write through their own channels. Browser cookies do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. @@ -262,7 +262,7 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner } ``` -Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). +Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs from the `ts-eids` or `sharedId` cookie only fill partner UIDs that are missing from KV (the `/auction` request body goes to bidders only and is never stored); a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). ### Pull-Sync Completeness Marker diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 3b314f5bec..1f64f99e98 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -865,12 +865,10 @@ page or auction. When available, the opaque value follows the standard path: 2. The current `/auction` request includes that entry. 3. Trusted Server merges and consent-gates it, then forwards it to Prebid Server as `user.ext.eids`. -4. After the response, the same `/auction` request ingests the body entry into - an EC/KV partner configured with `source_domain = "liveramp.com"`, when that - partner UID is not already stored. -5. The browser also persists the opaque value in the bounded `ts-eids` cookie, - which lets requests without an EID body, such as `GET /_ts/page-bids` and - page navigations, ingest it. +4. The browser also persists the opaque value in the bounded `ts-eids` cookie. + The next top-level page navigation ingests it into an EC/KV partner + configured with `source_domain = "liveramp.com"`, when that partner UID is + not already stored. `/auction` and `GET /_ts/page-bids` never write KV. Trusted Server treats the RampID envelope as an opaque string. Do not log, decode, publish, or dimension metrics by the value. Source names, counts, @@ -902,20 +900,20 @@ server-to-server ATS API. ### Degraded behavior -| Condition | Result | -| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | -| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | -| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | -| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | -| LiveRamp network resolution fails | The current auction continues without RampID | -| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | -| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | -| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | -| The resolved envelope exceeds the 1024-byte EID value cap | The envelope is dropped from both the `/auction` payload and EC persistence; the auction continues | -| The CMP is callable but never returns a settled result | The managed entry is never seeded; no vendor call, no identity storage, and the auction continues | -| Two managed entries address one Prebid submodule | The later entry is dropped with a logged error; the first entry's configuration takes effect | -| A silent CMP stub is replaced by a working CMP | The shim re-subscribes to the replacement; a CMP that already answered once keeps its own wait | +| Condition | Result | +| -------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | +| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | +| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | +| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | +| LiveRamp network resolution fails | The current auction continues without RampID | +| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | +| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | +| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | +| The resolved envelope exceeds the 512-byte EID value cap | The envelope is dropped from both the `/auction` payload and EC persistence; the auction continues | +| The CMP is callable but never returns a settled result | The managed entry is never seeded; no vendor call, no identity storage, and the auction continues | +| Two managed entries address one Prebid submodule | The later entry is dropped with a logged error; the first entry's configuration takes effect | +| A silent CMP stub is replaced by a working CMP | The shim re-subscribes to the replacement; a CMP that already answered once keeps its own wait | The TCF rows assume either the managed-ID automatic setup described above or a publisher-owned Prebid GDPR configuration. A CMP API and its policy remain @@ -932,7 +930,7 @@ domain, booleans, source names, counts, and status codes: `identityLink` entry with the test Placement ID. 2. With positive consent, confirm `idl_env` is created or refreshed. 3. Record the byte length of the resolved envelope — the length only, never the - value — and confirm it is at or below the 1024-byte EID value cap. Anything + value — and confirm it is at or below the 512-byte EID value cap. Anything above it is dropped as described in the degraded-behavior table. 4. Confirm `pbjs.getUserIdsAsEids()` reports source `liveramp.com` without recording its value. @@ -958,9 +956,9 @@ Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions 2. **Server-side EIDs from the EC/KV identity graph** are resolved on the edge from the current EC ID. 3. Trusted Server **merges and deduplicates** both sets before calling Prebid Server. 4. The merged result is forwarded downstream as `user.ext.eids` in the OpenRTB request. -5. After the response, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `/auction` request-body EIDs, then the `sharedId` cookie. Because the body carries the untrimmed EID set, `/auction` ingestion is not limited by the size-capped `ts-eids` cookie. +5. On the next top-level page navigation, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `sharedId` cookie. The `/auction` request-body EIDs go to bidders on that request only and are never stored, so ingestion is limited by the size-capped `ts-eids` cookie. -The `ts-eids` cookie still matters for requests without an EID body, such as `GET /_ts/page-bids` and page navigations, where it is the source for both EID fallback and identity-graph ingestion. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. +The `ts-eids` cookie also supplies the EID fallback for requests without an EID body, such as `GET /_ts/page-bids`. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. This means Prebid auctions get same-request transparency for browser-resolved IDs without giving up the durability of the server-managed EC identity graph. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 6418f21eaf..31a608ec38 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -86,6 +86,12 @@ passphrase = "ec_passphrase" ec_store = "ec_identity_store" # Max concurrent partner pull-sync requests. pull_sync_concurrency = 3 +# Optional identity-provider budgets (see [[ec.identity_providers]] below). +# Every page-phase provider shares one budget (1-1000 ms): +# identity_budget_ms = 200 +# Budget for providers with collect = "after_response", which run after the +# response is sent and write their results in a follow-up write (1-5000 ms): +# identity_after_response_budget_ms = 2000 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds @@ -107,31 +113,29 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false -# Optional server-side identity providers, looked up on publisher navigations -# before the origin fetch. Every enabled provider shares one time budget: -# identity_budget_ms = 200 # default 200 -# Budget for providers with collect = "after_response", which run after the -# response is sent and write their results in a follow-up write: -# identity_after_response_budget_ms = 2000 +# Optional server-side identity providers (Fastly only), looked up on publisher +# navigations before the origin fetch. Their budgets are set in [ec] above. An +# enabled provider whose kind this build does not include fails startup. # # [[ec.identity_providers]] # id = "example-provider" -# Must name a provider kind built into this release. # kind = "example-kind" # enabled = false # endpoint = "https://ids.example.com/v1/lookup" # The endpoint host must be listed here. # allowed_domains = ["ids.example.com"] -# Each source needs an [[ec.partners]] entry and must be claimed by only one -# enabled provider. +# Each source needs an [[ec.partners]] entry (for example one with +# source_domain = "ids.example.com") and must be claimed by only one enabled +# provider. # sources = ["ids.example.com"] -# Optional: require TCF vendor consent for this provider. +# Optional: require TCF vendor consent for lookups and for sending this +# provider's stored IDs to bidders. # tcf_vendor_id = 9999 # "page" (default) collects before the response; "after_response" collects -# after it is sent, only for an EC whose row already exists. +# after it is sent, only for an EC issued on an earlier request. # collect = "page" -# Optional: most lookups per EC per hour. On Fastly this is enforced per -# minute with a minimum of one, so values below 60 behave like 60. +# Optional: most lookups per EC hash (client IP) per hour. On Fastly this is +# enforced per minute with a minimum of one, so values below 60 behave like 60. # rate_limit_per_hour = 60 # Secret Store key name, resolved like other secret references. # [ec.identity_providers.secrets] From f844a52a098eb62aae1ee11ff5883a288f68cb54 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 10 Nov 2026 13:21:27 +0530 Subject: [PATCH 40/58] Add design spec for server-side identity provider EID collection Signed-off-by: dhruv8sh --- .../2026-10-30-s2s-eid-collection-design.md | 1622 +++++++++++++++++ 1 file changed, 1622 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md new file mode 100644 index 0000000000..d5b59cb591 --- /dev/null +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -0,0 +1,1622 @@ +# Server-to-Server EID Collection — Design Spec + +**Date:** 2026-10-30 +**Status:** Draft for Tech Lead review +**Source of intent:** `S2S_EID_Collection_Requirements.md` (sections referenced +as "R§n") + +## 1. Purpose + +Trusted Server (TS) collects external IDs (EIDs) only through the browser and +one fixed pull-sync protocol, and a single page load can write the EC KV row +several times. This spec defines: + +- a Rust trait that any identity provider implements so TS can collect IDs + server-to-server, with implementations isolated in a dedicated crate; +- one shared coordinator that owns eligibility, consent, transport, time + budget, validation, merging, persistence and the mirror cookie; +- a strict rule of **at most one KV write per page load**, made on the top-level + document navigation and never retried; an opt-in after-response phase + (§5.2.1) may add one guarded follow-up update with provider results only; +- a full-fidelity, ownership- and expiry-aware KV representation for + provider-written IDs, and how auctions and other consumers read it. + +Client-side collection (`ts-eids`, the `/auction` body) keeps working alongside +server-side collection, and both feed the same consent-gated KV row. + +## 2. Scope + +### 2.1 In scope + +- `IdentityProvider` trait, its input/output types and capability description. +- Dedicated provider crate boundary and registration. +- Identity coordinator: provider selection, consent gating, allowlisted + outbound HTTP, shared time budget, validation, merge, single write. +- Page-load write permit: navigation-only, single-attempt KV writes. +- Additive KV schema for multiple EIDs/UIDs with EID-level and UID-level + fields, owner and expiry. +- `__Host-ts-eids-mirror` first-party cookie. +- Consumer changes: `/auction`, `/_ts/page-bids`, `x-ts-eids`, + `/_ts/api/v1/identify`, browser ingest, batch sync on provider-owned sources. +- Re-expressing the existing pull sync as the first provider on the trait. +- Operator configuration, secrets, logs and metrics. + +### 2.2 Non-goals (firm, from R§6) + +- Logged-in identity and email contribution: no hashed-email intake, no + `ts-hem` cookie, no `tsjs.setEmail()`, no publisher sign-in changes. The + trait carries a typed `HashedEmail` slot that is always empty in V1; a source + that requires it is skipped before any call. TS never receives a raw email + from the page. +- Proxying vendor identity JavaScript through TS. +- Concrete vendor implementations (each gets its own spec, see §13.2 F1). +- Adapters other than Fastly. KV identity remains Fastly-only; Axum, + Cloudflare and Spin get this design when EC identity is ported to them. +- Cross-browser linking. +- Vendor segment products and ad-server targeting. +- Batch sync and admin write paths, except how batch sync treats a + provider-owned source (§8.5). +- Removing `ts-eids` or `x-ts-eids`. +- Storing the browser's complete EID set. Browser ingest keeps today's rules + (§8.4); full fidelity applies only to provider-written IDs. +- Unregistered sources for providers. A provider claims only sources that have + an `[[ec.partners]]` entry. + +### 2.3 Future tasks (R§5, not in this scope) + +Listed in §13.2 in their original order, stacked on `main`, each a Tech Lead +decision. + +## 3. Agreed decisions + +These were agreed before drafting and are binding on the rest of the spec. + +| # | Topic | Decision | +| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | +| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | +| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | +| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | +| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | +| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of `bidstream_enabled` sources only. Never trusted by the server over KV. Filled deterministically up to its size limit. | +| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | +| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | +| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | +| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | +| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | +| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | +| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | +| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | + +§5.2.1 (opt-in `collect = "after_response"`) and §5.6 (refresh state) extend +D2 and D3 additively; defaults are unchanged. + +### 3.1 Choices this spec makes to apply those decisions + +Reviewers should check these; each follows from a decision above. + +- **Navigation test (D1).** A request is a page-load navigation when it is a + `GET` on the publisher-proxy route with `Sec-Fetch-Dest: document` and + `Sec-Fetch-Mode: navigate`, and its `Sec-Purpose` header, if any, has no + `prefetch` token. Requests without Fetch Metadata never write. A navigation + whose final response is a redirect (3xx) does not write; the redirect target + is the page load. This is stricter than the existing `is_navigation_request` + (which falls back to `Accept: text/html`); that helper keeps its other uses, + and the write permit uses the stricter test. +- **Integration-proxy EC minting stops (D1).** Integration-proxy requests can no + longer create a KV row, so they no longer mint a new EC. Today a new EC is + only issued when its row exists, so this keeps that invariant. +- **Withdrawal is the one exception to the one-write rule (D1, D9).** Any + finalized request that carries a consent withdrawal and the EC cookie writes + the tombstone, navigation or not, keeping today's behaviour. It tombstones the + cookie's EC ID and the active EC ID (when they differ), using a bounded CAS + retry and, when a store listing proves the key exists, an unconditional + tombstone. It serves no IDs and expires the mirror cookie, and it expires the + EC cookie only when every tombstone succeeded, so a failed write is retried on + the next request instead of orphaning the row. Deferring the tombstone to the + next navigation would leave a withdrawn visitor's row live for as long as they + only make non-navigation requests, which privacy does not allow. +- **One owner per source (D8).** Config validation rejects two enabled providers + claiming the same source, so "the owning provider" of a source is always + unambiguous. +- **No retry loops on the navigation write (D4).** Existing loops on this path + (EC-ID collision on create, orphan recovery) become a single attempt. + +## 4. Terms + +| Term | Meaning | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| page load | One top-level document navigation and every request the page makes because of it: `/auction`, `/_ts/page-bids` (SPA view changes), integration proxies. | +| navigation | The request that starts a page load (§3.1). | +| row | The EC KV identity-graph entry keyed by EC ID ([`KvEntry`]). | +| source | An EID source domain registered in `[[ec.partners]]`. | +| provider | A configured `IdentityProvider` instance with an operator-chosen `ProviderId`. | +| owner | The provider that wrote a UID. Browser and batch UIDs have no owner. | +| provider-owned | A source claimed by an enabled provider. A source whose owner is disabled is not provider-owned; its owned UIDs serve until they expire or are pruned. | +| due source | A source an enabled provider claims whose owned UIDs are missing or all expired, or one of whose live owned UIDs has refresh state with `from <= now` (§5.6). | +| live UID | A stored UID with no `exp`, or with `exp` greater than now. | +| mirror | The `__Host-ts-eids-mirror` cookie. | + +## 5. End-to-end flow + +### 5.1 Inputs + +| Input | Arrives on | Persisted? | +| --------------------------------- | ----------------------------------------- | ------------------------------------------- | +| `ts-eids` cookie | every request | Navigation only, via browser ingest (§8.4) | +| `sharedId` cookie | every request | Navigation only, applied last in ingest | +| `/auction` body `user.eids` | `/auction` | Never; forwarded to bidders on that request | +| Provider lookup responses | navigation (outbound) | Navigation write | +| After-response provider responses | navigation, after the response (outbound) | Follow-up CAS update (§5.2.1) | +| Batch sync | its own endpoint | Its own write path (unchanged) | + +### 5.2 Navigation (the only writer) + +``` +browser TS (navigation) origin providers KV + | GET / (document) | | | | + |------------------>| read EC cookie, consent | | | + | | read row snapshot ---------------------------------->| + | | classify: navigation => PageLoadWrite permit | + | | coordinator.dispatch(): due sources, consent, | + | | allowlist ---------------------------> lookups | + | | fetch origin ------------------------>| (concurrent) | + | |<-------------------- origin headers --| | | + | | coordinator.collect(): wait until all done or | + | | dispatch + budget (≈200 ms); drop late responses | + | | validate + merge: browser ingest + provider results | + | | if anything changed: one write (permit consumed) ---->| + | |<---------------------------------------- ok / failed | + | | on ok: Set-Cookie EC (new rows) | +| | mirror from committed row, else snapshot row | + |<------------------| response streams | +``` + +Notes: + +- The budget is measured from dispatch. If the origin takes longer than the + budget, provider collection adds no latency. +- The added delay never exceeds the budget. Collection uses + `PlatformHttpClient::select_until` (added by PR #1433), which bounds both + header waits and body reads by an absolute deadline on every adapter, so a + slow or byte-trickling provider cannot hold the response. +- Response headers, including cookies, depend on the write result, so the write + completes before headers are sent. This is today's behaviour for new ECs. +- A navigation with nothing due and nothing new from the browser makes no + provider calls and no write. + +### 5.2.1 After-response collection (opt-in) + +A provider can be slower than any acceptable page budget (for example, one +that takes about a second). Setting `collect = "after_response"` on that +provider moves its lookup out of the page path: + +``` +browser TS (navigation) providers KV + | | page phase as in §5.2, without after-response providers + |<------------------| response sent (send_to_client) | | + | | dispatch after-response providers --------->| | + | | collect until after-response budget (≈2 s) | | + | | read row + generation --------------------------->| + | | one CAS update with their results (if changed) -->| +``` + +- Default is `collect = "page"`; with no provider set to `after_response`, + behaviour is exactly §5.2. No existing behaviour changes. +- Dispatch happens only after the response is sent, because `select_until` + abandons requests still pending at the page deadline on every adapter. +- The budget is `ec.identity_after_response_budget_ms` (default 2000), + measured from the after-response dispatch. Late responses are dropped as in + §8.1.1. +- Only a navigation whose permit is confirmed (not a redirect) and whose page + phase succeeded (consent allows the EC; the page write landed or had nothing + to change) runs the phase, with an `AfterResponseWrite` permit (§9.1). + Redirects and non-navigation requests get neither permit. +- The follow-up write contains provider results only; browser ingest stays in + the page write. It never creates or revives a row: a missing, tombstoned or + withdrawn row, or one whose EC ID changed during the request, is skipped. +- The write is a compare-and-swap on the generation read just before it. On + conflict the results are dropped; the source stays due and is retried on the + next navigation. +- Results are not mirrored in the already-sent response. They are served from + KV from the next page load. A browser fallback UID for an owned source is + replaced by the provider's UID, as in §8.3. +- Source ownership is validated across both phases: a source claimed by a + page provider and an after-response provider is rejected at startup. +- Cost: the instance stays busy after the response for up to the + after-response budget, which lowers per-instance throughput when sandboxes + are reused. Keep the budget no larger than the slowest provider needs. +- Only the Fastly adapter runs this phase (§2.2). Other adapters do not yet + run it, so their `after_response` providers make no lookups. +- Cookie reads and generated IDs of an after-response provider also land only + in the follow-up write, so cookie-only providers should stay `page`. + +### 5.3 What one navigation write contains + +Exactly one of these, chosen from the request snapshot: + +| Situation | Write | +| --------------------------------------- | ----------------------------------------------------------------------------- | +| Consent withdrawn, live row | Tombstone. No lookups, no ingest. EC cookie and mirror expired. | +| Row tombstoned, no fresh consent | None. No lookups. Nothing served. | +| New EC (or orphaned EC being recovered) | Create the row with browser ingest + valid provider results. | +| Returning EC, something changed | CAS update with browser ingest + valid provider results; expired UIDs pruned. | +| Returning EC, nothing changed | None. | + +### 5.4 Later requests in the same page load + +`/auction`, `/_ts/page-bids` and integration proxies read the row snapshot and +never write. They do not receive a permit, so the write API cannot be called +(§9.1). EIDs in the `/auction` body go to bidders on that request only. + +### 5.5 Lifecycle per ID origin + +| ID origin | Collected | Persisted | Mirrored | Served from KV to `/auction` | +| --------------------- | ------------------------------ | ----------------------------- | ---------------------- | --------------------------------------- | +| `ts-eids`/`sharedId` | Navigation carrying cookies | That navigation's write | Never | Next page load (same page via body) | +| `/auction` body | `/auction` | Never | Never | That request only, from the body | +| Provider lookup | Navigation where source is due | That navigation's write | Same response (§10) | Next page load; same page if write done | +| After-response lookup | Navigation where source is due | Follow-up CAS update (§5.2.1) | Never on that response | Next page load | +| Batch sync | Batch endpoint | Its own write path | Never | After its write | + +### 5.6 Expiry and refresh + +- A UID with `exp <= now` is never served, mirrored or forwarded. +- An expired owned UID makes its source due; the owning provider is asked again + on the next navigation. No other provider and no browser input can refresh it. +- Expired UIDs stay in the row until the next write on any navigation, which + prunes them. Pruning alone never triggers a write. +- UIDs without `exp` and without refresh state are never refreshed; they stay until replaced or removed. +- A provider may keep private refresh state with a UID it writes (§6.3). Once + the state's `from` time passes, the source is due even though the UID is + still live, and the provider builds its refresh from the stored state + (`LookupInput::current`). For example, a token provider keeps the vendor's + refresh token and refreshes the token server side without any login input. + Vendor cookies are read first on every navigation; a usable cookie ID is + used instead of a refresh request. Refresh state that is empty, over 2048 + bytes or has no stated time is dropped and the UID is kept. +- A UID is refresh-due only while it is still live. A failing refresh is + therefore retried at most until the UID expires, and the provider's + `rate_limit_per_hour` also bounds it. +- A refresh reply may report an opt-out (`LookupOutcome::OptedOut`). It is + handled like a cookie opt-out: the provider's sources are cleared from the + row. +- A provider whose IDs have no vendor server may generate an ID instead of a + lookup (`IdentityProvider::generate`), for a due source when its cookies hold + none. The ID is validated and written like a lookup result; no request is + sent and no cookie is set. For example, a publisher common ID. Generation is + not rate-limited, and a source holding a live UID is not regenerated. +- A request may carry provider context (`ProviderRequest::context`) that is + never sent and is handed back with its response, for example a key that + decrypts the reply. + +### 5.7 Consent withdrawal + +- On any finalized request with the EC cookie: the tombstone is written outside + the page-load permit (§3.1); provider calls are not made; any lookup already + dispatched is discarded; no IDs are served and the mirror is expired. +- A concurrent navigation in another tab that tries to write collection results + after the tombstone fails its CAS and is not retried, so the tombstone wins. A + tombstone that loses its CAS to such a write retries, so the withdrawal still + wins. +- After fresh consent, the existing revive path creates a new row; previous IDs + are not restored. + +## 6. Type design + +### 6.1 Principles + +- **Every seam is a trait.** Five traits make the solution pluggable; each has + one production implementation and can be replaced in tests or future work + without touching shared policy. + + | Trait | Varies by | Production implementation | Other implementations | + | --------------------- | --------------------------- | ------------------------- | --------------------------------- | + | `IdentityProvider` | vendor | one per provider kind | test providers | + | `ProviderKind` | vendor (typed construction) | one per provider kind | — | + | `LookupTransport` | runtime / environment | `PlatformLookupTransport` | test transport; F4 mock transport | + | `IdentityCoordinator` | orchestration policy | `ProviderCoordinator

` | test fake for navigation tests | + | `EcKvStore` (exists) | KV backend | Fastly KV | existing in-memory test store | + +- **Static dispatch on the request path.** The coordinator, transport and + navigation code are generic. The new traits use native `async fn` (with + `#[allow(async_fn_in_trait)]`), not `async_trait`, so they add no boxed + futures; the existing `PlatformHttpClient` underneath keeps its own + `async_trait` boxing. The only trait objects are the configured providers: operators choose a + heterogeneous set, so each is boxed once at startup. `ProviderCoordinator

` + stays generic over `P`, so tests and single-provider builds use a concrete + type with no boxing. +- **Borrow per-request data.** Provider input borrows the request and the row + snapshot. Providers parse responses into types that borrow the response body + (`Cow<'body, str>`, `&'body RawValue`). Only entries that pass validation and + will be stored become owned. +- **Share config-derived identifiers.** `ProviderId` and `SourceDomain` wrap + `Arc`; clones made from config cost a reference count. `Serialize` and + `Deserialize` are implemented by hand: serializing writes the string + directly, and deserializing visits a `&str` and allocates the `Arc` + once. +- **Keep opaque JSON opaque in storage.** `ext` objects are stored as `RawJson`, + a `RawValue` newtype: never parsed into a map between provider response and + KV. They are parsed once, into `serde_json::Value`, only when an OpenRTB + request or header is built, because `crate::openrtb` already uses `Value` for + `ext`. This needs `serde_json`'s `raw_value` feature; no new crate is added. +- **Parse, don't validate.** Raw provider output converts into stored types + through `TryFrom`; past that boundary an invalid UID, agent type or match + method cannot be represented. +- **No micro-optimisation.** Small `Vec`s sized by provider or UID count are + fine; no arenas, no small-vector crates. + +### 6.2 Value types + +```rust +/// Unix time in whole seconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +/// Operator-chosen identifier of a configured provider instance. +/// +/// `Serialize` and `Deserialize` are implemented by hand (§6.1). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +/// Normalized EID source domain. +/// +/// `Serialize` and `Deserialize` are implemented by hand (§6.1); +/// deserializing normalizes and rejects invalid domains. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +/// A non-empty UID of at most [`MAX_UID_LENGTH`] bytes. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl TryFrom for UidValue { + type Error = EidValidationError; + // ... +} + +/// Non-negative OpenRTB agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32")] +pub struct AgentType(i32); + +/// An opaque JSON object, compared by its bytes. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RawJson(Box); + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + #[display("UID is empty")] + EmptyUid, + #[display("UID is too long")] + UidTooLong, + #[display("agent type is negative")] + NegativeAgentType, + #[display("match method is unknown")] + UnknownMatchMethod, + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + #[display("source is not claimed by this provider")] + UnclaimedSource, + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// OpenRTB 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(into = "i32", try_from = "i32")] +pub enum MatchMethod { + Unknown, + NoMatching, + CookieSync, + Authenticated, + Observed, + Inference, +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(exp) => now < exp, + } + } + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} +``` + +`Expiry` is stored as an optional `exp` field: absent for `Never`. + +```rust +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +mod optional_exp { + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(exp) => exp.serialize(serializer), + // Never reached: the field is skipped when `Expiry::is_never`. + Expiry::Never => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} +``` + +Sketches in this spec omit doc comments on self-describing fields and elide +bodies with `// ...`; implementations follow the documentation rules in +`AGENTS.md`. + +### 6.3 Stored row + +All additions are optional `serde` fields on schema `v: 1`. Rows written before +this change read unchanged; rows written after it stay readable by older code +through the retained `uid` field. + +`uid` is rewritten on every write to the first live UID of `eids`, and new code +ignores it when `eids` is non-empty. A known limitation of staying on `v: 1` +(D12): during a rollout or rollback, older code reads only `uid`, so it can +serve that UID after its `exp`, and an older writer that re-saves the row drops +`eids`. Both end at the next write by new code. Stored values are built only by validation +(§8.2) or by deserializing rows TS wrote, so their fields are private to the +`ec` module and exposed through accessors. + +```rust +/// A partner ID record within a KV entry, keyed by source domain. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvPartnerId { + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. + uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +/// One stored EID entry with its EID-level fields. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvEid { + #[serde(default, skip_serializing_if = "Option::is_none")] + inserter: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + matcher: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + mm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + uids: Vec, +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + /// Provider value, or the source's configured `openrtb_atype`. + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + /// Provider that wrote this UID; `None` for browser and batch UIDs. + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde(default, rename = "exp", with = "optional_exp", skip_serializing_if = "Expiry::is_never")] + expiry: Expiry, + /// Owning provider's private refresh state (§5.6); never served. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. `Debug` hides +/// `state`, which may hold a credential. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework (at most 2048 bytes). + state: String, + /// When the owning provider should be asked to refresh the UID. + from: UnixSeconds, +} +``` + +Outbound EIDs (auction, mirror, `x-ts-eids`) are built from a UID's `id`, +`atype` and `ext` only, so `refresh` never leaves the row. Refresh state is +private: it is never served, mirrored, forwarded or shown. The authenticated +admin view (`/_ts/admin/ec/{id}`) shows each `refresh.state` as `[redacted]` +and keeps `from` visible. It is removed with the UID on opt-out, withdrawal or +expiry pruning. Rows without it read unchanged. + +A UID whose refresh state is empty or over 2048 bytes, or has no stated time, +keeps the UID and drops only the refresh state. + +The source domain stays the map key in `KvEntry::ids`, so one provider can own +several sources and each UID still records its owner. + +### 6.4 Reading the row + +Consumers read through borrowed views and build owned OpenRTB values once, at +the point they serialize a request or header: + +```rust +impl KvEntry { + /// Iterates entries for `bidstream_enabled` sources, yielding only live + /// UIDs and borrowing from the row. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + now: UnixSeconds, + ) -> impl Iterator> + 'row; +} + +/// A borrowed EID entry: either a single legacy UID or a full-fidelity entry. +pub enum EidView<'row> { + Single { source: &'row str, uid: &'row str, atype: AgentType }, + Full { source: &'row str, eid: &'row KvEid, now: UnixSeconds }, +} + +impl From> for crate::openrtb::Eid { + // ... +} +``` + +The in-process `crate::openrtb::Eid` gains optional `inserter`, `matcher`, +`mm` and `ext` fields so full-fidelity entries reach bidders unchanged. + +## 7. Provider interface and crate boundary + +### 7.1 Where things live + +| Location | Contents | +| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `trusted-server-core` `ec::eid_types` | Value types and stored row types (§6). | +| `trusted-server-core` `ec::provider` | `IdentityProvider`, `ProviderKind`, the generic `build_kind` helper, input and output types, errors. No vendor code. | +| `trusted-server-core` `ec::coordinator` | `IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`, validation, merge, mirror encoding. | +| `crates/trusted-server-identity-providers` | One module per provider kind, each implementing `ProviderKind` and `IdentityProvider`, and one `build` function that matches `kind` to a `ProviderKind`. | +| `trusted-server-adapter-fastly` | Builds providers through the provider crate and a `ProviderCoordinator` at startup, and drives the navigation path with `PlatformLookupTransport`. | + +Dependencies point one way: the provider crate depends on core; core never +depends on the provider crate. Adding a provider means adding a module and one +`build` match arm in the provider crate. Shared consent, KV, cookie and auction +code never gains provider-specific branches. + +### 7.2 `IdentityProvider` + +Providers are pure: they build a request and parse a response. They never do +HTTP, KV, cookie or clock work. The trait is dyn-compatible so a configured set +can hold `Box`. + +```rust +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. +pub trait IdentityProvider { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request(&self, lookup: &LookupInput<'_>) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be parsed + /// at all. Individually malformed entries are returned and dropped by the + /// coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; + + /// Generates IDs for due sources instead of a lookup (§5.6). + /// + /// The default returns `None`: the provider makes a lookup. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + None + } +} + +impl IdentityProvider for Box

{ + // Delegates each method to `**self`. +} +``` + +Capabilities: + +```rust +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +#[derive(Debug, Clone)] +pub struct SourceCapability { + pub source: SourceDomain, + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + pub max_eids: NonZeroUsize, + pub max_uids_per_eid: NonZeroUsize, + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + Preserved, + Ignored, +} +``` + +Input and request: + +```rust +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + pub ec_id: &'a str, + pub consent: &'a ConsentContext, + pub client: ClientSignals<'a>, + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + pub ip: Option, + pub user_agent: Option<&'a str>, +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + pub method: http::Method, + pub url: url::Url, + pub headers: http::HeaderMap, + pub body: bytes::Bytes, + /// Never sent; handed back with the response (§5.6). + pub context: Option, +} +``` + +`OwnedEntries<'a>` is a borrowed view over the row that yields this provider's +`(&str, &KvEid)` pairs, keyed by the row's source strings. Provider secrets and settings are not in the +input: they are given to the provider once, when it is built (§7.3). + +`LookupInput` also lends the provider the request's cookies (`cookies`), which +`IdentityProvider::read_request_cookies` reads into a `CookieRead`. These +predate this work and are unchanged. + +Response and parsed output: + +```rust +#[derive(Debug, Clone, Copy)] +pub struct ProviderResponse<'body> { + pub status: http::StatusCode, + pub headers: &'body http::HeaderMap, + pub body: &'body [u8], + /// The request's `context`, handed back. + pub context: Option<&'body str>, +} + +#[derive(Debug)] +pub enum LookupOutcome<'body> { + Found(Vec>), + NoMatch, + /// The visitor opted out (§8.3). + OptedOut, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + pub source: Cow<'body, str>, + pub inserter: Option>, + pub matcher: Option>, + pub mm: Option, + pub ext: Option<&'body RawValue>, + pub uids: Vec>, +} + +#[derive(Debug)] +pub struct ProviderUid<'body> { + pub id: Cow<'body, str>, + pub atype: Option, + pub ext: Option<&'body RawValue>, + pub expiry: ProviderExpiry, + /// Private refresh state to keep with the UID (§5.6). + pub refresh: Option>, +} + +#[derive(Debug)] +pub struct ProviderRefresh<'body> { + pub state: Cow<'body, str>, + /// When to refresh; resolved like `exp`. + pub from: ProviderExpiry, +} + +/// Expiry as a provider states it. The coordinator turns it into [`Expiry`] +/// using its own clock, so providers never read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + Unspecified, + At(UnixSeconds), + After(Duration), +} + +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + #[display("provider request could not be built")] + Request, + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} +``` + +Providers typically deserialize their wire format into private structs with +`#[serde(borrow)]` and map them into `ProviderEid<'body>`. + +### 7.3 `ProviderKind` and registration + +Construction is typed per provider; only the final value is erased. + +```rust +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +pub struct ProviderCommon<'a> { + pub endpoint: &'a url::Url, + pub sources: &'a [SourceDomain], + pub secrets: &'a ProviderSecrets, +} + +/// Parses `settings` for `K` and builds a boxed provider. Lives in core. +/// +/// `secrets` are resolved by the adapter at startup through the existing +/// secret path, from the key names in `config`. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when `settings` does not +/// deserialize into `K::Settings` or [`ProviderKind::build`] rejects it. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report>; +``` + +The provider crate's single entry point, shown with the `pull-sync` arm that +PR5 adds (PR3 ships only the fallback arm): + +```rust +/// Builds the provider for a configured `kind`. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + match config.kind() { + PullSync::KIND => build_kind::(config, secrets), + kind => Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))), + } +} +``` + +`ProviderSecrets` holds `Redacted` values resolved through the existing +secret path and is passed only at build time. + +```rust +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + #[display("provider kind is not registered")] + UnknownKind, + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} +``` + +`IdentityProviderConfig` is the deserialized `[[ec.identity_providers]]` entry +(§12.1). `HashedEmail` is a newtype over a validated lowercase hex SHA-256 +digest with no constructor reachable from request data in V1. + +## 8. Coordinator + +### 8.1 Responsibilities + +The coordinator owns, for every provider: + +1. **Eligibility.** Provider enabled; EC present or being created; row not + tombstoned; at least one due source; sources needing `HashedEmail` removed. +2. **Consent.** The existing EID-persistence check passes, and the provider's + `tcf_vendor_id`, when configured, has vendor consent in the effective TCF + string. A failure skips that provider only, before any request is built. +3. **Destination.** The built URL must be `https` and its host must be in the + provider's configured `allowed_domains`. The URL comes from operator config; + browser data never selects a destination. A rejected destination skips the + provider and is counted. +4. **Transport.** All eligible requests are started through a + `LookupTransport` and gathered until all complete or the shared deadline + passes. Late responses are dropped (§8.1.1). +5. **Validation** (§8.2), **merge** (§8.3) and preparing the single write. +6. **Mirror** encoding (§10) from the row as it stands after the navigation. +7. **Logs and metrics** (§12). + +#### 8.1.1 `LookupTransport` + +The transport sends provider requests and gathers responses within a deadline. +It is the only coordinator dependency that touches the network, so swapping it +is how tests and the future debug mode (F4) run real providers and the real +coordinator against mock responses. + +```rust +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + pub provider: ProviderIndex, + pub value: T, +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + pub status: http::StatusCode, + pub headers: http::HeaderMap, + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderIndex(u16); + +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + #[display("lookup request could not be sent")] + Send, + #[display("lookup did not complete before the deadline")] + Timeout, + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} +``` + +`Deadline` wraps the `web_time::Instant` that `select_until` takes, because +core is built into the Cloudflare Worker (CI wasm clock lint). +`PlatformLookupTransport<'s>` borrows `RuntimeServices` and is a thin adapter +over the platform client: `start` calls `send_async`, and `gather` calls +`select_until` with the shared deadline while requests remain in flight, +stopping when none remain (it rejects an empty list) or it returns `Ok(None)`, +and tags each result with its provider. It adds no timing logic of its own. Response +bodies are capped at 64 KiB. + +#### 8.1.2 `IdentityCoordinator` + +The navigation path depends on the `IdentityCoordinator` trait, generic over +the coordinator and the transport. The trait covers the coordinator's whole +surface on a navigation: start lookups, collect them, plan the single row +change, and build the mirror. Its production implementation is +`ProviderCoordinator

`. Tests of coordinator rules (validation, merge, +mirror) run `ProviderCoordinator` with test providers and a test transport; a +fake `IdentityCoordinator` is used only for navigation wiring tests, where +those rules are not under test. + +```rust +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget, validation, merging and mirror encoding. Providers never see +/// KV, cookies, HTTP or the clock. Implementations never write KV: they return +/// the change and the caller spends the [`PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when nothing is due, consent does not + /// permit EID collection, or no provider is enabled. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; + + /// Merges browser ingest and lookup results into the navigation's single + /// row change. + /// + /// Returns `None` when the row would not change, so no write is made. + fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; + + /// Builds the mirror cookie from the row as it stands after this + /// navigation: the committed row, or the snapshot when no write landed. + fn mirror(&self, row: Option<&KvEntry>, consent: &ConsentContext, now: UnixSeconds) -> MirrorCookie; +} +``` + +Supporting types: + +```rust +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + pub ec_id: &'a str, + pub snapshot: &'a EcKvSnapshot, + pub consent: &'a ConsentContext, + pub client: ClientSignals<'a>, + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. Only the coordinator module can +/// create a non-empty value. `Default` (empty) and `Debug` (counts only) are +/// implemented by hand so they do not require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// Request time, for validating `exp` and resolving `ProviderExpiry::After`. + now: UnixSeconds, + skipped: Vec>, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Default)] +pub struct LookupResults { + lookups: Vec, +} + +#[derive(Debug)] +pub struct ProviderLookup { + pub provider: ProviderId, + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `status` is `Found`. + pub entries: Vec<(SourceDomain, Vec)>, +} + +/// One outcome per provider per navigation, used for logs and metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + Found { kept: u16, dropped: DropCounts }, + NoMatch, + Timeout, + HttpError(http::StatusCode), + TransportFailed, + InvalidResponse, + Skipped(SkipReason), +} + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + pub invalid_uid: u16, + pub invalid_field: u16, + pub unclaimed_source: u16, + pub expired: u16, + pub over_limit: u16, + pub duplicate: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + Disabled, + Consent, + NotDue, + Input, + Destination, + RateLimited, + OptedOut, +} + +/// Inputs to [`IdentityCoordinator::plan_write`]. +/// +/// `PartnerIdUpdate` (today `pub(crate)` in `ec::kv`) becomes `pub` so this +/// public trait input does not leak a private type. +pub struct WritePlanInput<'a> { + pub snapshot: &'a EcKvSnapshot, + pub consent: &'a ConsentContext, + pub new_entry: Option<&'a KvEntry>, + /// Updates from `ts-eids` and `sharedId`, already reduced by today's rules. + pub browser_ingest: &'a [PartnerIdUpdate], + pub lookups: &'a LookupResults, + pub now: UnixSeconds, +} + +/// What to do with the mirror cookie on this response. +#[derive(Debug)] +pub enum MirrorCookie { + Set(MirrorValue), + Expire, +} + +/// An encoded mirror value within the size limit (§10). +#[derive(Debug)] +pub struct MirrorValue(String); +``` + +The production coordinator: + +```rust +/// The production coordinator, built once at startup. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation, `bidstream_enabled` + /// and config order for the mirror. + partners: Arc, + budget: Duration, +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated at startup. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +impl IdentityCoordinator for ProviderCoordinator

{ + // ... +} +``` + +`rate_limit_per_hour` is not a new traffic control: it carries over pull sync's +existing per-EC, per-partner hourly limit (PR5), which needs the `RateLimiter` +state that pure providers cannot hold. Operators may also set it per provider +with the optional `rate_limit_per_hour` config field (§12.1), for example to +bound repeat lookups for sources a provider never resolves; it is unset by +default. Broader traffic controls remain future work (F2, F3). + +The navigation path, in order: + +```rust +async fn run_navigation(coordinator: &C, transport: &T, navigation: Navigation<'_>) -> Response +where + C: IdentityCoordinator, + T: LookupTransport, +{ + let Navigation { permit, request, graph, ec, .. } = navigation; + let pending = coordinator.dispatch(transport, &ec.lookup_context()).await; + let mut response = fetch_origin(request).await; + let lookups = coordinator.collect(transport, pending).await; + let mut row = ec.snapshot_entry(); + let change = permit + .confirm(response.status()) + .zip(coordinator.plan_write(&ec.write_plan(&lookups))); + if let Some((permit, change)) = change { + // One attempt, never retried (§9.2). + match graph.commit_page_load(permit, ec.id(), change) { + Ok(committed) => row = Some(committed), + Err(report) => ec.record_write_failure(&report), + } + } + ec.apply_cookies(&mut response, row.as_ref(), coordinator.mirror(row.as_ref(), ec.consent(), ec.now())); + response +} +``` + +### 8.2 Validation + +Validation keeps valid entries and drops only invalid ones. Each drop is +counted by reason. + +| Level | Rule | +| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| EID | `source` normalizes to a source this provider claims in config; otherwise dropped. | +| EID | `inserter`/`matcher` at most 255 bytes; `mm` within OpenRTB values; `ext` a JSON object of at most 1 KiB. EID-level fields are dropped (not the entry) when `eid_level_fields` is `Ignored`. | +| UID | `id` non-empty after trimming and at most `MAX_UID_LENGTH` (1024) bytes; `ext` a JSON object of at most 1 KiB. | +| UID | Refresh `state` non-empty and at most 2048 bytes, with a stated `from`; otherwise the refresh state is dropped and the UID kept. `from` is resolved like `exp`. | +| UID | `atype` missing is filled from the source's `openrtb_atype`. | +| UID | `exp`, when present, must be greater than now; otherwise dropped. | +| Counts | Entries beyond `max_eids`, or UIDs beyond `max_uids_per_eid`, are dropped from the end. | +| Dedupe | UIDs with the same (`source`, `id`) keep the first occurrence, as in §11.1. | +| Row size | If the merged row would exceed 16 KiB serialized, new provider UIDs are dropped in reverse provider-config order, last UID first, until it fits. | + +An EID entry left with no UIDs is dropped. A response that cannot be parsed at +all is a failure for that provider; other providers and browser input are +unaffected. + +### 8.3 Merge into the row + +For each provider with a `Found` outcome and each source in its response: + +- The validated entries replace every UID of that source in the row, including + unowned ones (D8). Each written UID records `owner` and `exp`. +- Sources the provider claims but did not return stay unchanged. +- `NoMatch`, failures, timeouts and skipped providers leave the row unchanged. +- `OptedOut` (a cookie or a refresh reply) clears every source the provider + claims; generated and cookie IDs merge like `Found`. + +Browser ingest (§8.4) is applied in the same in-memory merge, before provider +results, so the write order never lets browser data overwrite provider data. + +### 8.4 Browser ingest (navigation only) + +Unchanged rules: registered sources only; one UID per source, the first valid +UID of the last matching entry, with `sharedId` applied last; no EID-level +fields; fill missing sources only, where a source holding only expired UIDs +counts as holding UIDs until a write prunes them. New rule: a provider-owned +source is skipped entirely. + +### 8.5 Batch sync + +Batch sync keeps its own write path. A record for a source claimed by an +enabled provider is rejected and counted as `provider_owned`; other records in +the same batch are processed as today. + +### 8.6 Disabled providers + +A disabled provider makes no outbound requests and is not considered for +eligibility, and its sources stop being provider-owned. UIDs it wrote earlier +keep serving until `exp`, or until replaced or removed; once pruned, browser +ingest may fill the source again. Other providers and client-side collection +are unaffected. + +## 9. KV write counting, failure and retry + +### 9.1 Permit + +The write limit is enforced by types. A navigation receives a +`NavigationPermit`; it becomes a `PageLoadWrite` only when the final response +is not a redirect; the write consumes it. Neither type is `Clone` or +`Copy`, `classify` is `pub(crate)` and called once by the router, and +`commit_page_load` consumes the permit, so a navigation cannot write twice and +no other request can write at all. + +```rust +/// Issued to a request classified as a page-load navigation (§3.1). +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation. + /// + /// Called once per request, by the router only. + #[must_use] + pub(crate) fn classify(request: &http::Request) -> Option; + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect: the redirect target is the page load. + #[must_use] + pub fn confirm(self, status: http::StatusCode) -> Option; +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +// Also on the navigation permit: confirms it together with the permit for the +// after-response phase (§5.2.1). +// pub fn confirm_with_after_response(self, status: http::StatusCode) +// -> Option<(PageLoadWrite, AfterResponseWrite)>; + +/// Proof for the one follow-up update; consumed by `commit_after_response_lookups`. +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { entry: KvEntry, expected: Generation }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + #[display("row changed since it was read")] + Conflict, + #[display("row already exists")] + AlreadyExists, + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +impl KvIdentityGraph { + /// Applies the navigation's single row change in one attempt. + /// + /// # Errors + /// + /// Returns [`KvWriteError`] on CAS conflict, an existing row on create, or + /// a store failure. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result>; +} +``` + +`Generation` is a newtype over the KV generation captured in +`EcKvSnapshot::Present`. When the snapshot carries no generation, +`plan_write` returns `None` for updates and the navigation counts +`no_generation`: TS never overwrites a row blindly. + +After PR2 all browser-driven KV writes except withdrawal tombstones go through +`commit_page_load`: new EC creation, orphan recovery and browser ingest; PR3 +adds provider results. Withdrawal tombstones keep their existing path (§3.1). The post-send pull-sync write is the one exception until +PR5 removes it. + +### 9.2 Failure behaviour + +| Failure | Effect | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| CAS conflict or store error | No retry. Row unchanged. Mirror built from the unchanged row. New EC cookie not set. Counted. | +| Tombstone write fails | Bounded CAS retry first. Mirror expired and no IDs served (consent gating). EC cookie kept so the next request retries it. | +| EC-ID collision on create | Same as above; the next navigation generates a new EC. | +| Provider timeout/outage/invalid | That provider's results are dropped; the write proceeds with everything else that is valid. | +| All providers fail | The write proceeds if browser ingest changed anything; otherwise no write. | +| Coordinator budget exceeded | Late responses dropped; the response is never held past the budget. | +| After-response row missing, withdrawn or EC changed | Nothing written (`no_row`); results dropped. | +| After-response write conflict | Results dropped; the source stays due and is retried on the next navigation. The sent response is unaffected. | + +Failed lookups are retried only by a later navigation where the source is +still due. + +## 10. Mirror cookie + +| Attribute | Value | +| --------- | -------------------------------------------------------------------------------------------- | +| Name | `__Host-ts-eids-mirror` | +| Flags | `Path=/; Secure; SameSite=Lax`; no `Domain`; not `HttpOnly` | +| Max-Age | Same as the EC cookie | +| Value | base64url of `{"v":1,"eids":[...]}`; each entry is an OpenRTB EID whose UIDs may carry `exp` | +| Size | Encoded value at most 3,584 bytes | + +- **Contents:** live, provider-owned UIDs of `bidstream_enabled` sources, with + their EID-level and UID-level fields. +- **Fill order:** sources in `[[ec.partners]]` config order; entries and UIDs in + stored order. Each UID is added if the encoded value still fits; otherwise it + is skipped and filling continues. KV keeps the complete set. +- **Set** on every consented navigation response from the row as it stands + after the navigation: the committed row, or the snapshot when no write + landed. Setting a cookie is not a KV write, and this restores a mirror the + browser lost. +- **Expired** (`Max-Age=0`) on any response where EID consent is absent or + withdrawn, whatever happened to the KV write, and on a navigation whose + mirror would be empty. +- **The server never reads it.** Readers on the page must drop UIDs whose `exp` + has passed and strip `exp` before use. + +## 11. Consumers + +All consumers read the row snapshot and gate by current consent and +`bidstream_enabled`, as today. None write KV. + +| Consumer | Behaviour | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/auction` | Resolves the row to EIDs, full-fidelity for provider entries, dropping UIDs that are not live. Merges with the body EIDs (§11.1). Server IDs need no browser resend. | +| `/_ts/page-bids` | Same as `/auction`. | +| `x-ts-eids` header | Built from the merged set with the existing truncation flag. | +| `/_ts/api/v1/identify` | Includes provider UIDs that are live, under existing authentication and partner gating. | +| Integration proxies | Use the same resolver where they forward EIDs. | + +### 11.1 Merge for auctions + +1. KV entries first, then body EIDs. +2. EID entries are distinct when (`source`, `inserter`, `matcher`, `mm`) differ; + both are kept. +3. A UID whose `id` already appears under the same source is a duplicate; the + first occurrence (KV) keeps its fields. +4. Distinct UIDs are kept under their entry. + +## 12. Configuration, secrets and observability + +### 12.1 Configuration + +```toml +[ec] +# Shared time budget for all provider lookups on one navigation. +identity_budget_ms = 200 +# Budget for providers with collect = "after_response", run after the +# response is sent (§5.2.1). +identity_after_response_budget_ms = 2000 + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +enabled = true +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry and must be in the kind's capabilities. +sources = ["ids.example.com"] +# Optional: require TCF vendor consent for this provider. +tcf_vendor_id = 9999 +# Optional: "page" (default) or "after_response" (§5.2.1). +collect = "page" +# Optional: most lookups per EC per hour (§8.1); unset means unlimited. +rate_limit_per_hour = 4 + +# Secret Store key names, resolved like other secret references. +[ec.identity_providers.secrets] +token = "example_provider_token" + +[ec.identity_providers.settings] +# Provider-specific settings, parsed by the provider implementation. +``` + +Startup validation rejects: unknown `kind`; a source with no `[[ec.partners]]` +entry; a source outside the kind's capabilities; a source claimed by two +enabled providers; an `endpoint` host outside `allowed_domains`; placeholder +secrets in deploy validation. + +The operator CLI's config checks (`ts config validate`, `push` and `diff`) +also build every enabled provider of a registered kind, as the adapter does +per navigation, so kind-specific setting errors fail before deploy. Core's +own validation cannot do this because provider kinds live outside core. + +Secrets are Secret Store key references, resolved at startup into +`Redacted` values through the existing secret path. They are never written to cookies or logs, and source-controlled +examples use placeholders. + +### 12.2 Logs and metrics + +Per provider and navigation, one outcome: `found`, `no_match`, `timeout`, +`http_error`, `transport_error`, `invalid_response`, `skipped_disabled`, `skipped_consent`, +`skipped_not_due`, `skipped_input`, `skipped_rate_limited`, `skipped_opted_out`, +`destination_rejected`; plus counts of +entries and UIDs kept and dropped by reason. Per navigation, one write outcome: +`written`, `no_change`, `conflict`, `error`, `not_permitted`, +`no_generation`. A navigation that runs the after-response phase (§5.2.1) has +one more write outcome: `written`, `unchanged`, `no_row`, `conflict` or +`failed`. Logs include the +provider ID, source, counts and durations, and the EC ID only in its existing +truncated form. Raw UIDs, EID values and encrypted envelopes are never logged. + +## 13. PR breakdown + +### 13.1 Required + +The work below is delivered as two pull requests: + +- **PR A** — the work units PR1, PR2 and PR3, stacked on the open platform PR + #1433, which provides the deadline-bounded `select_until` that + `PlatformLookupTransport` uses. +- **PR B** — the work units PR4, PR5 and PR6, stacked on PR A. + +PR A and PR B ship in the same release, because PR A alone leaves the old +post-send pull-sync write as a second write per page load. Within each pull +request the units are separate, ordered commits so they stay reviewable. + +| PR | Title | Contents | Depends on | +| --- | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| PR0 | Spec | This document. | — | +| PR1 | Full-fidelity EID storage and resolution | `ec::eid_types` value types (§6.2); `KvEid`/`KvUid` additive schema; EID-level fields on `crate::openrtb::Eid`; resolver drops UIDs that are not live; auction merge (§11.1); `x-ts-eids` and identify use it. No new writers. | PR0 | +| PR2 | Page-load write permit | Navigation classifier; `PageLoadWrite` and `commit_page_load`; single attempt; non-navigation requests read-only except withdrawal tombstones; browser ingest on navigation only; withdrawal handling (§3.1, §5.7); integration-proxy EC minting stops. | PR0 | +| PR3 | Provider trait and coordinator | `ec::provider` (`IdentityProvider`, `ProviderKind`), `ec::coordinator` (`IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`), provider crate with no kinds, config and validation, consent, allowlist, budgeted fan-out, validation, merge into the navigation write, provider-owned skip in ingest, metrics, example config. | PR1, PR2, #1433 | +| PR4 | Batch sync on provider-owned sources | Reject and count records for sources an enabled provider claims. | PR3 | +| PR5 | Pull sync as a provider | `pull-sync` kind built from existing `[[ec.partners]]` pull-sync settings with no config change; existing per-partner rate limit kept as provider-scoped eligibility; post-send dispatcher and completeness marker removed, leftover marker cookies expired. Behaviour change for existing pull-sync partners: their sources become provider-owned, so browser ingest skips them and, with PR4, batch sync rejects them; once PR6 lands their IDs are mirrored when `bidstream_enabled`. | PR3 | +| PR6 | Mirror cookie | Adds `IdentityCoordinator::mirror` (PR3 ships the trait without it); encoding, fill order, set and expiry rules (§10). | PR3 | + +### 13.2 Optional and future (Tech Lead decision) + +| ID | Task | Notes | Depends on | +| --- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| O1 | tsjs reads the mirror (D6b) | Reads the mirror, drops expired UIDs, strips `exp`, passes entries to Prebid user IDs. Separate PR. | PR6 | +| F1 | Concrete provider implementations | R§5.1. One spec and one PR per vendor named in R§6, each a module and `build` arm in the provider crate. | PR3 | +| F2 | Provider lookup traffic controls | R§5.2. Best-effort dedup per EC and provider, bounded no-match caching kept distinct from failures, refresh jitter. Strict cross-instance dedup is a separate decision. | PR3 | +| F3 | Per-provider cooldown | R§5.3. Pause after repeated failures, then probe for recovery. | PR3 | +| F4 | Provider debug mode | R§5.4. A mock `LookupTransport` runs real providers and the real coordinator against configurable responses; no real provider calls, no production KV or cookie mutation. | PR3 | + +## 14. Acceptance scenarios + +1. **First navigation, consent granted.** No EC. One provider is enabled and + due. The provider answers within budget. → One KV create holding the + provider UIDs (with owner and `exp`) and browser-ingested UIDs; the EC cookie + and mirror are set on the response; the provider is called once. +2. **One write per page load.** A navigation is followed by three `/auction` + calls, one `/_ts/page-bids` call and two integration-proxy calls, all + carrying new `ts-eids` content. → At most one KV write, on the navigation. +3. **Same-page and next-page availability.** Provider IDs written by the + navigation → the next page load's `/auction` contains them without the + browser resending them; a same-page `/auction` contains them only if it + read the row after the write. +4. **Slow provider.** Provider A answers in 50 ms, provider B in 500 ms, budget + 200 ms. → A's UIDs are written; B's response is dropped and counted as + `timeout`; the response is not held past the budget. +5. **Partial invalid response.** A provider returns three UIDs, one empty and + one over 1024 bytes. → One UID is stored; two drops are counted. +6. **Write failure.** The navigation's CAS fails. → No retry; mirror built from + the unchanged row; + on a new EC no EC cookie is set; the next navigation looks up again. +7. **Expiry.** A provider UID's `exp` passes. → It is excluded from `/auction`, + `x-ts-eids`, identify and the mirror; the next navigation asks only its + owning provider. +8. **No expiry.** A provider UID without `exp` → served until the provider + replaces it or consent is withdrawn; never refreshed unless it carries refresh state + (scenario 24). +9. **Withdrawal on a navigation.** → Tombstone written; no provider calls; EC + cookie and mirror expired; no IDs served. +10. **Withdrawal on `/auction`.** → Tombstone written (§3.1), EC cookie and + mirror expired, no IDs served. +11. **Tombstone beats a concurrent write.** Tab A writes a tombstone while tab + B's navigation is collecting provider results. → B's CAS fails and is not + retried; the provider IDs are never stored or served. +12. **Provider-owned source.** A browser `ts-eids` UID and a batch record arrive + for a source an enabled provider claims. → Browser UID ignored; batch + record rejected and counted; provider UIDs unchanged. +13. **Multi-source provider.** One provider returns entries for two claimed + sources with two UIDs each and EID-level fields. → Both sources stored with + all four UIDs and fields, each owned by that provider; `/auction` forwards + them at full fidelity. +14. **Disabled provider.** One of two providers is disabled. → No outbound + request to it; the other provider and browser ingest behave as before. +15. **Destination rule.** A provider builds a URL whose host is not in + `allowed_domains`. → No request is sent; counted as `destination_rejected`. +16. **Hashed-email source.** A source requires `HashedEmail`. → Skipped before + any call, counted as `skipped_input`. +17. **Mirror trimming.** Provider UIDs exceed 3,584 encoded bytes. → The mirror + holds the deterministic config-order fill of §10; KV holds all of them. +18. **No secrets or IDs in logs.** Every outcome above produces logs and metrics + without raw UIDs, envelopes or secrets. +19. **Nothing due.** A returning visitor with all owned UIDs live and no new + browser input. → No provider calls, no KV write. +20. **Redirect.** The navigation's origin response is a 302. → No KV write; the + redirect target navigation may write. +21. **Pull-sync partner after PR5.** A partner with pull sync enabled also + receives a browser `ts-eids` UID and a batch record. → Both are ignored or + rejected as for any provider-owned source; the pull-sync provider's UID is + stored and served. +22. **After-response provider.** Provider A is `collect = "page"` and answers + in 50 ms; provider B is `collect = "after_response"` and answers in + 900 ms; page budget 200 ms. → The response is sent without waiting for B; + the page write holds A's UIDs and browser ingest; after the response, one + CAS update adds B's UIDs with owner and `exp`; the next page load serves + them. +23. **After-response on a withdrawn or changed row.** The row is tombstoned, + or written by another request, between the page write and B's result. → + No row is created or revived; on a generation conflict B's results are + dropped and B is asked again on the next navigation. +24. **Provider refresh.** A provider-written UID is live until 5000 and holds + refresh state with `from` 900; the navigation is at 1000 and the request carries no usable vendor + cookie. → The source is + due; the provider's request is built from the stored state; the refreshed + UID and its new refresh state are written. With `from` 2000 the source is + not due and nothing is sent. +25. **Refresh reports an opt-out.** A refresh reply reports an opt-out. → The + provider's sources are cleared from the row; nothing is stored for them. +26. **Generated ID.** A provider that generates IDs has a due source and no + cookie ID. → No request is sent; the generated UID is validated and written + with its owner; on the next navigation the source holds a live UID and is + not regenerated. +27. **Refresh state is never served.** A row holds a UID with refresh state. + → `/auction`, the mirror cookie and `x-ts-eids` carry the UID's `id`, + `atype` and `ext` only. From 79cfe3ec4861fc92f34e5421cf26f94f1420f939 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 00:46:06 +0530 Subject: [PATCH 41/58] Keep opt-outs, stored records and over-cap rows safe in the write planner Provider results never restore UIDs over a newer remembered opt-out, on the after-response re-read or a confirmation-read adoption. The size cap drops this navigation's browser UIDs before any stored record, and trimming an over-cap row alone no longer plans a write. A withdrawn row no longer yields the after-response permit. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 111 ++++++++++++- .../trusted-server-core/src/ec/write_plan.rs | 156 ++++++++++++++++-- 2 files changed, 250 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index d563c5b182..c4bc9ed5d4 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -250,14 +250,20 @@ fn commit_returning( /// Writes the planned update for a returning EC whose row snapshot is held. /// -/// Returns `false` only when a planned write failed. +/// Returns `false` when a planned write failed, or when there was nothing to +/// write because the row is withdrawn. fn update_returning( commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, permit: PageLoadWrite, ) -> bool { let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { - return true; + // Nothing to write. A withdrawn row is not live and must not unlock + // the follow-up write. + return ec_context + .kv_snapshot() + .entry_for(commit.ec_id) + .is_none_or(|entry| entry.consent.ok); }; match commit.graph.commit_page_load(permit, commit.ec_id, change) { Ok(committed) => { @@ -797,9 +803,9 @@ mod tests { #[test] fn withdrawal_keeps_the_cookie_when_the_tombstone_write_fails() { - // A failed tombstone write is not retried. The EC cookie is kept so the - // next navigation can still write the tombstone; clearing it would - // orphan a live row (spec §9.2). + // A tombstone write that still fails after its bounded CAS retries + // keeps the EC cookie, so the next request that carries it retries; + // clearing it would orphan a live row (spec §9.2). let settings = create_test_settings(); let ec_id = sample_ec_id("dead01"); let consent = ConsentContext { @@ -2898,4 +2904,99 @@ mod tests { "should not allow after-response lookups when the page phase failed" ); } + + #[test] + fn finalize_withholds_after_response_lookups_on_a_tombstoned_row() { + // Arrange: a consented navigation reads a withdrawn row. + let settings = create_test_settings(); + let ec_id = sample_ec_id("after4"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed the tombstone"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + // Assert + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups for a withdrawn row" + ); + } + + #[test] + fn finalize_adopted_row_keeps_a_concurrent_provider_opt_out() { + // Arrange: the preload missed the row; meanwhile another request + // recorded provider P's opt-out on it, and P's lookup found a UID. + let settings = create_test_settings(); + let ec_id = sample_ec_id("optout"); + let graph = KvIdentityGraph::in_memory("test_store"); + let provider = ProviderId::new("example-provider"); + let mut live = live_entry(); + live.opt_outs + .insert(provider.clone(), current_timestamp() + 86_400); + graph.create(&ec_id, &live).expect("should seed the row"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + true, + ); + let uid = KvUid::new( + UidValue::try_from("p-restored").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(provider.clone()), + Expiry::Never, + ); + ec_context.set_lookup_results(LookupResults::new( + &ec_id, + vec![ProviderLookup { + provider, + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("provider.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + )); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("provider.example.com"), + None, + None, + &mut response, + ); + + // Assert + let (entry, _) = graph + .get(&ec_id) + .expect("should read row") + .expect("should keep row"); + assert!( + !entry.ids.contains_key("provider.example.com"), + "should not restore the opted-out provider's UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 66d6f39b15..96e137fbb2 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -8,6 +8,8 @@ //! source the provider cleared on an opt-out is never refilled from the //! browser's `ts-eids` cookie. +use std::collections::btree_map; + use super::EcKvSnapshot; use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; use super::eid_types::{SourceDomain, UnixSeconds}; @@ -71,7 +73,7 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { let pruned = entry.clone(); let lookups_apply = input.lookups.is_for(input.ec_id); - apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); + let ingested = apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); let written = if lookups_apply { apply_lookups(&mut entry, input.lookups, input.now) } else { @@ -83,9 +85,10 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } Vec::new() }; - enforce_size_cap(&mut entry, &written); - // Compared after the cap, which may trim the only real change away. - let changed = entry != pruned; + let had_change = entry != pruned; + enforce_size_cap(&mut entry, &written, &ingested); + // The cap may trim the only real change away; trimming alone never writes. + let changed = had_change && entry != pruned; match expected { None => Some(RowChange::Create(entry)), @@ -95,11 +98,14 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } /// Fills missing sources from the browser, skipping provider-owned ones. +/// +/// Returns the sources it inserted, in insertion order. fn apply_browser_ingest( entry: &mut KvEntry, updates: &[PartnerIdUpdate], registry: &PartnerRegistry, -) { +) -> Vec { + let mut ingested = Vec::new(); for update in updates { if registry.is_provider_owned(&update.partner_id) { log::trace!( @@ -108,11 +114,12 @@ fn apply_browser_ingest( ); continue; } - entry - .ids - .entry(update.partner_id.clone()) - .or_insert_with(|| KvPartnerId::single(update.uid.clone())); + if let btree_map::Entry::Vacant(slot) = entry.ids.entry(update.partner_id.clone()) { + slot.insert(KvPartnerId::single(update.uid.clone())); + ingested.push(update.partner_id.clone()); + } } + ingested } fn apply_lookups( @@ -122,6 +129,17 @@ fn apply_lookups( ) -> Vec { let mut written = Vec::new(); for lookup in lookups.iter() { + // A newer request may have recorded this provider's opt-out after + // the lookup was dispatched; never restore its UIDs over it. + if entry.opted_out(&lookup.provider, now) + && lookup.status != LookupStatus::Skipped(SkipReason::OptedOut) + { + log::info!( + "EC write planner: discarding results of opted-out provider '{}'", + lookup.provider + ); + continue; + } for (source, eids) in &lookup.entries { if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { entry.ids.insert(source.as_str().to_owned(), record); @@ -162,7 +180,7 @@ fn member_len(source: &str, record: &KvPartnerId) -> usize { /// Tracks the row size by the bytes each drop removes and serializes the /// whole row again only when that estimate reaches the cap, so the cost is /// linear in the number of dropped UIDs and records. -fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { +fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain], ingested: &[String]) { let mut size = serialized_len(entry); // Drop this navigation's provider UIDs first, newest source last-in first. while size > MAX_ENTRY_BYTES { @@ -191,11 +209,28 @@ fn enforce_size_cap(entry: &mut KvEntry, written: &[SourceDomain]) { size = serialized_len(entry); } } + // Then this navigation's browser UIDs, newest first: the browser offers + // them again, while a stored record would be lost. + for source in ingested.iter().rev() { + if size <= MAX_ENTRY_BYTES { + break; + } + if let Some(record) = entry.ids.remove(source) { + log::info!( + "EC write planner: dropping browser UID for '{source}' to fit the row size cap" + ); + size = size.saturating_sub(member_len(source, &record)); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } + } + } if size <= MAX_ENTRY_BYTES { return; } - // No provider UIDs from this navigation are left: drop whole records, - // largest first, so every later write still fits the cap. + // Only a row that was already over the cap (written before the cap + // existed) gets here: drop whole records, largest first, so every later + // write still fits the cap. let mut by_size: Vec<(usize, String)> = entry .ids .iter() @@ -854,4 +889,101 @@ mod tests { assert_eq!(record.uid, "p-1", "should replace the fallback"); assert!(record.is_provider_written(), "should store provider eids"); } + + fn over_cap_legacy_entry() -> KvEntry { + let mut entry = live_entry(); + for i in 0..40 { + entry.ids.insert( + format!("source{i:02}.example.com"), + KvPartnerId::single(format!("{i:03}{}", "x".repeat(500))), + ); + } + entry + } + + #[test] + fn over_cap_row_with_no_change_plans_no_write() { + // Arrange: a row written before the cap existed. + let registry = registry(&[]); + let entry = over_cap_legacy_entry(); + assert!( + serde_json::to_vec(&entry).expect("should serialize").len() > MAX_ENTRY_BYTES, + "should start over the cap" + ); + let lookups = LookupResults::default(); + let snapshot = EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(entry), + generation: Some(1), + }; + + // Act + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not write when only the cap would trim" + ); + } + + #[test] + fn browser_ingest_never_evicts_a_stored_record_to_fit() { + // Arrange: a stored provider record plus fillers, just under the cap. + let registry = registry(&["ids.example.com"]); + let big: Vec = (0..18) + .map(|i| format!("{i:03}{}", "x".repeat(500))) + .collect(); + let ids: Vec<&str> = big.iter().map(String::as_str).collect(); + let mut entry = provider_entry(&ids); + let mut filler = 0; + loop { + let mut candidate = entry.clone(); + candidate.ids.insert( + format!("filler{filler:03}.example.com"), + KvPartnerId::single("f".repeat(100)), + ); + if serde_json::to_vec(&candidate) + .expect("should serialize") + .len() + > MAX_ENTRY_BYTES - 200 + { + break; + } + entry = candidate; + filler += 1; + } + let ingest = [PartnerIdUpdate::new("browser.example.com", "b".repeat(500))]; + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert: the browser UID is dropped, so nothing changes. + assert!( + change.is_none(), + "should drop the new browser UID rather than evict a stored record" + ); + } + + #[test] + fn found_result_never_restores_uids_over_a_newer_opt_out() { + // Arrange: another request recorded the opt-out after this lookup ran. + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry.opt_outs.insert( + ProviderId::new("example-provider"), + 2000 + OPT_OUT_MEMORY_SECONDS, + ); + let lookups = found("ids.example.com", &["p-restored"]); + + // Act + let change = plan_write(&input(&present(&entry), None, &[], &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not write the opted-out provider's older result" + ); + } } From 68903040e1c0f6c5f529b8341ac3e9f4ff9b0f4e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 00:46:06 +0530 Subject: [PATCH 42/58] Send provider lookups only for an EC whose consented row exists A well-formed ts-ec cookie without a row no longer reaches vendors, which also keeps cookie rotation from resetting the rate-limit bucket. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 66 ++++++++++++++----- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index c3f6a9a642..c14d789e1b 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -463,14 +463,15 @@ impl IdentityCoordinator for ProviderCoordinator

{ return pending; } let entry = lookup.snapshot.entry_for(lookup.ec_id); - // Missing EC or EID consent, a withdrawal tombstone, or an unreadable - // row (which may hide a tombstone) skips every provider: a lookup - // discloses the EC ID before finalize could decline it. + // Missing EC or EID consent, a withdrawal tombstone, an unreadable + // row (which may hide a tombstone) or no row at all skips every + // provider: a lookup discloses the EC ID before finalize could + // decline it, and an EC without a row may never be issued. let readable = !matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); let consented = readable && ec_consent_granted(lookup.consent) && allows_eid_persistence(lookup.consent) - && entry.is_none_or(|entry| entry.consent.ok); + && entry.is_some_and(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); for (index, registered) in self.providers.iter().enumerate() { @@ -844,6 +845,17 @@ mod tests { } } + /// A consented row with no IDs yet. + fn live_row() -> EcKvSnapshot { + let mut entry = crate::ec::kv_types::KvEntry::tombstone(1000); + entry.consent.ok = true; + EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(entry), + generation: Some(1), + } + } + fn run_with( coordinator: &ProviderCoordinator

, transport: &CannedTransport, @@ -897,7 +909,7 @@ mod tests { let coordinator = coordinator(""); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report the provider"); assert!( @@ -914,6 +926,28 @@ mod tests { assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); } + #[test] + fn ec_without_a_row_is_not_sent_to_vendors() { + // Arrange: a well-formed `ts-ec` cookie whose row does not exist. + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run(&coordinator, &transport, &missing(), &granting()); + + // Assert + assert_eq!( + results + .iter() + .next() + .expect("should report the provider") + .status, + LookupStatus::Skipped(SkipReason::Consent), + "should skip a provider for an EC with no row" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + fn provider_record(id: &str) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from(id).expect("should build uid"), @@ -1072,7 +1106,7 @@ mod tests { let coordinator = coordinator("tcf_vendor_id = 9999"); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert!( !matches!(only_status(&results), LookupStatus::Skipped(_)), @@ -1113,7 +1147,7 @@ mod tests { let coordinator = coordinator("tcf_vendor_id = 9999"); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &tcf_consent(vec![1])); + let results = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![1])); assert_eq!( only_status(&results), @@ -1131,7 +1165,7 @@ mod tests { let results = run( &coordinator, &transport, - &missing(), + &live_row(), &tcf_consent(vec![9999]), ); @@ -1149,7 +1183,7 @@ mod tests { )); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert_eq!( only_status(&results), @@ -1169,7 +1203,7 @@ mod tests { let results = run_with( &coordinator, &transport, - &missing(), + &live_row(), &granting(), &Limiter(true), ); @@ -1201,7 +1235,7 @@ mod tests { }; let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert_eq!( only_status(&results), @@ -1226,7 +1260,7 @@ mod tests { let coordinator = coordinator(""); let transport = CannedTransport::new(reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report"); assert_eq!(lookup.status, expected, "should map the outcome"); @@ -1436,7 +1470,7 @@ mod tests { let coordinator = coordinator(""); let transport = CannedTransport::new(optout_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report the provider"); assert_eq!( @@ -1487,7 +1521,7 @@ mod tests { .expect("should build coordinator"); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert_eq!( only_status(&results), @@ -1558,7 +1592,7 @@ mod tests { let coordinator = generating_coordinator(); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report the provider"); assert!( @@ -1578,7 +1612,7 @@ mod tests { let coordinator = generating_coordinator_for("IDS.example.com"); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report the provider"); assert!( From ea83a929c1a52367e5725ed97f6a36b2f5fb81f9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:21:34 +0530 Subject: [PATCH 43/58] Test the identity phases through the real adapter path and pass the raw client IP run_after_response_lookups now takes the app state, and the unreachable call from the startup-error branch is gone. New Viceroy tests route navigations through an app state holding the echo provider and cover a live row, a minted EC, a changed EC, a missing permit and a row withdrawn before the phase, plus the page phase for returning and minted ECs. Providers now receive the trusted raw client address instead of the normalized EC-hash input, which dropped IPv6 addresses. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 401 +++++++++++++----- .../trusted-server-adapter-fastly/src/main.rs | 35 +- 2 files changed, 317 insertions(+), 119 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 82e9aee082..b668fa1568 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -432,10 +432,9 @@ struct ProviderRequestSignals<'a> { impl<'a> ProviderRequestSignals<'a> { fn read(req: &'a Request, ec: &EcRequestState) -> Self { Self { - client_ip: ec - .ec_context - .client_ip() - .and_then(|ip| ip.parse::().ok()), + // The trusted raw address: `EcContext::client_ip` is the + // normalized EC-hash input, which drops IPv6 separators. + client_ip: ec.services.client_info().client_ip, user_agent: req .headers() .get(header::USER_AGENT) @@ -1624,21 +1623,15 @@ mod tests { use futures::executor::block_on; use std::borrow::Cow; use std::num::NonZeroUsize; - use trusted_server_core::consent::ConsentContext; - use trusted_server_core::consent::jurisdiction::Jurisdiction; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; - use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; - use trusted_server_core::ec::coordinator::{ - IdentityCoordinator as _, LookupContext, ProviderCoordinator, RegisteredProvider, - }; + use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; use trusted_server_core::ec::device::DeviceSignals; - use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; + use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ - ClientSignals, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, }; - use trusted_server_core::ec::rate_limiter::RateLimiter; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::{ @@ -2917,7 +2910,9 @@ mod tests { } /// Returns `p-1` for the lookup it was sent, then reports nothing left. - struct LookupHttpClient; + /// Answers every lookup with `p-1` and counts the lookups sent. + #[derive(Default)] + struct LookupHttpClient(Arc); #[async_trait::async_trait(?Send)] impl PlatformHttpClient for LookupHttpClient { @@ -2932,6 +2927,7 @@ mod tests { &self, request: PlatformHttpRequest, ) -> Result> { + self.0.fetch_add(1, Ordering::SeqCst); Ok(PlatformPendingRequest::new(()).with_backend_name(request.backend_name)) } @@ -3010,20 +3006,20 @@ mod tests { } } - struct AllowAll; + struct AfterResponseFixture { + state: Arc, + sends: Arc, + graph: trusted_server_core::ec::kv::KvIdentityGraph, + } - impl RateLimiter for AllowAll { - fn exceeded( - &self, - _key: &str, - _hourly_limit: u32, - ) -> Result> { - Ok(false) - } + /// An app state whose after-response phase runs the echo provider. + fn after_response_fixture() -> AfterResponseFixture { + identity_fixture(true) } - #[test] - fn collected_results_reach_the_page_load_write() { + /// An app state that runs the echo provider in the page or the + /// after-response phase. + fn identity_fixture(after_response: bool) -> AfterResponseFixture { let provider_config: IdentityProviderConfig = toml::from_str( "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", ) @@ -3037,7 +3033,6 @@ mod tests { settings.ec.partners = vec![partner]; // This build registers no kind, so the app state is built without the // provider and the test supplies its registry and coordinator. - let router = TrustedServerApp::routes_for_state(&app_state_for_settings(settings.clone())); let partners = { let mut with_provider = settings.ec.clone(); with_provider.identity_providers = vec![provider_config.clone()]; @@ -3046,37 +3041,6 @@ mod tests { .expect("should build registry"), ) }; - let app_state = AppState { - partner_registry: Arc::clone(&partners), - ..Arc::into_inner(app_state_for_settings(settings.clone())) - .expect("should own the app state") - }; - let mut request = navigation_request("/article"); - // Viceroy geolocates loopback to a US privacy state; a US Privacy - // string without a sale opt-out lets the navigation mint an EC. - request.extensions_mut().insert(ClientInfo { - client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), - ..ClientInfo::default() - }); - request - .headers_mut() - .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); - let mut response = route(&router, request); - let mut state = take_finalize_state(&mut response); - let ec_id = state - .ec_context - .ec_value() - .expect("should generate an EC") - .to_owned(); - state.services = RuntimeServices::builder() - .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) - .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) - .kv_store(Arc::new(NoopKvStore) as Arc) - .backend(Arc::new(FixedBackend)) - .http_client(Arc::new(LookupHttpClient)) - .geo(Arc::new(crate::platform::FastlyPlatformGeo)) - .client_info(ClientInfo::default()) - .build(); let provider: Box = Box::new(EchoProvider(ProviderCapabilities { sources: Box::new([SourceCapability { source: SourceDomain::try_from("ids.example.com").expect("should parse source"), @@ -3090,58 +3054,299 @@ mod tests { })); let coordinator = ProviderCoordinator::new( vec![RegisteredProvider::new(provider_config, provider)], - partners, + Arc::clone(&partners), Duration::from_millis(200), ) .expect("should build coordinator"); - let transport = PlatformLookupTransport::new(&state.services); - let consent = ConsentContext { - jurisdiction: Jurisdiction::NonRegulated, - ..Default::default() + let graph = crate::require_identity_graph(&settings).expect("should open identity graph"); + let coordinator = Some(Arc::new(coordinator)); + let (page_identity, after_response_identity) = if after_response { + (None, coordinator) + } else { + (coordinator, None) }; - let pending = block_on(coordinator.dispatch( - &transport, - &LookupContext { - ec_id: &ec_id, - snapshot: state.ec_context.kv_snapshot(), - consent: &consent, - client: ClientSignals { - ip: None, - user_agent: None, - }, - publisher_domain: "test-publisher.com", - rate_limiter: &AllowAll, - now: UnixSeconds::new(1_700_000_000), - }, - )); + let state = Arc::new(AppState { + partner_registry: partners, + page_identity, + after_response_identity, + ..Arc::into_inner(app_state_for_settings(settings)).expect("should own the app state") + }); + AfterResponseFixture { + state, + sends: Arc::default(), + graph, + } + } - let result = crate::apply_edgezero_ec_finalize( - &settings, - Some(&app_state), - &mut state, - Some(super::IdentityLookups { - coordinator: Arc::new(coordinator), - pending, - }), + fn seed_live_row(graph: &trusted_server_core::ec::kv::KvIdentityGraph, ec_id: &str) { + let mut entry = trusted_server_core::ec::kv_types::KvEntry::tombstone( + trusted_server_core::ec::current_timestamp(), + ); + entry.consent.ok = true; + graph.create(ec_id, &entry).expect("should seed the row"); + } + + /// Routes one navigation, finalizes it and returns its after-response work. + fn finalized_navigation( + fixture: &AfterResponseFixture, + ec_id: Option<&str>, + client_ip: IpAddr, + ) -> (super::EcFinalizeState, Option) { + let (ec_state, _, lookups) = finalized_navigation_with_page(fixture, ec_id, client_ip); + (ec_state, lookups) + } + + /// Like [`finalized_navigation`], and also reports whether page-phase + /// lookups were dispatched. + fn finalized_navigation_with_page( + fixture: &AfterResponseFixture, + ec_id: Option<&str>, + client_ip: IpAddr, + ) -> ( + super::EcFinalizeState, + bool, + Option, + ) { + let router = TrustedServerApp::routes_for_state(&fixture.state); + let mut request = navigation_request("/article"); + request.extensions_mut().insert(ClientInfo { + client_ip: Some(client_ip), + ..ClientInfo::default() + }); + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out grants consent. + let cookie = ec_id.map_or_else( + || "us_privacy=1YNN".to_owned(), + |ec_id| format!("us_privacy=1YNN; ts-ec={ec_id}"), + ); + request.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(&cookie).expect("should build cookie header"), + ); + super::take_navigation_lookups(); + let mut response = route(&router, request); + let navigation = super::take_navigation_lookups(); + let lookups = navigation.after_response; + let page = navigation.page; + let page_dispatched = page.is_some(); + let mut ec_state = take_finalize_state(&mut response); + ec_state.services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(LookupHttpClient(Arc::clone(&fixture.sends)))) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + crate::apply_edgezero_ec_finalize( + &fixture.state.settings, + Some(&fixture.state), + &mut ec_state, + page, &mut response, + ) + .expect("should finalize"); + (ec_state, page_dispatched, lookups) + } + + #[test] + fn page_phase_lookups_reach_the_write_for_a_returning_ec() { + // Arrange + let fixture = identity_fixture(false); + let ec_id = format!("{}.pg0001", "6".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + + // Act + let (_, dispatched, _) = + finalized_navigation_with_page(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + + // Assert + assert!(dispatched, "should dispatch page-phase lookups"); + assert_eq!( + owned_uid(&fixture, &ec_id).as_deref(), + Some("p-1"), + "should write the provider UID in the page-load write" ); + } + + #[test] + fn page_phase_lookups_skip_a_newly_minted_ec() { + // Arrange + let fixture = identity_fixture(false); - result.expect("should finalize"); + // Act + let (ec_state, dispatched, _) = + finalized_navigation_with_page(&fixture, None, IpAddr::V4(Ipv4Addr::LOCALHOST)); + + // Assert assert!( - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .any(|value| value.to_str().is_ok_and(|cookie| cookie.contains(&ec_id))), - "should set the EC cookie after the page-load write" + ec_state.ec_context.ec_value().is_some(), + "should mint an EC" ); - let graph = crate::require_identity_graph(&settings).expect("should open identity graph"); - let snapshot = graph.load_snapshot(&ec_id); - let entry = snapshot.entry_for(&ec_id).expect("should store the row"); - let row = serde_json::to_string(entry).expect("should serialize row"); + assert!(!dispatched, "should not send a newly minted EC to vendors"); + } + + fn owned_uid(fixture: &AfterResponseFixture, ec_id: &str) -> Option { + let snapshot = fixture.graph.load_snapshot(ec_id); + snapshot + .entry_for(ec_id) + .and_then(|entry| entry.ids.get("ids.example.com")) + .map(|record| record.uid.clone()) + } + + #[test] + fn after_response_lookups_capture_the_raw_ipv6_address() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0006", "7".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let ipv6 = IpAddr::V6(std::net::Ipv6Addr::new( + 0x2001, 0xdb8, 0xabcd, 1, 0, 0, 0, 0x1234, + )); + + // Act + let (_, lookups) = finalized_navigation(&fixture, Some(&ec_id), ipv6); + + // Assert + assert_eq!( + lookups.and_then(|lookups| lookups.client_ip), + Some(ipv6), + "should pass the raw IPv6 address to providers" + ); + } + + #[test] + fn after_response_lookups_write_to_a_live_row() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0001", "1".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + + // Act + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 1, + "should send one lookup" + ); + assert_eq!( + owned_uid(&fixture, &ec_id).as_deref(), + Some("p-1"), + "should write the provider UID to the row" + ); + } + + #[test] + fn after_response_lookups_skip_a_newly_minted_ec() { + // Arrange + let fixture = after_response_fixture(); + + // Act + let (mut ec_state, lookups) = + finalized_navigation(&fixture, None, IpAddr::V4(Ipv4Addr::LOCALHOST)); + let ec_id = ec_state + .ec_context + .ec_value() + .expect("should mint an EC") + .to_owned(); + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + assert_eq!( + owned_uid(&fixture, &ec_id), + None, + "should not write provider UIDs" + ); + } + + #[test] + fn after_response_lookups_skip_when_the_ec_changed_since_capture() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0002", "2".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + let mut lookups = lookups.expect("should capture after-response work"); + lookups.ec_id = format!("{}.ar0003", "3".repeat(64)); + + // Act + crate::run_after_response_lookups(&fixture.state, &mut ec_state, Some(lookups)); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + } + + #[test] + fn after_response_lookups_skip_without_a_permit() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0004", "4".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); assert!( - row.contains("ids.example.com") && row.contains("p-1"), - "should write the provider EID into the row: {row}" + ec_state.ec_context.take_after_response_permit().is_some(), + "should have issued a permit" + ); + + // Act + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + } + + #[test] + fn after_response_lookups_skip_a_row_withdrawn_before_the_phase() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0005", "5".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + fixture.graph.delete(&ec_id).expect("should delete the row"); + fixture + .graph + .create( + &ec_id, + &trusted_server_core::ec::kv_types::KvEntry::tombstone( + trusted_server_core::ec::current_timestamp(), + ), + ) + .expect("should write the tombstone"); + + // Act + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + assert_eq!( + owned_uid(&fixture, &ec_id), + None, + "should keep the tombstone" ); } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index cc8ba854e8..6aafcc9c40 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -577,13 +577,9 @@ fn edgezero_main( counters.as_ref(), ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); - run_after_response_lookups( - settings, - app_state.as_deref(), - &partner_registry, - &mut ec_state, - after_response, - ); + if let Some(state) = app_state.as_deref() { + run_after_response_lookups(state, &mut ec_state, after_response); + } return; } Err(e) => { @@ -613,13 +609,6 @@ fn edgezero_main( &partner_registry, &ec_state, ); - run_after_response_lookups( - &settings, - None, - &partner_registry, - &mut ec_state, - after_response, - ); return; } Err(e) => { @@ -764,10 +753,8 @@ fn run_edgezero_pull_sync_after_send( /// the page. Skipped unless finalize confirmed the navigation (it issues the /// follow-up permit) and the EC is still the one the lookups were prepared /// for. -fn run_after_response_lookups( - settings: &Settings, - state: Option<&AppState>, - partner_registry: &Arc, +pub(crate) fn run_after_response_lookups( + state: &AppState, ec_state: &mut EcFinalizeState, lookups: Option, ) { @@ -784,9 +771,10 @@ fn run_after_response_lookups( log::debug!("After-response lookups skipped: EC changed during the request"); return; } - let Some(coordinator) = state.and_then(|state| state.after_response_identity.as_ref()) else { + let Some(coordinator) = state.after_response_identity.as_ref() else { return; }; + let settings = &*state.settings; let Some(graph) = maybe_identity_graph(settings) else { return; }; @@ -821,8 +809,13 @@ fn run_after_response_lookups( return; } let results = futures::executor::block_on(coordinator.collect(&transport, pending)); - let outcome = - commit_after_response_lookups(&graph, permit, partner_registry, &lookups.ec_id, &results); + let outcome = commit_after_response_lookups( + &graph, + permit, + &state.partner_registry, + &lookups.ec_id, + &results, + ); log::info!("After-response lookups: write outcome={outcome}"); } From 95dc52e18e8afe674286de20137efceabf4af02c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:21:34 +0530 Subject: [PATCH 44/58] Reject enabled providers at deploy time and skip disabled providers' secrets This release registers no provider kind, so every runtime refuses an enabled provider; deploy validation now refuses it too, so a config push cannot take a service down. A disabled provider's token reference is no longer resolved when the config blob loads. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/config.rs | 36 ++++++++++++++++++- .../trusted-server-core/src/config_payload.rs | 35 ++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cb98956130..1dddef193f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -281,9 +281,29 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report> { + match ec.identity_providers.iter().find(|config| config.enabled) { + Some(config) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.identity_providers: provider `{}` is enabled, but this release registers no \ + provider kinds; set `enabled = false`", + config.id + ), + })), + None => Ok(()), + } +} + /// Runs Trusted Server runtime validation after secret references are resolved. /// /// # Errors @@ -1374,9 +1394,22 @@ gam_network_id = "99999" validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); } + #[test] + fn deploy_validation_rejects_an_enabled_provider_while_no_kind_is_registered() { + let settings = settings_with_identity_provider(serde_json::json!({})); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject an enabled provider at deploy time"); + + assert!( + err.to_string().contains("registers no provider kinds"), + "should explain why the provider is rejected: {err:?}" + ); + } + #[test] fn deploy_validation_accepts_provider_with_unresolved_partner_token() { - let mut settings = settings_with_identity_provider(serde_json::json!({})); + let mut settings = settings_with_identity_provider(serde_json::json!({ "enabled": false })); settings.ec.partners[0].api_token = Some(Redacted::new("example_partner_api_token".to_owned())); @@ -1450,6 +1483,7 @@ gam_network_id = "99999" #[test] fn app_config_keeps_provider_token_key_reference() { let settings = settings_with_identity_provider(serde_json::json!({ + "enabled": false, "secrets": { "token": "example_provider_token" } })); diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index dd7545198b..c2cd900102 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -98,6 +98,28 @@ fn remove_inactive_secret_references(data: &mut serde_json::Value) { } } + if let Some(providers) = data + .pointer_mut("/ec/identity_providers") + .and_then(serde_json::Value::as_array_mut) + { + for provider in providers { + let Some(provider) = provider.as_object_mut() else { + continue; + }; + // `enabled` defaults to true, so only an explicit false disables. + let disabled = provider + .get("enabled") + .is_some_and(|enabled| !json_bool_or_string_is_true(Some(enabled))); + if disabled + && let Some(secrets) = provider + .get_mut("secrets") + .and_then(serde_json::Value::as_object_mut) + { + secrets.remove("token"); + } + } + } + let Some(datadome) = data .pointer_mut("/integrations/datadome") .and_then(serde_json::Value::as_object_mut) @@ -742,6 +764,12 @@ mod tests { .ec .partners .push(partner_with_pull_sync(false, "unused-partner-pull-token")); + original.ec.identity_providers.push( + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nenabled = false\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]\nsecrets = { token = \"unused-provider-token\" }", + ) + .expect("should parse disabled provider"), + ); let reconstructed = settings_from_config_blob( &envelope_json(&original), @@ -750,6 +778,13 @@ mod tests { ) .expect("should skip inactive optional feature references"); + assert!( + reconstructed.ec.identity_providers[0] + .secrets + .token + .is_none(), + "should not resolve a disabled provider's token" + ); assert!(reconstructed.tinybird.auction_token_secret.is_none()); assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); let datadome = reconstructed From bccf8d5f750dfed28a55cafbd27e9d3bcbf3f922 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:21:35 +0530 Subject: [PATCH 45/58] Skip the after-response re-read when no lookup can change the row Signed-off-by: dhruv8sh --- .../src/ec/after_response.rs | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index e14de5eb18..bc6385634e 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -7,7 +7,7 @@ //! next page load. Browser ingest never runs here: it belongs to the //! navigation's own write. -use super::coordinator::results::LookupResults; +use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; use super::eid_types::UnixSeconds; use super::kv::KvIdentityGraph; use super::page_load::{AfterResponseWrite, RowChange}; @@ -73,7 +73,14 @@ pub fn commit_after_response_lookups( ec_id: &str, lookups: &LookupResults, ) -> AfterResponseOutcome { - if lookups.is_empty() { + // Only found UIDs, cleared sources or an opt-out can change the row; a + // navigation whose lookups were all skipped or failed needs no re-read. + let can_change = lookups.iter().any(|lookup| { + !lookup.entries.is_empty() + || !lookup.cleared.is_empty() + || lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) + }); + if !can_change { return AfterResponseOutcome::Unchanged; } let snapshot = &graph.load_snapshot(ec_id); @@ -126,7 +133,7 @@ pub fn commit_after_response_lookups( #[cfg(test)] mod tests { use super::*; - use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup}; + use crate::ec::coordinator::results::{DropCounts, ProviderLookup}; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::TombstoneOutcome; use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; @@ -488,6 +495,37 @@ mod tests { ); } + #[test] + fn skipped_results_need_no_re_read() { + // Arrange: no row, so a re-read would report `NoRow`. + let graph = KvIdentityGraph::in_memory("after-response"); + let skipped = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &skipped, + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should return before reading the row" + ); + } + #[test] fn lookups_run_only_for_an_existing_consented_row() { // Arrange From fb2b5694c0a61090243842f010b7cb34ddb5c358 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:21:35 +0530 Subject: [PATCH 46/58] Correct stale comments on snapshots, tombstones and opt-out memory Signed-off-by: dhruv8sh --- .../src/platform/deadline.rs | 7 ++++--- crates/trusted-server-core/src/ec/kv.rs | 11 +++++------ crates/trusted-server-core/src/ec/kv_types.rs | 2 +- crates/trusted-server-core/src/ec/page_load.rs | 8 +++++--- crates/trusted-server-core/src/platform/http.rs | 7 ++++--- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index 6b074f385c..f13022c228 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -262,8 +262,9 @@ pub(super) fn select_until( } let expired_at_entry = Instant::now() >= deadline; if expired_at_entry { - // Take one nonblocking snapshot. Fully buffered results are cached; - // responses that would require a wait are abandoned, never retried. + // Take one final snapshot. Fully buffered results are cached; each + // body may wait once, at most `SNAPSHOT_SURFACE_TIMEOUT_MS`, for bytes + // that already arrived. Anything still pending is abandoned. for index in 0..pending.len() { let mut surfaced = false; loop { @@ -307,7 +308,7 @@ pub(super) fn select_until( for index in 0..pending.len() { if !expired_at_entry && Instant::now() >= deadline { // A timed host wait may wake with a ready event at the cutoff. - // Finish with the same nonblocking snapshot as expired entry. + // Finish with the same final snapshot as expired entry. return select_until( pending .into_iter() diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 3d2c40f10c..10dc89f686 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -384,7 +384,7 @@ impl KvIdentityGraph { /// /// The `permit` is consumed, so a navigation cannot write twice. Consent /// withdrawals do not go through here: their tombstone is a privacy write - /// on any request ([`Self::tombstone_existing_from_snapshot`]). + /// on any request (`tombstone_existing_from_snapshot`). /// /// # Errors /// @@ -558,8 +558,8 @@ impl KvIdentityGraph { /// Merges partner IDs using request-scoped persisted state as the first CAS input. /// /// A caller-supplied `Present` snapshot is *proof* that the row exists — - /// either an `Add`-confirmed create from [`generate_if_needed`] or an - /// earlier authoritative read in the same request. Partner-ID enrichment is + /// an earlier authoritative read or a confirmed write in the same + /// request. Partner-ID enrichment is /// best effort, so a refresh that reads absent or unreadable never /// downgrades that proof: the update is skipped and logged, and the proven /// snapshot is returned so cookie issuance continues from the confirmed @@ -570,8 +570,6 @@ impl KvIdentityGraph { /// updates are dropped, because a point read on an eventually-consistent /// store cannot prove absence and routes without orphan recovery have no /// later chance to retry. - /// - /// [`generate_if_needed`]: super::generate_if_needed pub(crate) fn upsert_partner_ids_from_snapshot( &self, ec_id: &str, @@ -1445,7 +1443,8 @@ impl KvIdentityGraph { /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). Consent - /// withdrawal uses a page-load tombstone via [`Self::commit_page_load`] instead. + /// withdrawal writes a tombstone through `tombstone_existing_from_snapshot` + /// instead. /// /// # Errors /// diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index f13b258dfc..3772f07f3c 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -50,7 +50,7 @@ pub const MAX_UID_LENGTH: usize = 512; pub const MAX_STORED_UID_LENGTH: usize = 1024; /// How long a provider opt-out keeps that provider from being called again, -/// in seconds. Matches the longest lifetime a provider UID may have. +/// in seconds (30 days). pub const OPT_OUT_MEMORY_SECONDS: u64 = 30 * 24 * 60 * 60; /// Full KV entry stored as the body of an EC identity graph record. diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs index f530127bcd..38125a0f10 100644 --- a/crates/trusted-server-core/src/ec/page_load.rs +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -6,9 +6,11 @@ //! Neither type is `Clone`, so one navigation makes at most one write before //! its response. //! -//! The one exception is [`AfterResponseWrite`], confirmed alongside the page -//! load's permit. It allows one follow-up update after the response has been -//! sent, carrying only the results of after-response identity lookups. +//! Two writes fall outside this rule. [`AfterResponseWrite`], confirmed +//! alongside the page load's permit, allows one follow-up update after the +//! response has been sent, carrying only the results of after-response +//! identity lookups. A consent withdrawal's tombstone is a privacy write on +//! any request that carries the EC cookie, and takes no permit. //! //! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index e1fe8ff1f2..a93753653e 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -414,8 +414,9 @@ pub trait PlatformHttpClient: Send + Sync { /// Collects an available response, waiting only until an absolute deadline. /// - /// On expiry, implementations take one nonblocking snapshot of complete - /// available responses before relinquishing unfinished requests. This + /// On expiry, implementations take one final snapshot of complete + /// available responses before relinquishing unfinished requests; it may + /// wait briefly for bytes the host already received. This /// preserves buffered bids when a publisher origin delays collection. /// Subsequent calls may drain only the completed snapshot. The deadline /// bounds network waiting, not eligibility by an upstream arrival timestamp. @@ -428,7 +429,7 @@ pub trait PlatformHttpClient: Send + Sync { /// /// Complete response bodies are buffered within adapter-specific limits. /// Deadline expiry drops incomplete bodies and pending handles after the - /// nonblocking snapshot; complete buffered responses remain collectible. + /// final snapshot; complete buffered responses remain collectible. /// /// # Examples /// From e789617703814c85b7deee594a8981706f536b00 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:23:50 +0530 Subject: [PATCH 47/58] Keep a disabled provider's vendor-consent gate on its stored UIDs A provider set to enabled = false no longer owns its sources, but its TCF vendor ID still gates egress of the UIDs it stored to auctions, x-ts-eids and /identify. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/registry.rs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index c82f6c0bd7..5c62b02c9a 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -255,6 +255,13 @@ impl PartnerRegistry { registry.mark_provider_owned(source, provider.tcf_vendor_id); } } + // A disabled provider's stored UIDs stay behind its vendor consent, + // though its sources go back to browser ingest. + for provider in ec.identity_providers.iter().filter(|p| !p.enabled) { + for source in &provider.sources { + registry.keep_vendor_gate(source, provider.tcf_vendor_id); + } + } Ok(registry) } @@ -305,6 +312,21 @@ impl PartnerRegistry { } } + /// Records a disabled provider's TCF vendor ID for `source` unless an + /// enabled provider already owns it, so egress still checks that vendor's + /// consent for the UIDs the provider stored. + fn keep_vendor_gate(&mut self, source: &str, tcf_vendor_id: Option) { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return; + }; + if let Some(partner) = self.by_source_domain.get_mut(&normalized) + && !partner.provider_owned + && partner.owner_tcf_vendor_id.is_none() + { + partner.owner_tcf_vendor_id = tcf_vendor_id; + } + } + /// Whether an enabled identity provider owns `source`. #[must_use] pub fn is_provider_owned(&self, source: &str) -> bool { @@ -532,7 +554,7 @@ mod tests { ) .expect("should parse enabled provider"), toml::from_str( - "id = \"b\"\nkind = \"k\"\nenabled = false\nendpoint = \"https://ids.example.com/\"\nsources = [\"other.example.com\"]", + "id = \"b\"\nkind = \"k\"\nenabled = false\ntcf_vendor_id = 42\nendpoint = \"https://ids.example.com/\"\nsources = [\"other.example.com\"]", ) .expect("should parse disabled provider"), ]; @@ -548,6 +570,14 @@ mod tests { !registry.is_provider_owned("other.example.com"), "should not mark disabled provider source" ); + assert_eq!( + registry + .get("other.example.com") + .and_then(|partner| partner.owner_tcf_vendor_id) + .map(NonZeroU16::get), + Some(42), + "should keep the disabled provider's vendor consent gate" + ); } #[test] From c6397bee5b88fc4820304d417998218676c4d522 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:23:50 +0530 Subject: [PATCH 48/58] Bound provider ext nesting to 32 levels Whole-row parses such as the admin view add about eight levels on top of an ext, so an ext under serde_json's own limit could still make the admin view fall back to the raw row. Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/eid_types.rs | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/eid_types.rs b/crates/trusted-server-core/src/ec/eid_types.rs index 9c9826e5d3..74b54aca88 100644 --- a/crates/trusted-server-core/src/ec/eid_types.rs +++ b/crates/trusted-server-core/src/ec/eid_types.rs @@ -17,6 +17,41 @@ use super::partner::normalize_partner_source_domain; /// Maximum serialized size of an `ext` object, in bytes. pub const MAX_EXT_BYTES: usize = 1024; +/// Maximum nesting depth of an `ext` object. +/// +/// Keeps every whole-row or whole-request parse that embeds the `ext` well +/// under `serde_json`'s 128-level recursion limit. +pub const MAX_EXT_DEPTH: usize = 32; + +/// Returns the deepest object or array nesting in JSON `text`. +/// +/// Iterative, so it cannot overflow the stack on hostile input. +fn json_depth(text: &str) -> usize { + let (mut depth, mut deepest) = (0_usize, 0_usize); + let (mut in_string, mut escaped) = (false, false); + for byte in text.bytes() { + if in_string { + match byte { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'"' => in_string = false, + _ => {} + } + continue; + } + match byte { + b'"' => in_string = true, + b'{' | b'[' => { + depth += 1; + deepest = deepest.max(depth); + } + b'}' | b']' => depth = depth.saturating_sub(1), + _ => {} + } + } + deepest +} + /// Maximum size of `inserter` and `matcher`, in bytes. pub const MAX_EID_FIELD_BYTES: usize = 255; @@ -377,7 +412,10 @@ impl PartialEq for RawJson { impl RawJson { fn check(value: &RawValue) -> Result<(), EidValidationError> { let text = value.get(); - if text.len() > MAX_EXT_BYTES || !text.trim_start().starts_with('{') { + if text.len() > MAX_EXT_BYTES + || !text.trim_start().starts_with('{') + || json_depth(text) > MAX_EXT_DEPTH + { return Err(EidValidationError::InvalidExt); } // `RawValue` skips nested values without serde_json's recursion limit, @@ -557,6 +595,30 @@ mod tests { ); } + #[test] + fn raw_json_bounds_ext_nesting_depth() { + let nested = |depth: usize| -> Box { + let text = format!("{}1{}", r#"{"a":"#.repeat(depth), "}".repeat(depth)); + serde_json::from_str(&text).expect("should read JSON as a raw value") + }; + + assert!( + RawJson::try_from(nested(MAX_EXT_DEPTH)).is_ok(), + "should accept an ext at the depth limit" + ); + assert!( + RawJson::try_from(nested(MAX_EXT_DEPTH + 1)).is_err(), + "should reject an ext past the depth limit" + ); + let braces_in_string: Box = + serde_json::from_str(&format!(r#"{{"a":"{}\""}}"#, "{[".repeat(50))) + .expect("should read JSON as a raw value"); + assert!( + RawJson::try_from(braces_in_string).is_ok(), + "should not count brackets inside strings" + ); + } + #[test] fn raw_json_compares_by_bytes() { let a = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); From bcd47331df1aac07d0e6cfee31b87281a94e1e4d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:27:44 +0530 Subject: [PATCH 49/58] Restore main's direct tests of the withdrawal tombstone Covers CAS retry, CAS exhaustion, a generation-unavailable snapshot, a store failure, a row that disappears on retry, a failed snapshot read and failing completion-marker operations. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/kv.rs | 260 ++++++++++++++++++++++++ 1 file changed, 260 insertions(+) diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 10dc89f686..f118e59951 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -3492,4 +3492,264 @@ mod tests { "hard delete should remove withdrawal completion" ); } + + /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition + /// and deletes the key, simulating a concurrent withdrawal that removes the + /// row between this writer's read and its write. + struct DisappearOnConflictEcKv { + inner: InMemoryEcKv, + conflicts_remaining: std::sync::Mutex, + } + + impl DisappearOnConflictEcKv { + fn new(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("disappear-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + } + } + + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize seeded entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed live entry"); + } + } + + impl EcKvStore for DisappearOnConflictEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + let mut remaining = self + .conflicts_remaining + .lock() + .expect("should lock conflict counter"); + if *remaining > 0 { + *remaining -= 1; + self.inner.delete(key).expect("should delete on conflict"); + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + #[test] + fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::with_marker_failures( + Arc::clone(&operations), + 1, + )); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); + + let outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "marker failures must not suppress the withdrawal write" + ); + assert_eq!( + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::Overwrite, + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + ], + "failed completion recording should still be attempted after the root write" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read stored row") + .expect("should preserve the root"); + assert!(!stored.consent.ok, "root should remain tombstoned"); + } + + #[test] + fn tombstone_existing_from_snapshot_retries_cas_conflict() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should retry the conflict and persist the tombstone" + ); + } + + #[test] + fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { + // A generation-unavailable snapshot refreshes once before its CAS. That + // refresh must not spend a CAS attempt, so a withdrawal tombstone still + // persists after four conflicts and a successful fifth write. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { + // Every CAS attempt loses its race, so the row stays live with consent + // granted while the browser cookie is already cleared. The caller must + // see a failure it can report rather than a silent no-op. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "CAS exhaustion must report a failed withdrawal" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + stored.consent.ok, + "the row is still live, which is exactly why the failure must be reported" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_store_failure_returns_failed() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a failed tombstone write should return failed snapshot state" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { + let store = DisappearOnConflictEcKv::new(1); + store.seed_live(&snapshot_ec_id()); + let graph = KvIdentityGraph::new(store); + let ec_id = snapshot_ec_id(); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a row that disappears during retry becomes a no-op" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "must not recreate the disappeared key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { + // A prior request-scoped read failed, so the snapshot is `Failed`. A + // withdrawal must not silently drop consent removal: re-read the store + // and tombstone the row if it is authoritatively present. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "a failed snapshot must re-read and persist the tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "withdrawal must reach the store"); + } } From a52b512e00549d55c151c5601d30f47a24b4fc4f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:29:34 +0530 Subject: [PATCH 50/58] Bring operator docs in line with minting, withdrawal and provider rules Documents redirect minting, the 16 KiB row cap, the snapshot wording, the deploy rejection of enabled providers, cookie retention on a failed tombstone, rollback loss of eids and opt-outs, and after-response lookups behind error responses. Signed-off-by: dhruv8sh --- CHANGELOG.md | 14 ++++++++------ docs/guide/api-reference.md | 4 ++-- docs/guide/auction-orchestration.md | 2 +- docs/guide/configuration.md | 20 +++++++++++++++----- docs/guide/edge-cookies.md | 6 +++--- docs/guide/integrations/prebid.md | 4 ++-- trusted-server.example.toml | 5 ++--- 7 files changed, 33 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1dc4349f8..42413f4177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,15 +19,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 rejects responses collected after its cutoff. Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. On Fastly the final snapshot may wait up to 1 ms per partially received body so - the host can surface bytes that already arrived, and keeps a response whose - headers arrived by the cutoff. + the host can surface bytes that already arrived, and reads on into the body of + a response whose headers arrived by the cutoff instead of abandoning it. - Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, pull sync skips them, and browser IDs never fill them. Set `tcf_vendor_id` to gate both lookups and the provider's stored IDs in bidstream and `/identify` on TCF vendor consent. See [Identity providers](docs/guide/configuration.md#identity-providers). -- Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its size limit, and continues with the remaining groups. -- **Breaking:** An EC is now minted, and the EC row written, only on a top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, integration proxies and `/auction` no longer mint one, and prefetches and prerenders never mint or write. Each navigation makes at most one conditional page-load write; a consent withdrawal's tombstone is still written on any request that carries the EC cookie. +- Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its new 16 KiB size limit, and continues with the remaining groups. +- **Breaking:** An EC is now minted, and the EC row written, only on a top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, a navigation answered with a 301, 302, 303, 307 or 308 redirect mints none, integration proxies no longer mint one, and prefetches and prerenders never mint or write. Each navigation makes at most one conditional page-load write; a consent withdrawal's tombstone is still written on any request that carries the EC cookie. - **Breaking:** Browser EIDs from the `/auction` request body are no longer stored, and `/auction` and `/_ts/page-bids` no longer write browser EIDs from `ts-eids` or `sharedId`. Those cookies are ingested on the next top-level navigation. -- **Breaking:** An EC minted on the current navigation is no longer sent as `user.id` to bidders, and identity providers are not called for it, until a later navigation finds its row. +- **Breaking:** An EC minted on the current navigation is no longer sent as `user.id` to bidders, that navigation's auction does not carry the browser's `ts-eids`, and identity providers are not called for it, until a later navigation finds its row. Identity providers are also not called for an EC cookie whose row does not exist. +- This release registers no identity provider kind, so `ts config push` rejects an enabled `[[ec.identity_providers]]` entry and every runtime refuses to start with one; configure providers with `enabled = false` until a kind ships. +- A consent withdrawal keeps the `ts-ec` cookie while its tombstone cannot be written (after bounded CAS retries, or when the row cannot be read), so the next request carrying the cookie retries. Previously the cookie was expired first. - **Breaking:** Axum, Cloudflare and Spin refuse to start when an enabled identity provider is configured, and Fastly refuses to start when an enabled provider names a `kind` this build does not include. -- **Breaking (rollback):** Stored provider UIDs may be up to 1024 bytes (browser, batch and pull ingest stay at 512). A binary from before this release cannot read an EC row holding a longer UID, so roll back only after such rows expire or are rewritten. A config blob that sets `ec.identity_providers` or a non-default identity budget cannot be loaded by an older binary; restore the defaults and re-push before rolling back. +- **Breaking (rollback):** Stored provider UIDs may be up to 1024 bytes (browser, batch and pull ingest stay at 512). A binary from before this release cannot read an EC row holding a longer UID, so roll back only after such rows expire or are rewritten. An older binary also ignores the full-fidelity `eids` and the 30-day provider `opt_outs` and drops them when it rewrites a row (for example in batch or pull sync), so opted-out visitors are looked up again after rolling forward. A config blob that sets `ec.identity_providers` or a non-default identity budget cannot be loaded by an older binary; restore the defaults and re-push before rolling back. - Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. - Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 1ddd0ab4ae..e6103be39f 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -365,8 +365,8 @@ per-partner, per-minute limit is `ec.partners[].batch_rate_limit`. Responses are `200` when every mapping is accepted or `207` when at least one is rejected; item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, `provider_owned` (the source belongs to a server-side identity provider), -`row_full` (the EC's row has no room for the UID; retrying gives the same -result), or `kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; +`row_full` (the EC's row would exceed its 16 KiB limit; retrying gives the +same result), or `kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; rate exhaustion returns `429`; an oversized body returns `413`; too many mappings returns `400`. The endpoint emits JSON but defines no dedicated cache or CORS headers. It is Fastly-only and requires the EC KV store. diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index cd40f2a94d..0306b69976 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -829,7 +829,7 @@ Until that callback lands, startup remains the mandatory target-aware gate. Auction requests start at dispatch and progress while the publisher origin loads. Response collection waits for headers and complete upstream bodies only until the original absolute deadline. At expiry, one final snapshot preserves complete -responses already available; unfinished providers become timeouts. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and keeps a response whose headers arrived by the cutoff. A slow origin +responses already available; unfinished providers become timeouts. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and reads on into the body of a response whose headers arrived by the cutoff instead of abandoning it. A slow origin never restarts the budget or causes an already-buffered bid to be discarded. Fastly has no upstream arrival timestamp, so this is a bound on network waiting, not bid eligibility based on an exact server-side arrival time. A mediator starts diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index acb84abaa8..dd2d0a4965 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -851,6 +851,9 @@ source_domain = "ids.example.com" [[ec.identity_providers]] id = "example-provider" kind = "example-kind" +# This release registers no provider kind; an enabled provider fails deploy +# validation and startup. +enabled = false endpoint = "https://ids.example.com/v1/lookup" allowed_domains = ["ids.example.com"] sources = ["ids.example.com"] @@ -862,8 +865,11 @@ token = "example_provider_token" `collect = "after_response"` lookups run after the response is sent, within `identity_after_response_budget_ms`, and only when `ec_store` is configured, the response is not a redirect, the navigation's own write succeeded (or had -nothing to change), and the EC was not minted on this navigation. Their results -reach the row in a follow-up write. +nothing to change), the row is live, and the EC was not minted on this +navigation. A row recovered from an orphaned cookie on the same navigation +qualifies. They also run behind an error response, including a `502` that +Trusted Server generates when the origin is unreachable. Their results reach +the row in a follow-up write. Providers run only on Fastly. ::: warning Rate limit granularity `rate_limit_per_hour` is counted per EC hash, which is derived from the client @@ -879,13 +885,17 @@ no sources, when a source has no `[[ec.partners]]` entry, when two enabled providers claim the same source, or when its `id` is empty or shared with another enabled provider. `identity_budget_ms` must be 1–1000 and `identity_after_response_budget_ms` 1–5000. Deploy validation also rejects an -empty `secrets.token` key name. +empty `secrets.token` key name, and, because this release registers no +provider kind, any enabled provider. A disabled provider's `secrets.token` is +not resolved. ::: warning Rollback A config blob that configures identity providers, or sets either identity budget to a non-default value, cannot be loaded by a build from before identity providers. Roll the runtime back only with a blob that leaves them -out. +out. Rows holding a provider UID over 512 bytes also fail to load on such a +build, and an older writer drops a row's `eids` and provider opt-outs when it +rewrites the row. ::: ### Field Details @@ -2551,7 +2561,7 @@ Auction response collection waits only until an absolute deadline measured from dispatch, including headers and complete response bodies. One final snapshot retains complete available responses after a slow origin; unfinished providers become timeouts and mediation cannot start with an exhausted budget. Backend -timeout rounding does not extend network waiting. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and keeps a response whose headers arrived by the cutoff. +timeout rounding does not extend network waiting. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and reads on into the body of a response whose headers arrived by the cutoff instead of abandoning it. Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index a3e8176a97..30c602264a 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -42,7 +42,7 @@ maps to a stable base. ### Request Lifecycle -Every request passes through four phases. A new EC is minted, and its row created, only on a top-level publisher navigation: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` token in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, and a prefetch or prerender (`Sec-Purpose: prefetch;prerender`) never mints or writes. Integration proxies and `/auction` use an existing EC but never mint one, and read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. +Every request passes through four phases. A new EC is minted, and its row created, only on a top-level publisher navigation: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` token in `Sec-Purpose`, answered with a response that is not a 301, 302, 303, 307 or 308 redirect. Clients that send no Fetch Metadata get no EC, and a prefetch or prerender (`Sec-Purpose: prefetch;prerender`) never mints or writes. Integration proxies and `/auction` use an existing EC but never mint one, and read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. ```mermaid sequenceDiagram @@ -182,7 +182,7 @@ Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shap The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. -Cookie persistence runs only on top-level document navigations (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`). `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. +Cookie persistence runs only on top-level publisher navigations: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. Each navigation makes at most one conditional page-load write, which carries browser ingest and page-phase identity-provider results together. If another writer wins, the navigation's changes are dropped without a re-read; a later navigation offers them again. Two writes sit outside that rule: a consent withdrawal's tombstone is written on any request that carries the EC cookie, with bounded retries, and `collect = "after_response"` identity providers make one follow-up write after the response is sent. Batch sync and pull sync write through their own channels. @@ -270,7 +270,7 @@ When the identity graph contains a UID for every pull-enabled partner, Trusted S A valid marker avoids a KV lookup only when pull-sync completeness is the sole reason to inspect the row. Auctions that need stored EIDs, browser EID-cookie ingestion, explicit withdrawal, and generation continue to use KV. The marker acts as recent proof that the row existed, so deletion of a previously complete row is not detected until the marker expires, at most one hour after issuance. Partner-set changes, passphrase rotation, malformed values, and expiration invalidate the marker and restore the normal lookup and orphan-recovery path. -Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal expires both `ts-ec` and any present `ts-ec-pull-complete` marker even when KV is unavailable. +Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal expires both `ts-ec` and any present `ts-ec-pull-complete` marker once the row is tombstoned or proven absent. While the tombstone cannot be written (KV unavailable, CAS retries exhausted, or a row that cannot be read), the cookie is kept so the next request that carries it retries the withdrawal. Issuing or expiring the marker adds `Set-Cookie` to the outgoing response. Cache-privacy handling makes an otherwise shareable response private when that happens. A valid marker is not refreshed on each request, so this cost is limited to responses that establish or clear marker state in exchange for avoiding later KV reads. diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 1f64f99e98..2de92cddcf 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -936,7 +936,7 @@ domain, booleans, source names, counts, and status codes: recording its value. 5. Confirm a controlled Prebid Server request contains that source in `user.ext.eids`. -6. Confirm a later request ingests the source into the configured +6. Confirm a later top-level navigation ingests the source into the configured `liveramp.com` EC partner. 7. Repeat with denied consent and confirm the envelope endpoint is not called, `idl_env` is not written, and no LiveRamp EID is forwarded. @@ -980,7 +980,7 @@ sequenceDiagram T->>P: OpenRTB request\nuser.ext.eids = merged set P-->>T: OpenRTB bid response T-->>B: Auction response - T->>K: Ingest body eids + ts-eids/sharedId cookies\n(TCF Purpose 1 + 4 under GDPR) + Note over B,K: /auction writes nothing to KV; ts-eids and sharedId\nare ingested on the next top-level navigation ``` ### Merge and deduplication rules diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 31a608ec38..c60aafdeb5 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -124,9 +124,8 @@ pull_sync_concurrency = 3 # endpoint = "https://ids.example.com/v1/lookup" # The endpoint host must be listed here. # allowed_domains = ["ids.example.com"] -# Each source needs an [[ec.partners]] entry (for example one with -# source_domain = "ids.example.com") and must be claimed by only one enabled -# provider. +# Each source needs an [[ec.partners]] entry with a matching source_domain and +# must be claimed by only one enabled provider. # sources = ["ids.example.com"] # Optional: require TCF vendor consent for lookups and for sending this # provider's stored IDs to bidders. From dfc52bc3705fe44f6995914e432a749d7e49de1b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:31:31 +0530 Subject: [PATCH 51/58] Correct spec drift and add the review log Covers the withdrawal row, rollback limits, opt-out memory, cookie reads arriving with F1, browser ingest pruning, the deploy gate, outcome labels, removed types and the PR A and PR B release relationship. Signed-off-by: dhruv8sh --- .../2026-10-30-s2s-eid-collection-design.md | 143 ++++++++++++------ 1 file changed, 96 insertions(+), 47 deletions(-) diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index d5b59cb591..eaec4e8f52 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1,9 +1,9 @@ # Server-to-Server EID Collection — Design Spec **Date:** 2026-10-30 -**Status:** Draft for Tech Lead review -**Source of intent:** `S2S_EID_Collection_Requirements.md` (sections referenced -as "R§n") +**Status:** Implemented by PR A (#1487); reviewed (see the review log) +**Source of intent:** the S2S EID collection requirements document, kept +outside this repository (sections referenced as "R§n") ## 1. Purpose @@ -226,8 +226,8 @@ browser TS (navigation) providers KV - Cost: the instance stays busy after the response for up to the after-response budget, which lowers per-instance throughput when sandboxes are reused. Keep the budget no larger than the slowest provider needs. -- Only the Fastly adapter runs this phase (§2.2). Other adapters do not yet - run it, so their `after_response` providers make no lookups. +- Only the Fastly adapter runs identity providers (§2.2). Other adapters + refuse to start with an enabled provider. - Cookie reads and generated IDs of an after-response provider also land only in the follow-up write, so cookie-only providers should stay `page`. @@ -237,7 +237,7 @@ Exactly one of these, chosen from the request snapshot: | Situation | Write | | --------------------------------------- | ----------------------------------------------------------------------------- | -| Consent withdrawn, live row | Tombstone. No lookups, no ingest. EC cookie and mirror expired. | +| Consent withdrawn, live row | No page-load write; the tombstone follows §3.1. No lookups, no ingest. | | Row tombstoned, no fresh consent | None. No lookups. Nothing served. | | New EC (or orphaned EC being recovered) | Create the row with browser ingest + valid provider results. | | Returning EC, something changed | CAS update with browser ingest + valid provider results; expired UIDs pruned. | @@ -480,14 +480,20 @@ bodies with `// ...`; implementations follow the documentation rules in ### 6.3 Stored row All additions are optional `serde` fields on schema `v: 1`. Rows written before -this change read unchanged; rows written after it stay readable by older code -through the retained `uid` field. +this change read unchanged. Rows written after it stay readable by older code +through the retained `uid` field, unless that `uid` (the first live provider +UID) is over 512 bytes: older code rejects such a row whole. + +The row also gains `opt_outs`, a map from provider ID to the Unix time until +which that provider's opt-out is remembered (30 days, §8.3). `uid` is rewritten on every write to the first live UID of `eids`, and new code ignores it when `eids` is non-empty. A known limitation of staying on `v: 1` (D12): during a rollout or rollback, older code reads only `uid`, so it can serve that UID after its `exp`, and an older writer that re-saves the row drops -`eids`. Both end at the next write by new code. Stored values are built only by validation +`eids` and `opt_outs`. A UID served past its `exp` and dropped `eids` end at the +next write by new code; a dropped opt-out is lost, so that provider is called +again. Stored values are built only by validation (§8.2) or by deserializing rows TS wrote, so their fields are private to the `ec` module and exposed through accessors. @@ -739,9 +745,10 @@ pub struct ProviderRequest { `(&str, &KvEid)` pairs, keyed by the row's source strings. Provider secrets and settings are not in the input: they are given to the provider once, when it is built (§7.3). -`LookupInput` also lends the provider the request's cookies (`cookies`), which -`IdentityProvider::read_request_cookies` reads into a `CookieRead`. These -predate this work and are unchanged. +PR3 ships `LookupInput` without the request's cookies. F1 (#1488) adds +`LookupInput::cookies` and `IdentityProvider::read_request_cookies`, which +returns a `CookieRead`; the cookie-first refresh (§5.6), cookie opt-outs (§8.3) +and scenarios 24 and 26 depend on it. Response and parsed output: @@ -908,8 +915,9 @@ digest with no constructor reachable from request data in V1. The coordinator owns, for every provider: -1. **Eligibility.** Provider enabled; EC present or being created; row not - tombstoned; at least one due source; sources needing `HashedEmail` removed. +1. **Eligibility.** Provider enabled; the EC's row exists and is not + tombstoned (an EC minted on this navigation, or a cookie EC with no row, is + not sent to vendors); at least one due source; sources needing `HashedEmail` removed. 2. **Consent.** The existing EID-persistence check passes, and the provider's `tcf_vendor_id`, when configured, has vendor consent in the effective TCF string. A failure skips that provider only, before any request is built. @@ -1045,12 +1053,6 @@ pub trait IdentityCoordinator { pending: PendingLookups, ) -> LookupResults; - /// Merges browser ingest and lookup results into the navigation's single - /// row change. - /// - /// Returns `None` when the row would not change, so no write is made. - fn plan_write(&self, input: &WritePlanInput<'_>) -> Option; - /// Builds the mirror cookie from the row as it stands after this /// navigation: the committed row, or the snapshot when no write landed. fn mirror(&self, row: Option<&KvEntry>, consent: &ConsentContext, now: UnixSeconds) -> MirrorCookie; @@ -1106,6 +1108,7 @@ pub enum LookupStatus { HttpError(http::StatusCode), TransportFailed, InvalidResponse, + RequestFailed, Skipped(SkipReason), } @@ -1122,7 +1125,6 @@ pub struct DropCounts { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SkipReason { - Disabled, Consent, NotDue, Input, @@ -1131,7 +1133,8 @@ pub enum SkipReason { OptedOut, } -/// Inputs to [`IdentityCoordinator::plan_write`]. +/// Inputs to the free function `write_plan::plan_write`, the navigation's +/// planner (it is not a coordinator method). /// /// `PartnerIdUpdate` (today `pub(crate)` in `ec::kv`) becomes `pub` so this /// public trait input does not leak a private type. @@ -1214,7 +1217,7 @@ where let mut row = ec.snapshot_entry(); let change = permit .confirm(response.status()) - .zip(coordinator.plan_write(&ec.write_plan(&lookups))); + .zip(write_plan::plan_write(&ec.write_plan(&lookups))); if let Some((permit, change)) = change { // One attempt, never retried (§9.2). match graph.commit_page_load(permit, ec.id(), change) { @@ -1257,7 +1260,11 @@ For each provider with a `Found` outcome and each source in its response: - Sources the provider claims but did not return stay unchanged. - `NoMatch`, failures, timeouts and skipped providers leave the row unchanged. - `OptedOut` (a cookie or a refresh reply) clears every source the provider - claims; generated and cookie IDs merge like `Found`. + claims and records the opt-out in the row's `opt_outs` map for 30 days. Until + it lapses the provider is skipped as `skipped_opted_out` and not called, and + a result from a lookup dispatched before the opt-out was recorded is + discarded rather than written over it. Generated and cookie IDs merge like + `Found`. Browser ingest (§8.4) is applied in the same in-memory merge, before provider results, so the write order never lets browser data overwrite provider data. @@ -1266,9 +1273,10 @@ results, so the write order never lets browser data overwrite provider data. Unchanged rules: registered sources only; one UID per source, the first valid UID of the last matching entry, with `sharedId` applied last; no EID-level -fields; fill missing sources only, where a source holding only expired UIDs -counts as holding UIDs until a write prunes them. New rule: a provider-owned -source is skipped entirely. +fields; fill missing sources only; UIDs at most `MAX_UID_LENGTH` (512) bytes. +The planner prunes expired UIDs first, so a source holding only expired UIDs +can be filled by the same write. New rule: a provider-owned source is skipped +entirely. ### 8.5 Batch sync @@ -1305,7 +1313,7 @@ impl NavigationPermit { /// /// Called once per request, by the router only. #[must_use] - pub(crate) fn classify(request: &http::Request) -> Option; + pub fn classify(request: &http::Request) -> Option; /// Confirms the permit once the final response status is known. /// @@ -1381,7 +1389,7 @@ PR5 removes it. | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | | CAS conflict or store error | No retry. Row unchanged. Mirror built from the unchanged row. New EC cookie not set. Counted. | | Tombstone write fails | Bounded CAS retry first. Mirror expired and no IDs served (consent gating). EC cookie kept so the next request retries it. | -| EC-ID collision on create | Same as above; the next navigation generates a new EC. | +| EC-ID collision on create | Same as a CAS conflict; the next navigation generates a new EC. | | Provider timeout/outage/invalid | That provider's results are dropped; the write proceeds with everything else that is valid. | | All providers fail | The write proceeds if browser ingest changed anything; otherwise no write. | | Coordinator budget exceeded | Late responses dropped; the response is never held past the budget. | @@ -1462,7 +1470,7 @@ sources = ["ids.example.com"] tcf_vendor_id = 9999 # Optional: "page" (default) or "after_response" (§5.2.1). collect = "page" -# Optional: most lookups per EC per hour (§8.1); unset means unlimited. +# Optional: most lookups per EC hash per hour (§8.1); unset means unlimited. rate_limit_per_hour = 4 # Secret Store key names, resolved like other secret references. @@ -1478,10 +1486,18 @@ entry; a source outside the kind's capabilities; a source claimed by two enabled providers; an `endpoint` host outside `allowed_domains`; placeholder secrets in deploy validation. -The operator CLI's config checks (`ts config validate`, `push` and `diff`) -also build every enabled provider of a registered kind, as the adapter does -per navigation, so kind-specific setting errors fail before deploy. Core's -own validation cannot do this because provider kinds live outside core. +Validation also rejects an empty or duplicate provider `id`, an empty +`sources` list, a non-HTTPS `endpoint`, and identity budgets outside 1–1000 ms +(page) and 1–5000 ms (after-response). + +PR3 registers no provider kind, so deploy validation (`ts config validate`, +`push` and `diff`) rejects every enabled provider; a push therefore cannot +take a running service down. F1 replaces that rule: the CLI builds every +enabled provider of a registered kind, as the adapter does at startup, and +rejects enabled providers for a target adapter that does not run them, so +kind-specific setting errors fail before deploy. Core's own validation cannot +do this because provider kinds live outside core. A disabled provider's +`secrets.token` is not resolved. Secrets are Secret Store key references, resolved at startup into `Redacted` values through the existing secret path. They are never written to cookies or logs, and source-controlled @@ -1490,7 +1506,7 @@ examples use placeholders. ### 12.2 Logs and metrics Per provider and navigation, one outcome: `found`, `no_match`, `timeout`, -`http_error`, `transport_error`, `invalid_response`, `skipped_disabled`, `skipped_consent`, +`http_error`, `transport_error`, `request_error`, `invalid_response`, `skipped_consent`, `skipped_not_due`, `skipped_input`, `skipped_rate_limited`, `skipped_opted_out`, `destination_rejected`; plus counts of entries and UIDs kept and dropped by reason. Per navigation, one write outcome: @@ -1507,13 +1523,15 @@ truncated form. Raw UIDs, EID values and encrypted envelopes are never logged. The work below is delivered as two pull requests: -- **PR A** — the work units PR1, PR2 and PR3, stacked on the open platform PR - #1433, which provides the deadline-bounded `select_until` that - `PlatformLookupTransport` uses. +- **PR A** — the work units PR1, PR2 and PR3, plus the PR4 batch-sync + `provider_owned` rejection, on top of the merged platform PR #1433, which + provides the deadline-bounded `select_until` that `PlatformLookupTransport` + uses. - **PR B** — the work units PR4, PR5 and PR6, stacked on PR A. -PR A and PR B ship in the same release, because PR A alone leaves the old -post-send pull-sync write as a second write per page load. Within each pull +PR A alone keeps the old post-send pull-sync write as a separate channel +(§9), as it is on main; PR A registers no provider kind, so no provider write +adds to it. PR B removes it. Within each pull request the units are separate, ordered commits so they stay reviewable. | PR | Title | Contents | Depends on | @@ -1528,13 +1546,13 @@ request the units are separate, ordered commits so they stay reviewable. ### 13.2 Optional and future (Tech Lead decision) -| ID | Task | Notes | Depends on | -| --- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| O1 | tsjs reads the mirror (D6b) | Reads the mirror, drops expired UIDs, strips `exp`, passes entries to Prebid user IDs. Separate PR. | PR6 | -| F1 | Concrete provider implementations | R§5.1. One spec and one PR per vendor named in R§6, each a module and `build` arm in the provider crate. | PR3 | -| F2 | Provider lookup traffic controls | R§5.2. Best-effort dedup per EC and provider, bounded no-match caching kept distinct from failures, refresh jitter. Strict cross-instance dedup is a separate decision. | PR3 | -| F3 | Per-provider cooldown | R§5.3. Pause after repeated failures, then probe for recovery. | PR3 | -| F4 | Provider debug mode | R§5.4. A mock `LookupTransport` runs real providers and the real coordinator against configurable responses; no real provider calls, no production KV or cookie mutation. | PR3 | +| ID | Task | Notes | Depends on | +| --- | --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| O1 | tsjs reads the mirror (D6b) | Reads the mirror, drops expired UIDs, strips `exp`, passes entries to Prebid user IDs. Separate PR. | PR6 | +| F1 | Concrete provider implementations | R§5.1. One spec and one PR per vendor named in R§6, each a module and `build` arm in the provider crate. Also adds cookie reads to the core coordinator (`LookupInput::cookies`, `read_request_cookies`) and the kind- and adapter-aware deploy check (§12.1). | PR3 | +| F2 | Provider lookup traffic controls | R§5.2. Best-effort dedup per EC and provider, bounded no-match caching kept distinct from failures, refresh jitter. Strict cross-instance dedup is a separate decision. | PR3 | +| F3 | Per-provider cooldown | R§5.3. Pause after repeated failures, then probe for recovery. | PR3 | +| F4 | Provider debug mode | R§5.4. A mock `LookupTransport` runs real providers and the real coordinator against configurable responses; no real provider calls, no production KV or cookie mutation. | PR3 | ## 14. Acceptance scenarios @@ -1620,3 +1638,34 @@ request the units are separate, ordered commits so they stay reviewable. 27. **Refresh state is never served.** A row holds a UID with refresh state. → `/auction`, the mirror cookie and `x-ts-eids` carry the UID's `id`, `atype` and `ext` only. + +## Review log + +### Round 1 (2026-11-09, PR #1487) + +Reviewer findings and resolutions: + +- Withdrawal outside permitted navigations: tombstone moved outside the + permit with bounded CAS retry (§3.1). +- Stored provider UIDs served after vendor consent withdrawal: egress gated on + the owning provider's TCF vendor ID. +- Config blob and 1024-byte UIDs breaking rollback: identity defaults left + out of the blob; non-provider writers stay at 512 bytes. +- Unknown kinds skipped silently, duplicate provider IDs, deeply nested `ext`, + batch sync aborting on a full row, minted ECs sent to vendors: fixed. +- Negative caching, a maximum provider UID lifetime, after-response retry, + lenient readers and client EIDs after an opt-out: deferred to follow-up + issues. + +### Round 2 (2026-11-10/11, PR #1487 at 889159cd6) + +- Opt-out race on after-response and confirmation-read adoption: results of + an opted-out provider are discarded (§8.3). +- Size cap: trimming alone plans no write; this navigation's browser UIDs are + dropped before any stored record. +- Lookups only for an EC with an existing consented row (§8.1). +- Deploy validation rejects enabled providers while no kind is registered; + disabled providers' secrets are not resolved (§12.1). +- Disabled providers keep their vendor-consent gate on stored UIDs; `ext` + nesting bounded to 32 levels; raw client IP passed to providers. +- Spec drift corrected (§5.3, §6.3, §7, §8.4, §9.2, §12, §13). From c6e5c9975a8d695d0b03172f684322d7f67f1d6b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:49:24 +0530 Subject: [PATCH 52/58] Keep adapter test KV hashes clear of the ec_kv prefix-count test Two new Viceroy tests seeded rows under the 5 and 6 hashes that counting_a_prefix_counts_only_the_keys_under_it counts in the shared local store, which aborted test-fastly. Move them to hashes no other test uses. Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-fastly/src/app.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index b668fa1568..97bbe85911 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -3154,7 +3154,7 @@ mod tests { fn page_phase_lookups_reach_the_write_for_a_returning_ec() { // Arrange let fixture = identity_fixture(false); - let ec_id = format!("{}.pg0001", "6".repeat(64)); + let ec_id = format!("{}.pg0001", "c".repeat(64)); seed_live_row(&fixture.graph, &ec_id); // Act @@ -3319,7 +3319,7 @@ mod tests { fn after_response_lookups_skip_a_row_withdrawn_before_the_phase() { // Arrange let fixture = after_response_fixture(); - let ec_id = format!("{}.ar0005", "5".repeat(64)); + let ec_id = format!("{}.ar0005", "b".repeat(64)); seed_live_row(&fixture.graph, &ec_id); let (mut ec_state, lookups) = finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); From 408950546925240558fbd4b386b994fd3379feb2 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:16:15 +0530 Subject: [PATCH 53/58] Drop client EIDs of a provider source whose vendor consent is withheld Applies to /auction and the navigation and page-bids auctions before the merge. Only sources owned by a configured provider carry a vendor ID, so with no providers the client EIDs pass through byte-identical. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 130 +++++++++++++++++- crates/trusted-server-core/src/publisher.rs | 8 +- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 1f5a8932b8..366618ae1f 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -9,7 +9,10 @@ use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; +use crate::consent::{ + ConsentContext, consent_allows_server_side_auction, gate_eids_by_consent, + tcf_vendor_consent_denied, +}; use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; @@ -377,6 +380,7 @@ pub async fn handle_auction( // Merge current-request client EIDs with KV-resolved EIDs, then apply // consent gating before attaching them to the auction request. + let client_eids = drop_vendor_denied_client_eids(client_eids, registry, ec_context.consent()); let merged_eids = merge_auction_eids(client_eids, eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = @@ -637,6 +641,29 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { Some(Uid { id, atype, ext }) } +/// Drops client EIDs whose source is owned by a configured identity provider +/// whose TCF vendor consent is withheld. +/// +/// Only sources marked by a configured provider carry a vendor ID, so with no +/// providers configured the input is returned unchanged. +pub(crate) fn drop_vendor_denied_client_eids( + client_eids: Option>, + registry: Option<&PartnerRegistry>, + consent: &ConsentContext, +) -> Option> { + let Some(registry) = registry else { + return client_eids; + }; + let mut eids = client_eids?; + eids.retain(|eid| { + !registry + .get(&eid.source) + .and_then(|partner| partner.owner_tcf_vendor_id) + .is_some_and(|vendor| tcf_vendor_consent_denied(consent, vendor)) + }); + Some(eids) +} + pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, @@ -774,6 +801,107 @@ mod tests { .build() } + fn vendor_denied_consent() -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(crate::consent::types::TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: Vec::new(), + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + + fn client_eids_for(sources: &[&str]) -> Option> { + Some( + sources + .iter() + .map(|source| { + Eid::new( + *source, + vec![Uid { + id: format!("uid-{source}"), + atype: Some(1), + ext: None, + }], + ) + }) + .collect(), + ) + } + + fn partner_registry(sources: &[&str]) -> PartnerRegistry { + let partners: Vec = sources + .iter() + .map(|source| { + serde_json::from_value(json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + PartnerRegistry::from_config(&partners).expect("should build registry") + } + + #[test] + fn client_eids_of_a_vendor_denied_provider_source_are_dropped() { + // Arrange + let mut registry = partner_registry(&["ids.example.com", "other.example.com"]); + assert!( + registry.mark_provider_owned("ids.example.com", std::num::NonZeroU16::new(42)), + "should mark the provider source" + ); + let client = client_eids_for(&["ids.example.com", "other.example.com"]); + + // Act + let kept = + drop_vendor_denied_client_eids(client, Some(®istry), &vendor_denied_consent()) + .expect("should keep the list"); + + // Assert + let sources: Vec<&str> = kept.iter().map(|eid| eid.source.as_str()).collect(); + assert_eq!( + sources, + vec!["other.example.com"], + "should drop only the vendor-denied provider source" + ); + } + + #[test] + fn client_eids_are_unchanged_without_providers() { + // Arrange + let registry = partner_registry(&["ids.example.com"]); + let client = client_eids_for(&["ids.example.com", "unregistered.example.com"]); + let before = serde_json::to_string(&client).expect("should serialize"); + + // Act + let kept = + drop_vendor_denied_client_eids(client, Some(®istry), &vendor_denied_consent()); + + // Assert + assert_eq!( + serde_json::to_string(&kept).expect("should serialize"), + before, + "should leave client EIDs byte-identical with no providers" + ); + } + fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { EcContext::new_for_test( ec_value.map(str::to_owned), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c02070a0ce..c2c00d08e3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -41,7 +41,8 @@ use sha2::Digest as _; use web_time::Instant; use crate::auction::endpoints::{ - merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, + drop_vendor_denied_client_eids, merge_auction_eids, resolve_auction_eids, + resolve_client_auction_eids, }; use crate::auction::formats::sanitize_publisher_page_url; use crate::auction::orchestrator::{ @@ -6171,6 +6172,11 @@ fn apply_auction_eids_and_device( targeting.partner_registry, targeting.ec_context, ); + let client_eids = drop_vendor_denied_client_eids( + client_eids, + targeting.partner_registry, + targeting.ec_context.consent(), + ); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = From 9ffdd46f6960a3dd756d1b662c1b2353b139806c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:17:14 +0530 Subject: [PATCH 54/58] Cap ts-eids cookie UIDs at 512 bytes again On main the auction merge filter was the only cap for cookie UIDs. With that filter at the 1024-byte stored limit, the cookie parser now drops UIDs over MAX_UID_LENGTH itself, so cookie UIDs over 512 bytes no longer reach bidders. Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/prebid_eids.rs | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 768a431188..8619fd0651 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -302,7 +302,8 @@ fn structured_cookie_eids_to_openrtb(entries: Vec) -> Vec Option { - if uid.id.is_empty() { + // Cookie UIDs keep the browser limit; only provider writes may be longer. + if uid.id.is_empty() || uid.id.len() > MAX_UID_LENGTH { return None; } @@ -322,7 +323,9 @@ fn structured_cookie_uid_to_openrtb(uid: StructuredCookieUid) -> Option { fn legacy_cookie_eids_to_openrtb(entries: Vec) -> Vec { entries .into_iter() - .filter(|entry| !entry.source.is_empty() && !entry.id.is_empty()) + .filter(|entry| { + !entry.source.is_empty() && !entry.id.is_empty() && entry.id.len() <= MAX_UID_LENGTH + }) .map(|entry| { Eid::new( entry.source, @@ -450,6 +453,35 @@ mod tests { ); } + #[test] + fn parse_prebid_eids_cookie_drops_uids_over_the_browser_limit() { + // Arrange + let long = "x".repeat(MAX_UID_LENGTH + 1); + let at_limit = "y".repeat(MAX_UID_LENGTH); + let structured = BASE64.encode( + json!([{ "source": "ids.example.com", "uids": [ + { "id": long, "atype": 1 }, + { "id": at_limit, "atype": 1 } + ]}]) + .to_string(), + ); + let legacy = BASE64 + .encode(json!([{ "source": "ids.example.com", "id": long, "atype": 1 }]).to_string()); + + // Act + let structured = parse_prebid_eids_cookie(&structured).expect("should decode"); + let legacy = parse_prebid_eids_cookie(&legacy).expect("should decode"); + + // Assert + let ids: Vec = structured[0].uids.iter().map(|uid| uid.id.len()).collect(); + assert_eq!( + ids, + vec![MAX_UID_LENGTH], + "should keep only the UID within the limit" + ); + assert!(legacy.is_empty(), "should drop an over-limit legacy UID"); + } + #[test] fn parse_prebid_eids_cookie_rejects_invalid_base64() { let result = parse_prebid_eids_cookie("not-valid-base64!!!"); From eaa1f1fa23e7d07e207b46f5bf0cef0d39f62155 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:19:07 +0530 Subject: [PATCH 55/58] Tie the provider startup tests to the error they expect The Fastly and provider-crate startup tests now assert that the error names the provider and its kind, and Axum, Cloudflare and Spin each test that an enabled provider fails startup with the adapter named. Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-axum/src/app.rs | 17 +++++++++++++++++ .../src/app.rs | 17 +++++++++++++++++ crates/trusted-server-adapter-fastly/src/app.rs | 9 ++++++--- crates/trusted-server-adapter-spin/src/app.rs | 17 +++++++++++++++++ crates/trusted-server-core/src/test_support.rs | 17 +++++++++++++++++ .../src/lib.rs | 9 ++++++--- 6 files changed, 80 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 8e3143bdaa..0d5ac4e366 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -746,6 +746,23 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { #[cfg(test)] mod trace_dispatch_tests { use super::*; + + #[test] + fn enabled_identity_provider_fails_startup() { + let mut settings = trace_settings(false); + settings.ec.identity_providers = + vec![trusted_server_core::test_support::fixtures::enabled_identity_provider()]; + + let Err(error) = build_state_with_settings(settings) else { + panic!("should refuse to start with an enabled identity provider"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("Axum"), + "should name the provider and the adapter: {message}" + ); + } use futures::executor::block_on; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedClientIpConfig; diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 3f96bc9217..44694f1baa 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -823,6 +823,23 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { mod tests { use super::*; + #[test] + fn enabled_identity_provider_fails_startup() { + let mut settings = aps_profile_settings(); + settings.ec.identity_providers = + vec![trusted_server_core::test_support::fixtures::enabled_identity_provider()]; + + let Err(error) = build_state_with_settings(settings) else { + panic!("should refuse to start with an enabled identity provider"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("Cloudflare"), + "should name the provider and the adapter: {message}" + ); + } + fn aps_profile_settings() -> Settings { let mut settings = Settings::from_toml( r#" diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 97bbe85911..9bc296dd96 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -2849,11 +2849,14 @@ mod tests { .expect("should parse provider"), ); - let result = build_state_from_settings(settings); + let Err(error) = build_state_from_settings(settings) else { + panic!("should refuse to start with a provider this build cannot run"); + }; + let message = format!("{error:?}"); assert!( - result.is_err(), - "should refuse to start with a provider this build cannot run" + message.contains("example-provider") && message.contains("example-kind"), + "should name the provider and its unregistered kind: {message}" ); } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index a9f4e031d2..dd1a20a014 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -1010,6 +1010,23 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { mod tests { use super::*; + #[test] + fn enabled_identity_provider_fails_startup() { + let mut settings = multi_provider_settings(); + settings.ec.identity_providers = + vec![trusted_server_core::test_support::fixtures::enabled_identity_provider()]; + + let Err(error) = build_state_with_settings(settings) else { + panic!("should refuse to start with an enabled identity provider"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("Spin"), + "should name the provider and the adapter: {message}" + ); + } + fn multi_provider_settings() -> Settings { let mut settings = Settings::from_toml( r#" diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 187b22a276..7c7067bdb1 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -634,3 +634,20 @@ pub mod nextjs_auction { #[cfg(feature = "test-utils")] pub(crate) mod async_html_finalization; + +/// Fixtures shared with adapter crates through the `test-utils` feature. +#[cfg(any(test, feature = "test-utils"))] +pub mod fixtures { + #[must_use] + /// Returns an enabled identity provider configuration for tests. + /// + /// # Panics + /// + /// Panics if the embedded TOML is invalid. + pub fn enabled_identity_provider() -> crate::settings::IdentityProviderConfig { + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse identity provider") + } +} diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 23197150ff..e11e69abf3 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -208,11 +208,14 @@ mod tests { .expect("should parse"), ]; - let result = build_coordinator(&settings); + let Err(error) = build_coordinator(&settings) else { + panic!("should fail instead of skipping an enabled provider of an unknown kind"); + }; + let message = format!("{error:?}"); assert!( - result.is_err(), - "should fail instead of skipping an enabled provider of an unknown kind" + message.contains("`p`") && message.contains("missing"), + "should name the provider and its unregistered kind: {message}" ); } From e17c0b26a45dd2e7cec54e84d967b0c03242b824 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:20:14 +0530 Subject: [PATCH 56/58] Give Ec::default the identity budget defaults The derived Default set both budgets to 0, which failed range validation and serialized both identity keys. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/settings.rs | 42 +++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index f623aa1852..8632c56dd9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -668,7 +668,7 @@ pub struct ProviderSecrets { /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, /// KV store names, and partner registry. -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] pub struct Ec { /// Publisher passphrase used as HMAC key for EC generation. @@ -744,6 +744,24 @@ pub struct Ec { pub identity_providers: Vec, } +// Not derived: the identity budgets default to their serde defaults so a +// default `Ec` passes validation and serializes no identity keys. +impl Default for Ec { + fn default() -> Self { + Self { + passphrase: Redacted::default(), + ec_store: None, + pull_sync_concurrency: 0, + cluster_trust_threshold: 0, + cluster_recheck_secs: 0, + partners: Vec::new(), + identity_budget_ms: Self::default_identity_budget_ms(), + identity_after_response_budget_ms: Self::default_identity_after_response_budget_ms(), + identity_providers: Vec::new(), + } + } +} + impl Ec { /// Known placeholder values that must not be used in production. pub const PASSPHRASE_PLACEHOLDERS: &[&str] = &[ @@ -4361,6 +4379,28 @@ where #[cfg(test)] mod tests { use super::*; + + #[test] + fn default_ec_uses_the_identity_budget_defaults() { + let ec = Ec::default(); + + assert_eq!( + (ec.identity_budget_ms, ec.identity_after_response_budget_ms), + ( + Ec::default_identity_budget_ms(), + Ec::default_identity_after_response_budget_ms() + ), + "should match the serde defaults" + ); + let serialized = serde_json::to_value(&ec).expect("should serialize"); + assert!( + serialized.get("identity_budget_ms").is_none() + && serialized + .get("identity_after_response_budget_ms") + .is_none(), + "should serialize no identity keys" + ); + } use crate::auction::build_orchestrator; use crate::integrations::{ IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, From 56146e3129be812dde002ee11ac73366219f96c3 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:21:13 +0530 Subject: [PATCH 57/58] Restore the completion-marker tombstone test Brings back RecordingEcKv::completed_with_root_check_failure and tombstone_stale_miss_valid_marker_skips_root_check_and_writes from main. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/kv.rs | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index f118e59951..5482562f9d 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -2519,6 +2519,49 @@ mod tests { } } + fn completed_with_root_check_failure( + operations: Arc, + ec_id: &str, + ) -> Self { + let store = Self { + root_check_fails: true, + stale_lookups_remaining: std::sync::Mutex::new(u32::MAX), + ..Self::new(operations) + }; + let tombstone = KvEntry::tombstone(current_timestamp()); + let (body, metadata) = + KvIdentityGraph::serialize_entry(&tombstone, store.inner.store_name()) + .expect("should serialize tombstone"); + store + .inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &metadata, + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed tombstone"); + store + .inner + .insert( + &KvIdentityGraph::withdrawal_marker_key( + ec_id, + tombstone.consent.updated + TOMBSTONE_TTL.as_secs(), + ), + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed completion marker"); + store + } + fn with_marker_delete_failure( operations: Arc, remove_before_error: bool, @@ -3572,6 +3615,47 @@ mod tests { } } + #[test] + fn tombstone_stale_miss_valid_marker_skips_root_check_and_writes() { + let ec_id = snapshot_ec_id(); + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::completed_with_root_check_failure( + Arc::clone(&operations), + &ec_id, + )); + + let outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a completion marker should resolve a repeated stale miss" + ); + assert_eq!( + operations.lookup_count(), + 1, + "should retry the stale point read once" + ); + assert_eq!( + operations.list_count(), + 1, + "should strongly list completion markers once" + ); + assert_eq!( + operations.exact_check_count(), + 0, + "valid marker should bypass the failing root check" + ); + assert!( + operations.inserts().is_empty(), + "valid marker should prevent redundant writes" + ); + } + #[test] fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { let operations = Arc::new(RecordedEcKvOperations::default()); From 4e7a6e02b836aea8c9e53203bf6b4ffb5aee7509 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:21:25 +0530 Subject: [PATCH 58/58] Describe page-write logging and add a compatibility section to the spec Signed-off-by: dhruv8sh --- .../2026-10-30-s2s-eid-collection-design.md | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index eaec4e8f52..c07cc7a521 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1509,14 +1509,43 @@ Per provider and navigation, one outcome: `found`, `no_match`, `timeout`, `http_error`, `transport_error`, `request_error`, `invalid_response`, `skipped_consent`, `skipped_not_due`, `skipped_input`, `skipped_rate_limited`, `skipped_opted_out`, `destination_rejected`; plus counts of -entries and UIDs kept and dropped by reason. Per navigation, one write outcome: -`written`, `no_change`, `conflict`, `error`, `not_permitted`, -`no_generation`. A navigation that runs the after-response phase (§5.2.1) has -one more write outcome: `written`, `unchanged`, `no_row`, `conflict` or -`failed`. Logs include the +entries and UIDs kept and dropped by reason. The page-load write emits no +outcome label in PR A: a skipped write is logged with its error (a conflict, +an existing row or a store error), and an update without a generation is +logged as `no_generation`. A navigation that runs the after-response phase +(§5.2.1) logs one write outcome: `written`, `unchanged`, `no_row`, `conflict` +or `failed`. Logs include the provider ID, source, counts and durations, and the EC ID only in its existing truncated form. Raw UIDs, EID values and encrypted envelopes are never logged. +### 12.3 Compatibility and rollout + +PR A changes behaviour that operators and integrators can observe: + +- **EC minting.** A new EC is minted, and its row written, only on a + top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, + `Sec-Fetch-Mode: navigate`, no `prefetch` in `Sec-Purpose`, and a response + that is not a 301, 302, 303, 307 or 308 redirect. Clients without Fetch + Metadata get no EC, and integration proxies no longer mint one. +- **`user.id` withheld.** An EC minted on the current navigation is not sent + as `user.id`, and that navigation's auction carries no browser `ts-eids`, + until a later navigation finds its row. +- **Browser EIDs.** `/auction` and page-bids no longer write browser EIDs; the + next navigation ingests them. +- **Row size cap.** Rows are capped at 16 KiB (`MAX_ENTRY_BYTES`). Batch sync + reports `row_full`; a row written before the cap is trimmed only when a + write also carries a real change. +- **Fastly snapshot.** The final auction snapshot may wait up to 1 ms per + partially received body and reads on into a response whose headers arrived + by the cutoff. +- **Config blob rollback.** A blob that configures identity providers or sets + a non-default identity budget cannot be loaded by an older build; default + values are left out of the blob so an unchanged config stays compatible. +- **Row rollback.** Rows with a provider UID over 512 bytes fail to load on an + older build, and an older writer drops `eids` and `opt_outs` (§6.3). +- **Providers.** PR A registers no provider kind, so deploy validation and + every runtime refuse an enabled provider. + ## 13. PR breakdown ### 13.1 Required