diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7a145e02..58e6a4ac7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -230,9 +230,14 @@ jobs: run: | cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings + - name: Set up Chrome for browser fixture tests + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh + env: + CHROME: ${{ steps.setup-chrome.outputs.chrome-path }} test-typescript: name: vitest diff --git a/.gitignore b/.gitignore index 24b9e06aa..b71945639 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ src/*.html /guest-profiles /benchmark-results/** +/.worktrees/ # Playwright browser tests /crates/trusted-server-integration-tests/browser/node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f05208294..32b027a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. Coverage of the dynamic `/_ts/admin/ec/{id}` route is no longer inferred from ID-shaped samples: the router accepts any segment after `/_ts/admin/ec/` and Basic Auth runs on the raw path before routing, so patterns anchored to the EC ID grammar (for example `^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$`) are rejected in favor of a prefix-level matcher. Placeholder and well-known weak handler passwords (`changeme`, `password`, `admin`, `replace-with-…`) now fail startup on every handler rather than only on handlers inferred to cover an admin endpoint, because first-match-wins handler selection lets a narrow handler shadow the admin namespace. +- Publisher HTML uses the browser-only `Cache-Control: private, max-age=60` policy for successful GET document responses and their `304 Not Modified` revalidations when server-side ad templates are structurally inactive, while preserving origin `private`/`no-store` policies and request-scoped bot, prefetch, or consent-denied responses. The `private` directive prevents shared caches that use `Cache-Control` from storing the document. Cookie-bearing responses using the generated inactive policy are finalized as `private, max-age=0`; CDN-specific cache headers remain unchanged and continue to control supporting CDNs independently. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers; an absent configuration, an unmatched slot, or a disabled auction also make the stack structurally inactive. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. +- Publisher HTML now uses `Cache-Control: max-age=60` when server-side ad templates are inactive, while preserving origin `private`/`no-store` policies and CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. - **Breaking** — Integration configuration strings are no longer globally reinterpreted as JSON scalars. Operators upgrading should audit `[integrations.*]` settings and use native TOML/typed-config booleans and numbers (for example, `enabled = true`, not `enabled = "true"`); quoted numeric and boolean scalars now fail validation instead of silently converting. - **Breaking** — Sourcepoint browser module inclusion now requires explicit `[integrations.sourcepoint].enabled = true`; operators relying on the previous unconditional Sourcepoint module should enable the integration before upgrading. diff --git a/Cargo.lock b/Cargo.lock index e29380b77..65b1d9507 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1427,7 +1427,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "toml", ] @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1463,7 +1463,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1478,7 +1478,7 @@ dependencies = [ "log", "serde_json", "tempfile", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", "worker", ] @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-stream", @@ -1508,14 +1508,14 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1535,14 +1535,14 @@ dependencies = [ "subtle", "thiserror 2.0.18", "toml", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "chrono", "clap", @@ -1567,7 +1567,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-compression", @@ -1598,14 +1598,14 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=5f3d648c3c6c38fc6e6b22b5c65c66177363aad8#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 3.0.3", "toml", "validator", ] @@ -4858,6 +4858,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5163,6 +5174,19 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -5413,8 +5437,11 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", + "glob", + "http", "http-body-util", "hyper", "hyper-util", @@ -5426,12 +5453,15 @@ dependencies = [ "scraper", "serde", "serde_json", + "similar", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", + "tracing", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 25c367181..02d6995cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", rev = "5f3d648c3c6c38fc6e6b22b5c65c66177363aad8", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", rev = "5f3d648c3c6c38fc6e6b22b5c65c66177363aad8", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", rev = "5f3d648c3c6c38fc6e6b22b5c65c66177363aad8", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", rev = "5f3d648c3c6c38fc6e6b22b5c65c66177363aad8", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", rev = "5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", rev = "5f3d648c3c6c38fc6e6b22b5c65c66177363aad8", default-features = false } env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" @@ -96,6 +96,7 @@ scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" @@ -108,6 +109,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/README.md b/README.md index b87fe61ad..81794720c 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 45cbedc2c..e2852ef36 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -7,6 +7,7 @@ use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; +use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -35,7 +36,11 @@ impl FinalizeResponseMiddleware { #[async_trait(?Send)] impl Middleware for FinalizeResponseMiddleware { - async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + sanitize_trusted_client_ip_headers( + ctx.request_mut(), + self.settings.trusted_client_ip.as_ref(), + ); let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, &mut response); Ok(response) @@ -111,8 +116,17 @@ pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Respons mod tests { use super::*; + use std::collections::HashMap; + use std::sync::Mutex; + use edgezero_core::body::Body; - use edgezero_core::http::response_builder; + use edgezero_core::context::RequestContext; + use edgezero_core::http::{Method, request_builder, response_builder}; + use edgezero_core::middleware::Next; + use edgezero_core::params::PathParams; + use futures::executor::block_on; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::TrustedClientIpConfig; fn empty_response() -> Response { response_builder() @@ -120,6 +134,17 @@ mod tests { .expect("should build empty test response") } + fn empty_ctx() -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri("/test") + .header("x-reader-ip", "198.51.100.7") + .header("x-reader-ip-auth", "fictional-shared-secret-0123456789") + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } + fn settings_with_response_headers(headers: Vec<(&str, &str)>) -> Settings { let mut s = Settings::from_toml( r#" @@ -197,4 +222,36 @@ mod tests { "should apply operator-configured response headers" ); } + + #[test] + fn finalize_middleware_strips_configured_trust_headers_before_routing() { + let mut settings = settings_with_response_headers(vec![]); + settings.trusted_client_ip = Some(TrustedClientIpConfig { + ip_header: "x-reader-ip".to_owned(), + auth_header: "x-reader-ip-auth".to_owned(), + shared_secret: Redacted::new("fictional-shared-secret-0123456789".to_owned()), + }); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let observed = Arc::new(Mutex::new(None)); + let handler_observed = Arc::clone(&observed); + let handler = Arc::new(move |ctx: RequestContext| { + let handler_observed = Arc::clone(&handler_observed); + async move { + *handler_observed.lock().expect("should lock observation") = Some(( + ctx.request().headers().contains_key("x-reader-ip"), + ctx.request().headers().contains_key("x-reader-ip-auth"), + )); + Ok::(empty_response()) + } + }); + + block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should run middleware"); + + assert_eq!( + *observed.lock().expect("should lock observation"), + Some((false, false)), + "should remove both configured trust headers before the handler" + ); + } } diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 5b605bcff..745c793cd 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -7,6 +7,7 @@ use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; +use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -35,7 +36,7 @@ impl FinalizeResponseMiddleware { #[async_trait(?Send)] impl Middleware for FinalizeResponseMiddleware { - async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { let geo_available = ctx .request() .headers() @@ -44,6 +45,11 @@ impl Middleware for FinalizeResponseMiddleware { .filter(|s| !s.is_empty() && *s != "XX") .is_some(); + sanitize_trusted_client_ip_headers( + ctx.request_mut(), + self.settings.trusted_client_ip.as_ref(), + ); + let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, geo_available, &mut response); Ok(response) @@ -124,8 +130,17 @@ pub(crate) fn apply_finalize_headers( mod tests { use super::*; + use std::collections::HashMap; + use std::sync::Mutex; + use edgezero_core::body::Body; - use edgezero_core::http::response_builder; + use edgezero_core::context::RequestContext; + use edgezero_core::http::{Method, request_builder, response_builder}; + use edgezero_core::middleware::Next; + use edgezero_core::params::PathParams; + use futures::executor::block_on; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::TrustedClientIpConfig; fn empty_response() -> Response { response_builder() @@ -133,6 +148,17 @@ mod tests { .expect("should build empty test response") } + fn empty_ctx() -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri("/test") + .header("x-reader-ip", "198.51.100.7") + .header("x-reader-ip-auth", "fictional-shared-secret-0123456789") + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } + fn settings_with_response_headers(headers: Vec<(&str, &str)>) -> Settings { // Build from explicit test settings: the settings baked into the // binary contain placeholder secrets that `get_settings()` rejects @@ -230,4 +256,36 @@ mod tests { "should apply operator-configured response headers" ); } + + #[test] + fn finalize_middleware_strips_configured_trust_headers_before_routing() { + let mut settings = settings_with_response_headers(vec![]); + settings.trusted_client_ip = Some(TrustedClientIpConfig { + ip_header: "x-reader-ip".to_owned(), + auth_header: "x-reader-ip-auth".to_owned(), + shared_secret: Redacted::new("fictional-shared-secret-0123456789".to_owned()), + }); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let observed = Arc::new(Mutex::new(None)); + let handler_observed = Arc::clone(&observed); + let handler = Arc::new(move |ctx: RequestContext| { + let handler_observed = Arc::clone(&handler_observed); + async move { + *handler_observed.lock().expect("should lock observation") = Some(( + ctx.request().headers().contains_key("x-reader-ip"), + ctx.request().headers().contains_key("x-reader-ip-auth"), + )); + Ok::(empty_response()) + } + }); + + block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should run middleware"); + + assert_eq!( + *observed.lock().expect("should lock observation"), + Some((false, false)), + "should remove both configured trust headers before the handler" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 41e5e65ee..5e78a87ed 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -2195,13 +2195,27 @@ mod tests { server_region: Some("US-East".to_string()), }); - let _ = route(&router, req); + let response = route(&router, req); let observed = captured .lock() .expect("should lock captured client info") .clone() .expect("request filter should have observed the entry-point ClientInfo"); + assert_eq!( + observed.client_ip, + Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7))), + "request-scoped services should preserve the resolved client IP used by EC" + ); + let finalize = response + .extensions() + .get::() + .expect("fallback response should carry EC finalization state"); + assert_eq!( + finalize.ec_context.client_ip(), + Some("203.0.113.7"), + "EC should capture the resolved client IP from request-scoped services" + ); assert_eq!( observed.tls_protocol.as_deref(), Some("TLSv1.3"), diff --git a/crates/trusted-server-adapter-fastly/src/compat.rs b/crates/trusted-server-adapter-fastly/src/compat.rs index b46e8cc9b..5ccd9c423 100644 --- a/crates/trusted-server-adapter-fastly/src/compat.rs +++ b/crates/trusted-server-adapter-fastly/src/compat.rs @@ -1,8 +1,13 @@ //! Compatibility bridge between `fastly` SDK types and `http` crate types. +use std::net::IpAddr; + use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::Response as HttpResponse; use trusted_server_core::http_util::SPOOFABLE_FORWARDED_HEADERS; +use trusted_server_core::settings::TrustedClientIpConfig; + +use crate::platform::resolve_client_ip; /// Convert an [`HttpResponse`] into a `fastly::Response`. pub(crate) fn to_fastly_response(resp: HttpResponse) -> fastly::Response { @@ -52,9 +57,18 @@ pub(crate) fn to_fastly_response_skeleton(resp: HttpResponse) -> fastly::Respons /// Sanitize forwarded headers on a `fastly::Request`. /// -/// Strips headers that clients can spoof before any request-derived context -/// is built or the request is converted to core HTTP types. -pub(crate) fn sanitize_fastly_forwarded_headers(req: &mut fastly::Request) { +/// Strips configured trust headers and headers that clients can spoof before +/// any request-derived context is built or the request is converted to core +/// HTTP types. +pub(crate) fn sanitize_fastly_forwarded_headers( + req: &mut fastly::Request, + config: Option<&TrustedClientIpConfig>, +) { + if let Some(config) = config { + req.remove_header(config.ip_header.as_str()); + req.remove_header(config.auth_header.as_str()); + } + for &name in SPOOFABLE_FORWARDED_HEADERS { if req.get_header(name).is_some() { log::debug!("Stripped spoofable header: {name}"); @@ -63,9 +77,32 @@ pub(crate) fn sanitize_fastly_forwarded_headers(req: &mut fastly::Request) { } } +/// Resolve the trusted client IP, then strip every trust and spoofable header. +/// +/// Resolution has to observe the trust headers *before* sanitization removes +/// them. Both steps live behind this one call so that ordering is structural +/// rather than a convention the entry point has to remember. +pub(crate) fn resolve_and_sanitize_client_ip( + req: &mut fastly::Request, + config: Option<&TrustedClientIpConfig>, +) -> Option { + let client_ip = resolve_client_ip(req, req.get_client_ip_addr(), config); + sanitize_fastly_forwarded_headers(req, config); + client_ip +} + #[cfg(test)] mod tests { use super::*; + use trusted_server_core::redacted::Redacted; + + fn trusted_client_ip_config(ip_header: &str) -> TrustedClientIpConfig { + TrustedClientIpConfig { + ip_header: ip_header.to_owned(), + auth_header: "x-trusted-client-auth".to_owned(), + shared_secret: Redacted::new("fictional-shared-secret-0123456789".to_owned()), + } + } #[test] fn sanitize_fastly_forwarded_headers_strips_spoofable() { @@ -74,9 +111,10 @@ mod tests { req.set_header("x-forwarded-host", "evil.example.com"); req.set_header("x-forwarded-proto", "http"); req.set_header("fastly-ssl", "1"); + req.set_header("fastly-client-ip", "198.51.100.7"); req.set_header("host", "example.com"); - sanitize_fastly_forwarded_headers(&mut req); + sanitize_fastly_forwarded_headers(&mut req, None); assert!( req.get_header("forwarded").is_none(), @@ -94,9 +132,86 @@ mod tests { req.get_header("fastly-ssl").is_none(), "should strip fastly-ssl" ); + assert!( + req.get_header("fastly-client-ip").is_none(), + "should strip fastly-client-ip" + ); + assert!(req.get_header("host").is_some(), "should preserve host"); + } + + #[test] + fn sanitize_fastly_forwarded_headers_strips_configured_headers() { + let config = trusted_client_ip_config("x-trusted-client-ip"); + let mut req = fastly::Request::get("https://example.com/"); + req.set_header("x-trusted-client-ip", "198.51.100.7"); + req.set_header( + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + ); + req.set_header("host", "example.com"); + + sanitize_fastly_forwarded_headers(&mut req, Some(&config)); + + assert!( + req.get_header("x-trusted-client-ip").is_none(), + "should strip the configured IP header" + ); + assert!( + req.get_header("x-trusted-client-auth").is_none(), + "should strip the configured auth header" + ); assert!(req.get_header("host").is_some(), "should preserve host"); } + #[test] + fn sanitize_fastly_forwarded_headers_allows_static_and_dynamic_overlap() { + let config = trusted_client_ip_config("fastly-client-ip"); + let mut req = fastly::Request::get("https://example.com/"); + req.set_header("fastly-client-ip", "198.51.100.7"); + req.set_header( + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + ); + + sanitize_fastly_forwarded_headers(&mut req, Some(&config)); + + assert!( + req.get_header("fastly-client-ip").is_none(), + "should tolerate removing fastly-client-ip twice" + ); + assert!( + req.get_header("x-trusted-client-auth").is_none(), + "should strip the configured auth header" + ); + } + + #[test] + fn resolve_and_sanitize_client_ip_reads_trust_headers_before_stripping_them() { + let config = trusted_client_ip_config("x-trusted-client-ip"); + let mut req = fastly::Request::get("https://example.com/"); + req.set_header("x-trusted-client-ip", "198.51.100.7"); + req.set_header( + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + ); + + let resolved = resolve_and_sanitize_client_ip(&mut req, Some(&config)); + + assert_eq!( + resolved, + Some(IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7))), + "should resolve the forwarded IP before sanitization removes the headers" + ); + assert!( + req.get_header("x-trusted-client-ip").is_none(), + "should strip the configured IP header after resolving" + ); + assert!( + req.get_header("x-trusted-client-auth").is_none(), + "should strip the configured auth header after resolving" + ); + } + #[test] fn to_fastly_response_with_streaming_body_produces_empty_body() { use edgezero_core::http::StatusCode; diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index a19d0485d..743d6eafd 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -124,9 +124,14 @@ fn edgezero_main(mut req: FastlyRequest) { let (app, app_state) = TrustedServerApp::build_app_with_state(); let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); + let trusted_client_ip = settings_snapshot + .as_deref() + .and_then(|settings| settings.trusted_client_ip.as_ref()); - // Strip client-spoofable forwarded headers before dispatch. - compat::sanitize_fastly_forwarded_headers(&mut req); + // Resolve the trusted client IP, then strip client-spoofable forwarded + // headers before dispatch. One call keeps resolution ahead of the + // sanitization that removes the headers it reads. + let resolved_client_ip = compat::resolve_and_sanitize_client_ip(&mut req, trusted_client_ip); // Re-inject a trusted TLS scheme signal after sanitization has stripped any // client-sent fastly-ssl header. Setting it from Fastly's native TLS @@ -138,9 +143,6 @@ fn edgezero_main(mut req: FastlyRequest) { req.set_header("fastly-ssl", "1"); } - // Capture client IP before the request is consumed by dispatch. - let client_ip = req.get_client_ip_addr(); - // Strip any client-supplied x-ts-tls-* headers before injecting the trusted // values from the Fastly SDK. Must run after sanitize_fastly_forwarded_headers. req.remove_header("x-ts-tls-protocol"); @@ -160,7 +162,8 @@ fn edgezero_main(mut req: FastlyRequest) { // Capture metadata from the original FastlyRequest before conversion. These // accessors only return real values on the client request, so store them in // request extensions for build_per_request_services and EC bot classification. - let client_info = client_info_from_request(&req); + let client_info = client_info_from_request(&req, resolved_client_ip); + let client_ip = client_info.client_ip; let device_signals = derive_device_signals(&req); // Dispatch directly through the EdgeZero router without an intermediate @@ -602,7 +605,7 @@ mod tests { .headers() .get("cache-control") .and_then(|value| value.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!(response.headers().get("surrogate-control").is_none()); assert!(response.headers().get("cdn-cache-control").is_none()); @@ -628,7 +631,7 @@ mod tests { .headers() .get("cache-control") .and_then(|value| value.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!(response.headers().get("surrogate-control").is_none()); assert!(response.headers().get("cdn-cache-control").is_none()); @@ -713,7 +716,7 @@ mod tests { .headers() .get("cache-control") .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "request-scoped diagnostics HTML must never become shared-cacheable" ); assert!( diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 153d90295..8329c46b1 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -25,7 +25,7 @@ use trusted_server_core::constants::{ HEADER_X_TS_ENV, HEADER_X_TS_VERSION, }; use trusted_server_core::geo::GeoInfo; -use trusted_server_core::platform::PlatformGeo; +use trusted_server_core::platform::{ClientInfo, PlatformGeo}; use trusted_server_core::settings::Settings; pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; @@ -67,7 +67,10 @@ impl FinalizeResponseMiddleware { #[async_trait(?Send)] impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { - let client_ip = FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip); + let client_ip = ctx.request().extensions().get::().map_or_else( + || FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), + |info| info.client_ip, + ); let mut response = match next.run(ctx).await { Ok(r) => r, @@ -249,7 +252,7 @@ mod tests { use std::collections::HashMap; use std::net::IpAddr; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; use edgezero_core::context::RequestContext; @@ -259,7 +262,8 @@ mod tests { use edgezero_core::params::PathParams; use error_stack::Report; use futures::executor::block_on; - use trusted_server_core::platform::{PlatformError, PlatformGeo}; + use trusted_server_core::platform::{ClientInfo, PlatformError, PlatformGeo}; + use trusted_server_core::response_privacy::apply_inactive_ad_stack_browser_cache_policy; fn empty_response() -> Response { response_builder() @@ -284,6 +288,23 @@ mod tests { } } + struct RecordingGeo { + lookups: Arc>>>, + } + + impl PlatformGeo for RecordingGeo { + fn lookup( + &self, + client_ip: Option, + ) -> Result, Report> { + self.lookups + .lock() + .expect("should lock recorded geo lookups") + .push(client_ip); + Ok(None) + } + } + fn test_settings() -> Settings { Settings::from_toml( r#" @@ -431,29 +452,38 @@ mod tests { } #[test] - fn enforce_set_cookie_cache_privacy_downgrades_late_cookie() { + fn enforce_set_cookie_cache_privacy_downgrades_late_cookie_policies() { // Mirrors the EdgeZero post-ec_finalize guard: a Set-Cookie added after - // finalize headers ran (origin-public response) must be downgraded. - let mut response = response_with_headers(&[ - ("set-cookie", "ts-ec=abc; Path=/"), - ("cache-control", "public, max-age=600"), - ("surrogate-control", "max-age=600"), - ]); + // finalize headers ran must override both origin-public and inactive + // template cache policies. + for (cache_control, generated_inactive_policy) in [ + ("public, max-age=600", false), + ("private, max-age=60", true), + ] { + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", cache_control), + ("surrogate-control", "max-age=600"), + ]); + if generated_inactive_policy { + apply_inactive_ad_stack_browser_cache_policy(&mut response); + } - enforce_set_cookie_cache_privacy(&mut response); + enforce_set_cookie_cache_privacy(&mut response); - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("private, max-age=0"), - "should downgrade a late public cookie response to private" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "should strip surrogate-control from the late cookie response" - ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade {cache_control} on a cookie response" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from a {cache_control} cookie response" + ); + } } #[test] @@ -525,6 +555,110 @@ mod tests { // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- + #[test] + fn finalize_handle_uses_client_info_ip_for_geo_lookup() { + let reader_ip = IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7)); + let peer_ip = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); + let settings = settings_with_response_headers(vec![]); + let lookups = Arc::new(Mutex::new(Vec::new())); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(RecordingGeo { + lookups: Arc::clone(&lookups), + }), + ); + let mut ctx = empty_ctx(); + ctx.request_mut().extensions_mut().insert(ClientInfo { + client_ip: Some(reader_ip), + ..ClientInfo::default() + }); + FastlyRequestContext::insert( + ctx.request_mut(), + FastlyRequestContext { + client_ip: Some(peer_ip), + }, + ); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); + + assert_eq!( + *lookups.lock().expect("should lock recorded geo lookups"), + vec![Some(reader_ip)], + "should look up geo using the resolved ClientInfo IP" + ); + } + + #[test] + fn finalize_handle_preserves_none_from_present_client_info() { + let peer_ip = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); + let settings = settings_with_response_headers(vec![]); + let lookups = Arc::new(Mutex::new(Vec::new())); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(RecordingGeo { + lookups: Arc::clone(&lookups), + }), + ); + let mut ctx = empty_ctx(); + ctx.request_mut() + .extensions_mut() + .insert(ClientInfo::default()); + FastlyRequestContext::insert( + ctx.request_mut(), + FastlyRequestContext { + client_ip: Some(peer_ip), + }, + ); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); + + assert_eq!( + *lookups.lock().expect("should lock recorded geo lookups"), + vec![None], + "should preserve no IP from the authoritative ClientInfo" + ); + } + + #[test] + fn finalize_handle_uses_fastly_context_ip_when_client_info_is_absent() { + let peer_ip = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); + let settings = settings_with_response_headers(vec![]); + let lookups = Arc::new(Mutex::new(Vec::new())); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(RecordingGeo { + lookups: Arc::clone(&lookups), + }), + ); + let mut ctx = empty_ctx(); + FastlyRequestContext::insert( + ctx.request_mut(), + FastlyRequestContext { + client_ip: Some(peer_ip), + }, + ); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); + + assert_eq!( + *lookups.lock().expect("should lock recorded geo lookups"), + vec![Some(peer_ip)], + "should fall back to the Fastly request context IP" + ); + } + #[test] fn finalize_handle_injects_geo_unavailable_on_ok_response() { let settings = settings_with_response_headers(vec![]); diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 9e7920e1c..638aed82b 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -21,6 +21,7 @@ use trusted_server_core::platform::{ PlatformImageOptimizerRegion, PlatformKvStore, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, }; +use trusted_server_core::settings::TrustedClientIpConfig; // --------------------------------------------------------------------------- // FastlyPlatformConfigStore @@ -694,7 +695,53 @@ impl PlatformGeo for FastlyPlatformGeo { } } -/// Extract [`ClientInfo`] from the original Fastly request. +fn single_utf8_header<'a>(req: &'a Request, name: &str) -> Option<&'a str> { + let mut values = req.get_header_all(name); + let value = values.next()?; + if values.next().is_some() { + return None; + } + value.to_str().ok() +} + +/// Resolve the request's client IP from an authenticated forwarding header. +/// +/// When no trusted-client-IP configuration is present, or when either header +/// is missing, duplicated, malformed, or unauthenticated, this returns the +/// Fastly SDK peer address unchanged. +/// +/// Every fallback taken while a configuration *is* present logs at debug level +/// so a rotated secret or renamed header is diagnosable. Debug rather than warn +/// keeps a direct client from driving log volume by sending junk trust headers. +#[must_use] +pub(crate) fn resolve_client_ip( + req: &Request, + peer_ip: Option, + config: Option<&TrustedClientIpConfig>, +) -> Option { + let Some(config) = config else { + return peer_ip; + }; + let Some(auth_candidate) = single_utf8_header(req, &config.auth_header) else { + log::debug!("Trusted client IP: auth header is missing, duplicated, or not UTF-8"); + return peer_ip; + }; + if !config.authenticates(auth_candidate) { + log::debug!("Trusted client IP: auth header did not match the configured shared secret"); + return peer_ip; + } + let Some(ip_candidate) = single_utf8_header(req, &config.ip_header) else { + log::debug!("Trusted client IP: IP header is missing, duplicated, or not UTF-8"); + return peer_ip; + }; + + ip_candidate.parse::().ok().or_else(|| { + log::debug!("Trusted client IP: IP header is not a bare IPv4 or IPv6 address"); + peer_ip + }) +} + +/// Extract [`ClientInfo`] from the original Fastly request and resolved client IP. /// /// Fastly's TLS, JA4, and HTTP/2 fingerprint accessors only return real values /// on the client request before it is converted to platform HTTP types. This @@ -703,9 +750,9 @@ impl PlatformGeo for FastlyPlatformGeo { /// extensions so `build_per_request_services` can read back metadata the /// reconstructed request cannot expose. #[must_use] -pub fn client_info_from_request(req: &Request) -> ClientInfo { +pub fn client_info_from_request(req: &Request, client_ip: Option) -> ClientInfo { ClientInfo { - client_ip: req.get_client_ip_addr(), + client_ip, tls_protocol: req.get_tls_protocol().ok().flatten().map(str::to_string), tls_cipher: req .get_tls_cipher_openssl_name() @@ -741,6 +788,184 @@ mod tests { use super::*; use edgezero_core::body::Body; use edgezero_core::http::request_builder; + use fastly::http::HeaderValue; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::TrustedClientIpConfig; + + const PEER_IP: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); + + fn trusted_client_ip_config() -> TrustedClientIpConfig { + TrustedClientIpConfig { + ip_header: "fastly-client-ip".to_owned(), + auth_header: "x-trusted-client-auth".to_owned(), + shared_secret: Redacted::new("fictional-shared-secret-0123456789".to_owned()), + } + } + + fn authenticated_request(ip: impl AsRef<[u8]>) -> Request { + let mut req = Request::get("https://example.com/"); + req.set_header( + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + ); + req.set_header("fastly-client-ip", ip.as_ref()); + req + } + + #[test] + fn resolve_client_ip_uses_peer_when_config_is_absent() { + let req = authenticated_request("198.51.100.7"); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), None); + + assert_eq!(resolved, Some(PEER_IP), "should preserve the peer IP"); + } + + #[test] + fn resolve_client_ip_accepts_authenticated_ipv4() { + let req = authenticated_request("198.51.100.7"); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!( + resolved, + Some(IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7))), + "should use the authenticated IPv4 address" + ); + } + + #[test] + fn resolve_client_ip_accepts_authenticated_ipv6() { + let req = authenticated_request("2001:db8::7"); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!( + resolved, + Some(IpAddr::V6(std::net::Ipv6Addr::new( + 0x2001, 0xdb8, 0, 0, 0, 0, 0, 7, + ))), + "should use the authenticated IPv6 address" + ); + } + + #[test] + fn resolve_client_ip_uses_peer_when_auth_is_missing_empty_or_wrong() { + for auth_value in [None, Some(""), Some("fictional-wrong-secret")] { + let mut req = Request::get("https://example.com/"); + if let Some(auth_value) = auth_value { + req.set_header("x-trusted-client-auth", auth_value); + } + req.set_header("fastly-client-ip", "198.51.100.7"); + + let resolved = + resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!( + resolved, + Some(PEER_IP), + "should fall back for auth value {auth_value:?}" + ); + } + } + + #[test] + fn resolve_client_ip_uses_peer_when_auth_is_duplicated() { + let mut req = authenticated_request("198.51.100.7"); + req.append_header( + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + ); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!(resolved, Some(PEER_IP), "should reject duplicate auth"); + } + + #[test] + fn resolve_client_ip_uses_peer_when_auth_is_not_utf8() { + let mut req = Request::get("https://example.com/"); + req.set_header( + "x-trusted-client-auth", + HeaderValue::from_bytes(b"fictional-shared-secret-0123456789\xff") + .expect("should build non-UTF-8 auth header"), + ); + req.set_header("fastly-client-ip", "198.51.100.7"); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!(resolved, Some(PEER_IP), "should reject non-UTF-8 auth"); + } + + #[test] + fn resolve_client_ip_uses_peer_when_ip_is_missing() { + let mut req = Request::get("https://example.com/"); + req.set_header( + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + ); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!(resolved, Some(PEER_IP), "should require an IP header"); + } + + #[test] + fn resolve_client_ip_uses_peer_when_ip_text_is_invalid() { + for ip_value in [ + " 198.51.100.7", + "198.51.100.7 ", + "198.51.100.7:443", + "2001:db8::7%example0", + "198.51.100.7, 203.0.113.10", + "", + ] { + let req = authenticated_request(ip_value); + + let resolved = + resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!( + resolved, + Some(PEER_IP), + "should reject invalid IP value {ip_value:?}" + ); + } + } + + #[test] + fn resolve_client_ip_uses_peer_when_ip_is_duplicated() { + let mut req = authenticated_request("198.51.100.7"); + req.append_header("fastly-client-ip", "203.0.113.10"); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!(resolved, Some(PEER_IP), "should reject duplicate IP values"); + } + + #[test] + fn resolve_client_ip_uses_peer_when_ip_is_not_utf8() { + let req = authenticated_request( + HeaderValue::from_bytes(b"198.51.100.7\xff").expect("should build non-UTF-8 IP header"), + ); + + let resolved = resolve_client_ip(&req, Some(PEER_IP), Some(&trusted_client_ip_config())); + + assert_eq!(resolved, Some(PEER_IP), "should reject non-UTF-8 IP"); + } + + #[test] + fn client_info_from_request_preserves_supplied_client_ip() { + let req = Request::get("https://example.com/"); + let supplied_ip = Some(IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7))); + + let client_info = client_info_from_request(&req, supplied_ip); + + assert_eq!( + client_info.client_ip, supplied_ip, + "should preserve the supplied client IP" + ); + } #[test] fn edge_request_to_fastly_replaces_url_derived_host_header() { diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 1bcede1fc..da68b27c8 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -7,6 +7,7 @@ use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; +use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -34,9 +35,14 @@ impl FinalizeResponseMiddleware { #[async_trait(?Send)] impl Middleware for FinalizeResponseMiddleware { - async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { let geo_available = false; + sanitize_trusted_client_ip_headers( + ctx.request_mut(), + self.settings.trusted_client_ip.as_ref(), + ); + let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, geo_available, &mut response); Ok(response) @@ -151,8 +157,17 @@ pub(crate) fn apply_finalize_headers( mod tests { use super::*; + use std::collections::HashMap; + use std::sync::Mutex; + use edgezero_core::body::Body; - use edgezero_core::http::response_builder; + use edgezero_core::context::RequestContext; + use edgezero_core::http::{Method, request_builder, response_builder}; + use edgezero_core::middleware::Next; + use edgezero_core::params::PathParams; + use futures::executor::block_on; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::TrustedClientIpConfig; fn empty_response() -> Response { response_builder() @@ -160,6 +175,17 @@ mod tests { .expect("should build empty test response") } + fn empty_ctx() -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri("/test") + .header("x-reader-ip", "198.51.100.7") + .header("x-reader-ip-auth", "fictional-shared-secret-0123456789") + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } + fn settings_with_response_headers(headers: Vec<(&str, &str)>) -> Settings { // Build from explicit test settings: the settings baked into the // binary contain placeholder secrets that `get_settings()` rejects @@ -257,4 +283,36 @@ mod tests { "should apply operator-configured response headers" ); } + + #[test] + fn finalize_middleware_strips_configured_trust_headers_before_routing() { + let mut settings = settings_with_response_headers(vec![]); + settings.trusted_client_ip = Some(TrustedClientIpConfig { + ip_header: "x-reader-ip".to_owned(), + auth_header: "x-reader-ip-auth".to_owned(), + shared_secret: Redacted::new("fictional-shared-secret-0123456789".to_owned()), + }); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let observed = Arc::new(Mutex::new(None)); + let handler_observed = Arc::clone(&observed); + let handler = Arc::new(move |ctx: RequestContext| { + let handler_observed = Arc::clone(&handler_observed); + async move { + *handler_observed.lock().expect("should lock observation") = Some(( + ctx.request().headers().contains_key("x-reader-ip"), + ctx.request().headers().contains_key("x-reader-ip-auth"), + )); + Ok::(empty_response()) + } + }); + + block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should run middleware"); + + assert_eq!( + *observed.lock().expect("should lock observation"), + Some((false, false)), + "should remove both configured trust headers before the handler" + ); + } } diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index fe9c3664b..e08114850 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -18,16 +18,21 @@ workspace = true chromiumoxide = { workspace = true } clap = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } +glob = { workspace = true } +http = { workspace = true } log = { workspace = true } regex = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } @@ -62,4 +67,5 @@ tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..48ab71f3e --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,830 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +use serde::Deserialize; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "decoded for schema stability; the collector records no APS calls" +)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "reserved decoded shape for the optional bids phase" +)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + #[allow(dead_code, reason = "reserved for the optional bids phase")] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: Option, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime omits this slot on this path", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + Some(dom.phase), + ) + } else { + (SlotStatus::Missing, None, None, None) + }; + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: MediaType::Banner, + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: MediaType::Video, + }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + assert!( + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!(result.slots[0].warnings.is_empty()); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..9392963ff --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,336 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Configured media type. + pub media_type: MediaType, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(&format!("./{relative}")) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..e12c9eebc --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,480 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator, +//! which assembles these wire types from the URL/gate context and comparison result. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: Some(EvidencePhaseJson::InitialLoad), + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..bee536146 --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,171 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +#[cfg(test)] +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..6938808f5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,241 @@ +// Bounded ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [] +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 128 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } +} + +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null + return [width, height] +} + +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" + }) +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase() + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag + } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: descriptor ? descriptor.enumerable : true, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + } + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) + } + } + return googletag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, + get() { + return internal + }, + set(value) { + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + } + }) +} + +__ts_install("googletag", __ts_wrap_googletag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = __ts_text(element.id) + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), + sizes, + phase: __ts_phase() + }) + } + } catch (error) { + __ts_warn("gpt_scrape_failed", error) + } + } + } + } catch (error) { + __ts_warn("collect_failed", error) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..cb11a0a0c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,1007 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; +use crate::run::RunOutcome; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { + args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + )?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.pages.iter().any(|page| page.error.is_some()) { + Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) + } +} + +fn validate_cookie_scope(urls: &[url::Url], cookies: &[(String, String)]) -> Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> Result { + let init_script = build_init_script(creative)?; + + let requests: Vec<_> = urls + .iter() + .map(|url| BrowserCollectRequest { + url: url.clone(), + init_scripts: vec![init_script.clone()], + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }) + .collect(); + let collected_pages = collector.collect_pages(&requests); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for (url, collected) in urls.iter().zip(collected_pages) { + match collected { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + Ok(VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + }) +} + +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + if requested.host_str() != final_url.host_str() { + return true; + } + + match (requested.scheme(), final_url.scheme()) { + ("http", "https") => { + requested.port_or_known_default() != Some(80) + || final_url.port_or_known_default() != Some(443) + } + (requested_scheme @ ("http" | "https"), final_scheme) + if requested_scheme == final_scheme => + { + requested.port_or_known_default() != final_url.port_or_known_default() + } + // Refuse HTTPS downgrades and any unexpected scheme transition. + _ => true, + } +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config) +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested} to {final_url}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates(matched, auction_enabled)), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: result.phase.map(to_phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: media_type_label(&format.media_type).to_string(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for warning in &report.warnings { + write_warning(out, "", warning)?; + } + for page in &report.pages { + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; + } + for slot in &page.slots { + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", + } +} + +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + batch_calls: Cell, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse final URL"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { + pages, + batch_calls: Cell::new(0), + } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + self.batch_calls.set(self.batch_calls.get() + 1); + requests + .iter() + .cloned() + .map(|request| self.collect_page(request)) + .collect() + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("should parse URL")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + .expect("typed collector configuration should serialize") + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("should have warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("should have slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn same_host_http_to_https_upgrade_is_accepted() { + let collector = FakeCollector::page( + "http://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["http://www.example.com/news/story"], + ); + + assert!(report.ok, "a default-port HTTPS upgrade should be accepted"); + } + + #[test] + fn downgrade_and_port_changes_are_rejected() { + for (requested, final_url) in [ + ( + "https://www.example.com/news/story", + "http://www.example.com/news/story", + ), + ( + "https://www.example.com:8443/news/story", + "https://www.example.com:9443/news/story", + ), + ( + "http://www.example.com:8080/news/story", + "https://www.example.com:8443/news/story", + ), + ] { + let collector = FakeCollector::page(requested, final_url, confirmed_news_evidence()); + let report = report_for(&collector, true, true, &[requested]); + assert!(!report.ok, "redirect {requested} -> {final_url} must fail"); + } + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "page_fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + assert_eq!( + collector.batch_calls.get(), + 1, + "all verifier URLs should use one collector batch" + ); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..c8cbffddf --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,1202 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, BrowserProfile, CollectedPage, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +pub(crate) const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "Google Chrome", + "Google Chrome for Testing", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Bound for each CDP operation after navigation. +const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); +/// Default quiet window (no new resources) marking the page settled. +const DEFAULT_SETTLE_QUIET_MS: u64 = 750; +/// Default hard cap on settling so slow/ad-heavy pages still terminate. +const DEFAULT_SETTLE_MAX_MS: u64 = 10_000; + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// Immutable browser/session settings shared by every URL in one audit batch. +struct BrowserSessionOptions<'a> { + chrome: &'a std::path::Path, + profile_dir: &'a std::path::Path, + settle: SettleConfig, + accept_invalid_certs: bool, + headful: bool, + assume_consent: bool, + proxy: Option<&'a str>, + profile: BrowserProfile, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, + /// Run visible Chrome rather than new headless Chrome. + headful: bool, + /// Install the standard consent API stub before publisher scripts. + assume_consent: bool, + /// Optional browser proxy endpoint. + proxy: Option, + /// Device viewport/user-agent profile. + profile: BrowserProfile, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(DEFAULT_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(DEFAULT_SETTLE_MAX_MS), + accept_invalid_certs: false, + headful: false, + assume_consent: true, + proxy: None, + profile: BrowserProfile::Desktop, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + headful: opts.headful, + assume_consent: !opts.no_assume_consent, + proxy: opts.browser_proxy.clone(), + profile: opts.profile, + } + } +} + +/// Pre-document consent behavior shared with the generation crawler. +pub(crate) const CONSENT_STUB_SCRIPT: &str = include_str!("consent_stub.js"); + +/// Shared browser launch inputs used by both audit collectors. +pub(crate) struct BrowserLaunchOptions<'a> { + pub(crate) chrome: &'a std::path::Path, + pub(crate) profile_dir: &'a std::path::Path, + pub(crate) headful: bool, + pub(crate) proxy: Option<&'a str>, + pub(crate) accept_invalid_certs: bool, + pub(crate) viewport: Viewport, + pub(crate) user_agent: Option<&'a str>, +} + +/// Builds the common Chrome configuration for all browser-backed audits. +pub(crate) fn build_browser_config( + options: BrowserLaunchOptions<'_>, +) -> Result { + let mut builder = BrowserConfig::builder() + .chrome_executable(options.chrome) + .user_data_dir(options.profile_dir); + if !options.accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = options.proxy { + let endpoint = if proxy.contains("://") { + proxy.to_string() + } else { + format!("http://{proxy}") + }; + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } + builder = if options.headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; + builder = builder + .window_size(options.viewport.width, options.viewport.height) + .viewport(options.viewport); + if let Some(user_agent) = options.user_agent { + builder = builder.arg(("user-agent", user_agent)); + } + builder + .build() + .map_err(|error| format!("failed to build browser config: {error}")) +} + +fn browser_profile(profile: BrowserProfile) -> (Viewport, Option<&'static str>) { + match profile { + BrowserProfile::Desktop => ( + Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + None, + ), + BrowserProfile::Mobile => ( + Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + ), + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +pub(crate) fn resolve_chrome( + override_path: Option<&std::path::Path>, +) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. +pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } + let mut cookie = CookieParam::new(name.to_string(), value.to_string()); + cookie.url = Some(origin.ascii_serialization()); + cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); + Ok(cookie) +} + +fn format_cookie_install_error(name: &str, _error: impl std::fmt::Display) -> String { + // Do not forward the CDP error: a browser implementation may include the + // rejected cookie value in its diagnostic. + format!("failed to set cookie `{name}`") +} + +/// Installs host-only, root-scoped cookies before a page has an origin. +pub(crate) async fn set_browser_cookies( + browser: &Browser, + cookies: &[(String, String)], + url: &url::Url, +) -> Result<(), String> { + for (name, value) in cookies { + let cookie = host_cookie(name, value, url)?; + browser + .set_cookies(vec![cookie]) + .await + .map_err(|error| format_cookie_install_error(name, error))?; + } + Ok(()) +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.collect_pages(std::slice::from_ref(&request)) + .into_iter() + .next() + .expect("should return one result for one browser request") + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + if requests.is_empty() { + return Vec::new(); + } + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = match resolve_chrome(self.chrome.as_deref()) { + Ok(chrome) => chrome, + Err(error) => return vec![Err(error); requests.len()], + }; + let profile = match tempfile::tempdir() { + Ok(profile) => profile, + Err(error) => { + let error = format!("failed to create browser profile dir: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let error = format!("failed to build browser runtime: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + let accept_invalid_certs = self.accept_invalid_certs; + let headful = self.headful; + let assume_consent = self.assume_consent; + let proxy = self.proxy.clone(); + let browser_profile = self.profile; + let request_count = requests.len(); + let requests = requests.to_vec(); + let result = runtime.block_on(async move { + let options = BrowserSessionOptions { + chrome: &chrome, + profile_dir: profile.path(), + settle, + accept_invalid_certs, + headful, + assume_consent, + proxy: proxy.as_deref(), + profile: browser_profile, + }; + collect(requests, &options).await + }); + match result { + Ok(results) => results, + Err(error) => vec![Err(error); request_count], + } + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + requests: Vec, + options: &BrowserSessionOptions<'_>, +) -> Result>, String> { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let (viewport, user_agent) = browser_profile(options.profile); + let config = build_browser_config(BrowserLaunchOptions { + chrome: options.chrome, + profile_dir: options.profile_dir, + headful: options.headful, + proxy: options.proxy, + accept_invalid_certs: options.accept_invalid_certs, + viewport, + user_agent, + })?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + results.push( + collect_with_browser(&browser, request, options.settle, options.assume_consent).await, + ); + } + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + Ok(results) +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + set_browser_cookies(browser, &request.cookies, &request.url).await?; + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + let result = collect_open_page(&page, &request, settle_config, assume_consent).await; + let close_result = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, page.close()).await; + + match (result, close_result) { + (Err(error), _) => Err(error), + (Ok(mut collected), Err(_)) => { + collected.warnings.push(Warning { + code: "page_close_timeout".to_string(), + message: "timed out closing the browser tab after collection".to_string(), + }); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(Warning { + code: "page_close_failed".to_string(), + message: format!("failed to close the browser tab after collection: {error}"), + }); + Ok(collected) + } + (Ok(collected), Ok(Ok(_))) => Ok(collected), + } +} + +/// Collects from an open tab. The caller owns tab teardown so every return path, +/// including an error from this function, closes the page before continuing. +async fn collect_open_page( + page: &Page, + request: &BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + let mut warnings = Vec::new(); + + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); + } + page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") + .await + .map_err(|error| format!("failed to increase resource timing buffer: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + match tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "navigation_wait_failed".to_string(), + message: format!( + "navigation load event could not be read ({error}); continuing with settled page evidence" + ), + }), + Err(_) => warnings.push(Warning { + code: "navigation_wait_timeout".to_string(), + message: format!( + "navigation did not fire its load event within {} seconds; continuing with settled page evidence", + NAVIGATION_TIMEOUT.as_secs() + ), + }), + } + + settle(page, settle_config, &mut warnings).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + if tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ), + ) + .await + .is_err() + { + warnings.push(Warning { + code: "ad_evidence_snapshot_timeout".to_string(), + message: "timed out snapshotting ad evidence before scroll".to_string(), + }); + } + } + scroll_page(page, &mut warnings).await; + settle(page, settle_config, &mut warnings).await; + } + + let final_url_text = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.url()) + .await + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or_else(|| "browser page URL was empty after navigation".to_string())?; + let final_url = url::Url::parse(&final_url_text).map_err(|error| { + format!("browser returned invalid final URL `{final_url_text}`: {error}") + })?; + let title = match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.get_title()).await { + Ok(Ok(title)) => title.unwrap_or_default(), + Ok(Err(error)) => { + warnings.push(Warning { + code: "page_title_failed".to_string(), + message: format!("failed to read page title: {error}"), + }); + String::new() + } + Err(_) => { + warnings.push(Warning { + code: "page_title_timeout".to_string(), + message: "timed out reading page title".to_string(), + }); + String::new() + } + }; + let script_count = eval_usize(page, "document.querySelectorAll('script').length") + .await + .unwrap_or_else(|message| { + warnings.push(Warning { + code: "script_count_failed".to_string(), + message, + }); + 0 + }); + let resource_count = resource_count(page).await.unwrap_or_else(|message| { + warnings.push(Warning { + code: "resource_count_failed".to_string(), + message, + }); + 0 + }); + + if resource_count >= 250 { + warnings.push(Warning { + code: "resource_timing_heavy".to_string(), + message: format!("page recorded {resource_count} network resources"), + }); + } + + if let Ok(Ok(frames)) = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.frames()).await + && frames.len() > 1 + { + warnings.push(Warning { + code: "child_frames_not_inspected".to_string(), + message: format!( + "ad-template evidence inspected only the main frame; {} child frame(s) were present", + frames.len() - 1 + ), + }); + } + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig, warnings: &mut Vec) { + let start = std::time::Instant::now(); + let mut last = None; + let mut quiet_since = None; + + loop { + if start.elapsed() >= config.max { + warnings.push(Warning { + code: "settle_timeout".to_string(), + message: "page did not settle before the configured maximum wait".to_string(), + }); + return; + } + + let ready_state = match eval_string(page, "document.readyState").await { + Ok(state) => state, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let current = match resource_count(page).await { + Ok(count) => count, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let ready = matches!(ready_state.as_str(), "interactive" | "complete"); + if ready && last == Some(current) { + let quiet_start = quiet_since.get_or_insert_with(std::time::Instant::now); + if quiet_start.elapsed() >= config.quiet { + return; + } + } else { + quiet_since = None; + } + last = Some(current); + + let remaining_max = config.max.saturating_sub(start.elapsed()); + let remaining_quiet = quiet_since + .map(|quiet_start| config.quiet.saturating_sub(quiet_start.elapsed())) + .unwrap_or(config.quiet); + let sleep_for = Duration::from_millis(SETTLE_POLL_MS) + .min(remaining_max) + .min(remaining_quiet.max(Duration::from_millis(1))); + tokio::time::sleep(sleep_for).await; + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> Result { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +/// Performs a deterministic stepped scroll to trigger lazy ad loading. +async fn scroll_page(page: &Page, warnings: &mut Vec) { + // Mark subsequent observations as scroll-phase for the collector. + eval_discard(page, "window.__tsScrollPhase = true", warnings).await; + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + eval_discard(page, script, warnings).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + eval_discard(page, "window.scrollTo(0, 0)", warnings).await; +} + +async fn eval_discard(page: &Page, expression: impl Into, warnings: &mut Vec) { + match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression.into())).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "page_evaluation_failed".to_string(), + message: format!("browser page evaluation failed: {error}"), + }), + Err(_) => warnings.push(Warning { + code: "page_evaluation_timeout".to_string(), + message: "browser page evaluation timed out".to_string(), + }), + } +} + +async fn eval_usize(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +async fn eval_string(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let evaluation = tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )), + ) + .await; + + let envelope = match evaluation { + Ok(Ok(result)) => match result.into_value::() { + Ok(envelope) => Some(envelope), + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence envelope: {error}"), + }); + return None; + } + }, + Ok(Err(error)) => { + warnings.push(Warning { + code: "ad_evidence_read_failed".to_string(), + message: format!("failed to read ad-template evidence: {error}"), + }); + return None; + } + Err(_) => { + warnings.push(Warning { + code: "ad_evidence_read_timeout".to_string(), + message: "timed out reading ad-template evidence".to_string(), + }); + return None; + } + }; + + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + } + } + } +} + +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + +#[cfg(test)] +mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + use std::sync::mpsc; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + const AD_TEMPLATE_COLLECTOR_JS: &str = include_str!("ad_template_collector.js"); + + #[test] + fn rust_and_javascript_evidence_entry_caps_match() { + let expected_declaration = format!("const __ts_max_entries = {MAX_EVIDENCE_ENTRIES}"); + assert!( + AD_TEMPLATE_COLLECTOR_JS + .lines() + .any(|line| line.trim() == expected_declaration), + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" + ); + } + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + + #[test] + fn supplied_cookie_is_host_only_and_root_scoped() { + let url = + url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); + let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); + + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); + assert_eq!(cookie.path.as_deref(), Some("/")); + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" + ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); + } + + #[test] + fn cookie_install_error_identifies_name_without_a_value() { + let error = format_cookie_install_error( + "datadome", + "invalid cookie value operator-secret-cookie-value", + ); + + assert_eq!(error, "failed to set cookie `datadome`"); + assert!(!error.contains("operator-secret-cookie-value")); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn supplied_cookie_reaches_first_navigation() { + if !browser_fixture_available() { + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let (request_tx, request_rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + request_tx + .send(String::from_utf8_lossy(&request).into_owned()) + .expect("should send captured request"); + + let body = b"cookie fixture"; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .expect("should write fixture headers"); + stream.write_all(body).expect("should write fixture body"); + }); + + let collector = BrowserCollector { + settle_quiet: Duration::from_millis(100), + settle_max: Duration::from_secs(1), + ..BrowserCollector::new() + }; + collector + .collect_page(BrowserCollectRequest { + url: url::Url::parse(&format!("http://{address}/")) + .expect("should parse fixture URL"), + init_scripts: Vec::new(), + scroll: false, + collect_ad_evidence: false, + cookies: vec![("clearance".to_string(), "token".to_string())], + }) + .expect("cookie should be installed before first navigation"); + + let request = request_rx + .recv_timeout(Duration::from_secs(5)) + .expect("fixture should receive the first navigation"); + assert!( + request.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("cookie") + && value + .trim() + .split(';') + .any(|cookie| cookie.trim() == "clearance=token") + }) + }), + "first navigation should carry the supplied cookie; request was {request:?}" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..64e62fbd0 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,340 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::{Args, ValueEnum}; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Default quiet window for generation's browser collector. +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for generation's browser collector. +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Browser device profile used for viewport and user-agent emulation. + #[arg(long = "browser-profile", value_enum, default_value_t = BrowserProfile::Desktop)] + pub profile: BrowserProfile, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = GENERATE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = GENERATE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: GENERATE_SETTLE_QUIET_MS, + settle_max_ms: GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } + } +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +/// Browser device profile shared by page audits and ad-template verification. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum BrowserProfile { + /// Desktop Chrome at 1280×800. + #[default] + Desktop, + /// Mobile-sized viewport with a mobile user agent. + Mobile, +} + +impl BrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub(crate) struct CollectedPage { - pub(crate) requested_url: String, - pub(crate) final_url: String, - pub(crate) page_title: Option, - pub(crate) html: String, - pub(crate) script_tags: Vec, - pub(crate) network_requests: Vec, - pub(crate) warnings: Vec, +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); + } + Ok(()) +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of ` "#; +/// Rendering owner for selected APS bids. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ApsRenderingMode { + /// Render through Trusted Server's opaque static renderer route. + #[default] + TrustedServer, + /// Render through the injected APS runner in a publisher-origin friendly frame. + PublisherNative, +} + /// Configuration for the APS `OpenRTB` integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[validate(schema(function = "validate_inventory_identity_override"))] @@ -141,6 +153,9 @@ pub struct ApsConfig { /// Whether APS script creatives are eligible before winner selection. #[serde(default)] pub allow_script_creatives: bool, + /// Rendering owner for selected APS bids. + #[serde(default)] + pub rendering_mode: ApsRenderingMode, /// APS-authorized inventory domain used instead of the deployment hostname. #[serde(default, skip_serializing_if = "Option::is_none")] #[validate(custom(function = "validate_inventory_domain"))] @@ -314,6 +329,7 @@ impl Default for ApsConfig { timeout_ms: default_timeout_ms(), debug: false, allow_script_creatives: false, + rendering_mode: ApsRenderingMode::TrustedServer, inventory_domain: None, inventory_page_origin: None, } @@ -1184,7 +1200,9 @@ impl AuctionProvider for ApsAuctionProvider { } #[derive(Debug)] -struct ApsRendererIntegration; +struct ApsRendererIntegration { + rendering_mode: ApsRenderingMode, +} #[async_trait(?Send)] impl IntegrationProxy for ApsRendererIntegration { @@ -1193,7 +1211,10 @@ impl IntegrationProxy for ApsRendererIntegration { } fn routes(&self) -> Vec { - vec![IntegrationEndpoint::get(APS_RENDERER_ROUTE)] + (self.rendering_mode == ApsRenderingMode::TrustedServer) + .then(|| IntegrationEndpoint::get(APS_RENDERER_ROUTE)) + .into_iter() + .collect() } async fn handle( @@ -1225,6 +1246,23 @@ impl IntegrationProxy for ApsRendererIntegration { } } +impl IntegrationHeadInjector for ApsRendererIntegration { + fn integration_id(&self) -> &'static str { + APS_INTEGRATION_ID + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + (self.rendering_mode == ApsRenderingMode::PublisherNative) + .then_some(("data-ts-aps-rendering-mode", "publisher_native")) + .into_iter() + .collect() + } +} + /// Register the APS static renderer endpoint when APS is enabled. /// /// # Errors @@ -1233,16 +1271,21 @@ impl IntegrationProxy for ApsRendererIntegration { pub fn register( settings: &Settings, ) -> Result, Report> { - let Some(_config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { return Ok(None); }; - let integration = Arc::new(ApsRendererIntegration); - Ok(Some( - IntegrationRegistration::builder(APS_INTEGRATION_ID) - .with_proxy(integration) - .without_js() - .build(), - )) + let integration = Arc::new(ApsRendererIntegration { + rendering_mode: config.rendering_mode, + }); + let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) + .without_js() + .with_head_injector(integration.clone()); + let registration = if config.rendering_mode == ApsRenderingMode::TrustedServer { + registration.with_proxy(integration) + } else { + registration + }; + Ok(Some(registration.build())) } /// Register the APS auction provider when enabled. @@ -1262,6 +1305,11 @@ pub fn register_providers( "APS debug mode is ON — raw request and response data, including creative markup, will be included in client-visible /auction responses" ); } + if config.rendering_mode == ApsRenderingMode::PublisherNative && config.allow_script_creatives { + log::warn!( + "APS publisher-native rendering with script creatives is ON; selected bidder scripts execute with publisher-origin privileges" + ); + } Ok(vec![Arc::new(ApsAuctionProvider::new(config))]) } @@ -1273,6 +1321,7 @@ mod tests { UserInfo, }; use crate::consent::ConsentContext; + use crate::integrations::IntegrationDocumentState; use crate::openrtb::{Eid, Uid}; use crate::platform::GeoInfo; use crate::platform::test_support::{ @@ -1289,6 +1338,7 @@ mod tests { timeout_ms: 800, debug: false, allow_script_creatives: false, + rendering_mode: ApsRenderingMode::TrustedServer, inventory_domain: None, inventory_page_origin: None, } @@ -1405,6 +1455,7 @@ mod tests { assert!(!canonical.debug); assert!(debug.debug); assert!(!canonical.allow_script_creatives); + assert_eq!(canonical.rendering_mode, ApsRenderingMode::TrustedServer); assert!(canonical.endpoint.ends_with("/e/pb/bid")); } @@ -1466,6 +1517,14 @@ mod tests { })) .is_err() ); + assert!( + serde_json::from_value::(json!({ + "account_id": "example-account", + "rendering_mode": "unsupported" + })) + .is_err(), + "should reject an unknown APS rendering mode" + ); for endpoint in [ "http://aps.example/e/pb/bid", "https://", @@ -2319,7 +2378,9 @@ mod tests { #[test] fn registers_and_serves_only_static_renderer_route() { - let integration = ApsRendererIntegration; + let integration = ApsRendererIntegration { + rendering_mode: ApsRenderingMode::TrustedServer, + }; let routes = integration.routes(); assert_eq!(routes.len(), 1, "should register one route"); assert_eq!(routes[0].method, Method::GET); @@ -2374,9 +2435,103 @@ mod tests { assert_eq!(registration.integration_id, APS_INTEGRATION_ID); assert_eq!(registration.proxies.len(), 1); + assert_eq!(registration.head_injectors.len(), 1); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "publisher.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + assert!( + registration.head_injectors[0] + .head_inserts(&context) + .is_empty(), + "should not inject a native-mode head marker by default" + ); + assert!( + registration.head_injectors[0] + .tsjs_script_tag_attributes() + .is_empty(), + "should not authorize native rendering by default" + ); assert!(registration.js_disabled); } + #[test] + fn publisher_native_config_registers_runner_mode_without_renderer_route() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + APS_INTEGRATION_ID, + &json!({ + "enabled": true, + "account_id": "example-account", + "rendering_mode": "publisher_native" + }), + ) + .expect("should insert native APS config"); + + let registration = register(&settings) + .expect("should register APS") + .expect("should return enabled registration"); + assert!( + registration.proxies.is_empty(), + "should not register the static renderer" + ); + assert_eq!(registration.head_injectors.len(), 1); + + let integration = ApsRendererIntegration { + rendering_mode: ApsRenderingMode::PublisherNative, + }; + assert!( + integration.routes().is_empty(), + "should expose no renderer route" + ); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "publisher.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + assert!( + integration.head_inserts(&context).is_empty(), + "should not inject a forgeable native-mode marker" + ); + assert_eq!( + integration.tsjs_script_tag_attributes(), + vec![("data-ts-aps-rendering-mode", "publisher_native")], + "should authorize native mode on the publisher bundle tag" + ); + } + + #[test] + fn publisher_native_script_creatives_remain_available_for_controlled_validation() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + APS_INTEGRATION_ID, + &json!({ + "enabled": true, + "account_id": "example-account", + "allow_script_creatives": true, + "rendering_mode": "publisher_native" + }), + ) + .expect("should insert native APS script config"); + + let providers = register_providers(&settings).expect("should register APS provider"); + + assert_eq!( + providers.len(), + 1, + "should retain the controlled experiment" + ); + } + #[test] fn config_without_enabled_does_not_register_provider_or_renderer() { let mut settings = create_test_settings(); @@ -2429,6 +2584,10 @@ mod tests { assert!(APS_RENDERER_DOCUMENT.contains("message.nonce!==expected")); assert!(APS_RENDERER_DOCUMENT.contains("prebid/creative/render")); assert!(APS_RENDERER_DOCUMENT.contains("window._aps instanceof Map")); + assert!( + APS_RENDERER_DOCUMENT + .contains("html,body{margin:0;padding:0}body>iframe{display:block}") + ); assert!(APS_RENDERER_DOCUMENT.contains("store:new Map([['listeners',new Map()]])")); assert!(APS_RENDERER_DOCUMENT.contains("account.queue.push(new CustomEvent")); assert!( diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index d95ee35ee..d47999a68 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -68,7 +68,6 @@ use serde_json::Value as JsonValue; use url::Url; use validator::Validate; -use crate::constants::ENV_FASTLY_IS_STAGING; use crate::error::TrustedServerError; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, @@ -90,7 +89,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; -/// Fixed request header used by the staging-only protection test bypass. +/// Fixed request header used by the configuration-gated protection test bypass. pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic @@ -122,7 +121,7 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { /// Temporary static-header bypass for server-side `DataDome` protection. /// -/// This is intended only for an access-controlled staging environment. A +/// This is intended only for an access-controlled test environment. A /// matching `x-ts-datadome-bypass` header bypasses the server-side Protection /// API and is removed before the publisher origin receives the request. The /// credential itself is loaded from the Secret Store at runtime. @@ -224,7 +223,7 @@ pub struct DataDomeConfig { )] pub protection_exclusion_rules: Vec, - /// Temporary static-header bypass for access-controlled staging tests. + /// Temporary static-header bypass for access-controlled tests. #[serde(default)] pub protection_test_bypass: Option, @@ -478,10 +477,6 @@ impl DataDomeIntegration { } fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { - if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() != Ok("1") { - return None; - } - self.config .protection_test_bypass .as_ref() @@ -945,17 +940,7 @@ fn build( }; let integration = DataDomeIntegration::try_new(config)?; - let protection_test_bypass_configured = integration - .config - .protection_test_bypass - .as_ref() - .is_some_and(|bypass| bypass.enabled); let protection_test_bypass_active = integration.active_protection_test_bypass().is_some(); - if protection_test_bypass_configured && !protection_test_bypass_active { - log::warn!( - "[datadome] DataDome test bypass is configured but inactive because FASTLY_IS_STAGING is not 1" - ); - } log::info!( "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", integration.config.sdk_origin, @@ -963,8 +948,6 @@ fn build( integration.config.enable_protection, if protection_test_bypass_active { "active" - } else if protection_test_bypass_configured { - "configured-inactive" } else { "disabled" }, diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 75de88afb..4d1988b6a 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -836,7 +836,7 @@ fn truncate_utf8(value: &str, limit: i32) -> String { mod tests { use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, @@ -852,8 +852,6 @@ mod tests { use super::*; - static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); - fn protection_integration() -> Arc { let config = DataDomeConfig { enabled: true, @@ -871,24 +869,19 @@ mod tests { .expect("should build filter request") } - fn filter_with_staging( + fn filter_request( integration: &DataDomeIntegration, settings: &Settings, services: &RuntimeServices, request: &mut Request, ) -> RequestFilterDecision { - let _guard = FASTLY_IS_STAGING_ENV_LOCK - .lock() - .expect("should lock staging environment test guard"); - temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { - futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { - settings, - services, - request, - geo_info: None, - is_integration_route: false, - })) - }) + futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { + settings, + services, + request, + geo_info: None, + is_integration_route: false, + })) } fn filter_marks_request( @@ -973,7 +966,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1036,7 +1029,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("stale-test-credential"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1062,7 +1055,7 @@ mod tests { } #[test] - fn protection_test_bypass_is_inactive_outside_staging() { + fn protection_test_bypass_works_without_staging_environment() { let config = DataDomeConfig { enabled: true, enable_protection: true, @@ -1075,20 +1068,11 @@ mod tests { }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); secrets.insert( "datadome_test_bypass".to_string(), b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); let services = build_services_with_secret_and_http_client( HashMapSecretStore::new(secrets), http_client.clone(), @@ -1100,44 +1084,26 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); - let _guard = FASTLY_IS_STAGING_ENV_LOCK - .lock() - .expect("should lock staging environment test guard"); - let decision = temp_env::with_var( - crate::constants::ENV_FASTLY_IS_STAGING, - None::<&str>, - || { - futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )) - }, - ); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), - "an allowed Protection API response should continue" + "a matching test credential should continue without a staging environment" + ); + assert!( + has_client_tag_suppression_marker(&request), + "the bypass should suppress the automatic DataDome client tag" ); assert!( request .headers() .get(super::super::HEADER_DATADOME_TEST_BYPASS) .is_none(), - "the bypass credential must be stripped outside staging" + "the bypass credential must not reach the publisher origin" ); assert!( - !has_client_tag_suppression_marker(&request), - "the bypass must not suppress the DataDome client tag outside staging" - ); - assert_eq!( - http_client.recorded_backend_names().len(), - 1, - "the bypass must still call the Protection API outside staging" + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" ); } @@ -1179,7 +1145,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1234,7 +1200,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1302,7 +1268,7 @@ mod tests { ); } - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!(matches!(decision, RequestFilterDecision::Continue(_))); assert!( @@ -1368,7 +1334,7 @@ mod tests { .expect("should build bypass header"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!(matches!(decision, RequestFilterDecision::Continue(_))); assert_eq!(has_client_tag_suppression_marker(&request), should_match); diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index b4a188f2a..94e4607cd 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -589,7 +589,7 @@ mod tests { assert_eq!( response.headers()[header::CACHE_CONTROL], - "no-store, private", + "private, no-store", "should stamp diagnostics responses non-storable" ); assert_eq!(response.headers()[header::SET_COOKIE], SET_CONSOLE_COOKIE); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 16a10333d..a288a9c17 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -71,6 +71,7 @@ use crate::platform::{ }; use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::response_privacy::{ + apply_inactive_ad_stack_browser_cache_policy, cache_control_forbids_shared_storage, enforce_synthesized_html_cache_privacy, enforce_terminal_private_cache_privacy, }; use crate::rsc_flight::RscFlightUrlRewriter; @@ -3001,29 +3002,47 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -/// Returns true only when the publisher request should run the full -/// server-side ad stack: auction dispatch plus initial ad-slot injection. +#[derive(Debug, Clone, Copy)] +struct ServerSideAdStackConfig { + /// Dedicated `[creative_opportunities].enabled` switch. + ad_templates_enabled: bool, + /// Global `[auction].enabled` gate used by publisher/page-bids flows. + auction_enabled: bool, +} + +/// Returns whether request-scoped signals permit an ad-eligible navigation. +fn is_server_side_ad_eligible_navigation( + is_get: bool, + is_navigation: bool, + is_prefetch: bool, + is_bot: bool, + consent_allows_auction: bool, +) -> bool { + is_get && is_navigation && !is_prefetch && !is_bot && consent_allows_auction +} + +/// Returns true only when the publisher should inject and run server-side ad templates. /// -/// `auction_enabled` is the global `[auction].enabled` kill switch — when -/// false, no automatic server-side auction or ad-slot injection runs. -pub(crate) fn should_run_server_side_ad_stack( +/// This includes auction dispatch plus initial ad-slot injection. +fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, is_prefetch: bool, is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, - auction_enabled: bool, + config: ServerSideAdStackConfig, ) -> bool { - is_get - && is_navigation - && !is_prefetch - && !is_bot + is_server_side_ad_eligible_navigation( + is_get, + is_navigation, + is_prefetch, + is_bot, + consent_allows_auction, + ) && config.ad_templates_enabled && has_matched_slots - && consent_allows_auction - && auction_enabled + && config.auction_enabled } - /// Write winning bids from an auction result into the shared `ad_bids_state` lock. /// Build the request origin (`scheme://host`, where `host` includes any port) /// used to emit absolute first-party URLs in inline creatives. Returns an empty @@ -4114,7 +4133,11 @@ pub async fn handle_publisher_request( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - let matched_slots = if is_get { + let ad_templates_enabled = settings + .creative_opportunities + .as_ref() + .is_some_and(|co_config| co_config.enabled); + let matched_slots = if is_get && ad_templates_enabled { settings .creative_opportunities .as_ref() @@ -4137,7 +4160,10 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, - auction.orchestrator.is_enabled(), + ServerSideAdStackConfig { + ad_templates_enabled, + auction_enabled: auction.orchestrator.is_enabled(), + }, ); let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with @@ -4604,7 +4630,7 @@ pub async fn handle_publisher_request( let gate_content_type = response .headers() .get(header::CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) + .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_string(); let mut template_cache_key = template_cache_reservation.and_then(|reservation| { @@ -4703,33 +4729,38 @@ pub async fn handle_publisher_request( let origin_content_type = response .headers() .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default() - .to_string(); + .and_then(|h| h.to_str().ok()) + .unwrap_or_default(); // `template_cache_key` is `Some` only for a response the gate authorized, which is // exactly a response that will be assembled. Those must be private regardless of // `should_run_ad_stack`: a bot, prefetch, kill-switched or consent-denied request can // assemble an empty-bids document and would otherwise keep the origin's public // caching directives, letting a downstream cache serve it to a later eligible reader. let assembled_response_must_be_private = template_cache_key.is_some(); - if (should_run_ad_stack || assembled_response_must_be_private) - && is_html_content_type(&origin_content_type) - { - enforce_synthesized_html_cache_privacy(&mut response); + let is_not_modified = response.status() == StatusCode::NOT_MODIFIED; + if is_html_content_type(origin_content_type) || is_not_modified { + if should_run_ad_stack || assembled_response_must_be_private { + enforce_synthesized_html_cache_privacy(&mut response); + } else if is_server_side_ad_eligible_navigation( + is_get, + is_navigation, + is_prefetch, + is_bot, + consent_allows_auction, + ) && (response.status() == StatusCode::OK || is_not_modified) + { + // Issue #1007 caps browser caching for structurally inactive + // server-side ad templates. The cap also applies to 304 responses + // so revalidation cannot restore the origin freshness policy. + // Request-scoped skips retain the origin policy because the same URL + // can otherwise render templates. + if !cache_control_forbids_shared_storage(response.headers()) { + apply_inactive_ad_stack_browser_cache_policy(&mut response); + } + } } - apply_datadome_client_tag_cache_privacy( - &mut response, - &request_method, - suppress_datadome_client_side_tag, - &origin_content_type, - ); - apply_publisher_asset_cache_policy( - settings, - &request_path, - &request_method, - edge_header, - &mut response, - )?; + + crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); let content_type = response .headers() @@ -4738,6 +4769,13 @@ pub async fn handle_publisher_request( .unwrap_or_default() .to_string(); + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &content_type, + ); + let status = response.status(); let content_encoding = response @@ -4752,6 +4790,14 @@ pub async fn handle_publisher_request( set_response_compression(&mut response, reader_compression); } + apply_publisher_asset_cache_policy( + settings, + &request_path, + &request_method, + edge_header, + &mut response, + )?; + match route { ResponseRoute::PassThrough => { log::debug!( @@ -5784,10 +5830,6 @@ fn surrogate_control_freshness( .map_or((directive, None), |(name, value)| (name, Some(value))); let name = name.trim().to_ascii_lowercase(); match name.as_str() { - // Deliberately not `cache_policy::cache_control_headers_are_private_or_no_store`: - // this gate additionally treats `no-cache` as non-shareable, because "revalidate - // before reuse" is correct for an HTTP cache and too permissive for a spike-owned - // one. Consolidating the two would loosen this gate rather than tidy it. "private" | "no-store" | "no-cache" => { return Err(TemplateCacheBypassReason::OriginNotShareable); } @@ -5967,7 +6009,7 @@ fn template_cache_ttl( // Core Cache has no HTTP semantics. Fastly's documented Surrogate-Control subset // is parsed by `origin_shared_ttl`; every other vendor-specific policy remains a // bypass rather than guessing that unrelated CDNs share its grammar or precedence. - if crate::cache_policy::EDGE_CACHE_HEADER_NAMES + if crate::response_privacy::CDN_CACHE_HEADERS .iter() .filter(|name| **name != "surrogate-control") .any(|name| response_headers.contains_key(*name)) @@ -6224,6 +6266,30 @@ fn page_bids_unknown_format() -> Response { /// The SPA hook sends `location.pathname`, but the parameter is /// client-controlled: strip any query string or fragment and force a leading /// `/` so slot `page_patterns` always match against a canonical path shape. +/// How the page-bids endpoint serializes its answer. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum PageBidsFormat { + /// `application/json`. What the SPA navigation hook consumes. + #[default] + Json, +} + +impl PageBidsFormat { + /// Parse the `format` query parameter. + /// + /// # Errors + /// + /// Returns the offending value if it names no known format. Unknown values are + /// rejected rather than defaulting so callers cannot silently negotiate a response + /// representation the endpoint no longer supports. + fn parse(raw: Option<&str>) -> Result { + match raw { + None | Some("json") => Ok(Self::Json), + Some(other) => Err(other.to_string()), + } + } +} + fn normalize_page_bids_path(raw: &str) -> String { let path = raw.split(['?', '#']).next().unwrap_or(""); if path.starts_with('/') { @@ -6333,21 +6399,28 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let format = req.uri().query().and_then(|query| { - url::form_urlencoded::parse(query.as_bytes()) - .find(|(key, _)| key == "format") - .map(|(_, value)| value.into_owned()) - }); - if !matches!(format.as_deref(), None | Some("json")) { - log::warn!( - "page-bids: rejecting unknown format `{}`", - format.as_deref().unwrap_or_default() - ); - return Ok(page_bids_unknown_format()); - } - - let matched_slots = match_renderable_slots(auction.slots, co_config, &path_param); + let format = match PageBidsFormat::parse( + req.uri() + .query() + .and_then(|query| { + url::form_urlencoded::parse(query.as_bytes()) + .find(|(k, _)| k == "format") + .map(|(_, v)| v.into_owned()) + }) + .as_deref(), + ) { + Ok(format) => format, + Err(unknown) => { + log::warn!("page-bids: rejecting unknown format `{unknown}`"); + return Ok(page_bids_unknown_format()); + } + }; + let matched_slots = if co_config.enabled { + match_renderable_slots(auction.slots, co_config, &path_param) + } else { + Vec::new() + }; let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); let consent_context = ec_context.consent(); @@ -6538,6 +6611,7 @@ pub async fn handle_page_bids( Vec::new() }; + debug_assert_eq!(format, PageBidsFormat::Json); let body = serde_json::json!({ "slots": slots_json, "bids": bid_map, @@ -7679,7 +7753,7 @@ mod tests { registry: None, }, req, - EdgeCacheHeader::SurrogateControl, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request") @@ -8047,7 +8121,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!( response @@ -9937,7 +10011,7 @@ mod tests { warm.headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "reusing template cache must not make the assembled response browser-cacheable" ); let warm = String::from_utf8(body_of(warm).await) @@ -10132,7 +10206,7 @@ mod tests { ); assert_eq!( header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private"), + Some("private, no-store"), "an assembled response is per-user even when its template is not" ); @@ -10170,7 +10244,7 @@ mod tests { ); assert_eq!( header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private") + Some("private, no-store") ); } @@ -10679,7 +10753,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!(response.headers().contains_key(header::SET_COOKIE)); let document = String::from_utf8(body_of(response).await) @@ -10930,7 +11004,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "an assembled response must be private whatever the ad stack decided" ); } @@ -11470,7 +11544,7 @@ mod tests { #[test] fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { + for name in crate::response_privacy::CDN_CACHE_HEADERS { let mut split = shareable(); split.insert( header::HeaderName::from_static(name), @@ -11495,7 +11569,7 @@ mod tests { #[test] fn unsupported_vendor_freshness_does_not_authorize_template_cache() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES + for name in crate::response_privacy::CDN_CACHE_HEADERS .iter() .filter(|name| **name != "surrogate-control") { @@ -12262,6 +12336,7 @@ mod tests { // Some(..)` here would silently no-op and make the inline assertion // below vacuous. settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: Some(500), price_granularity: Default::default(), @@ -12339,252 +12414,70 @@ mod tests { } } - #[tokio::test] - async fn publisher_asset_cache_policy_applies_to_non_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} - - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { response, .. } = response else { - panic!("should pass through non-HTML asset response"); + mod ssat_cache_policy_tests { + use super::*; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + use crate::platform::{ + ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, }; + use crate::test_support::tests::crate_test_settings_str; - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "matched publisher asset should receive immutable browser policy" - ); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "matched publisher asset should receive Fastly edge policy" - ); - } + const ORIGIN_ETAG: &str = "\"origin-tag\""; + const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; + const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; - #[tokio::test] - async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { - let settings = Settings::from_toml(&format!( - r#"{} + struct DispatchingTestProvider; - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + struct RangeAwareHttpClient { + stub: StubHttpClient, + } - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { mut response, .. } = response else { - panic!("should pass through non-HTML asset response"); - }; - crate::response_privacy::apply_response_headers_with_cache_privacy( - &settings, - &mut response, - ); + impl RangeAwareHttpClient { + fn new() -> Self { + Self { + stub: StubHttpClient::new(), + } + } + } - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=0"), - "publisher asset with Set-Cookie must become private after finalization" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "publisher asset with Set-Cookie must not retain a shared-cache header" - ); - } + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RangeAwareHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + if request.request.headers().contains_key(header::RANGE) { + self.stub.push_response_with_headers( + 206, + b"partial".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-range", "bytes 0-18/39"), + ], + ); + } else { + self.stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + } + self.stub.send(request).await + } - #[tokio::test] - async fn publisher_asset_cache_policy_skips_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} - - [[cache.asset_rules]] - id = "broad-publisher-path" - enabled = true - path_glob = "/news/*.html" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - fingerprint_style = "hex" - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"news".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/news/story.0123abcd.html") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = run_publisher_proxy(&settings, &services, request).await; - let response = match response { - PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { - response - } - PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { - panic!("should classify HTML response for processing") - } - }; - - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=60"), - "asset policy must not cache publisher HTML" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "HTML response must not receive a shared-cache header" - ); - } - - mod ssat_cache_policy_tests { - use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; - use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; - use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; - use crate::platform::{ - ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, - }; - use crate::test_support::tests::crate_test_settings_str; - - const ORIGIN_ETAG: &str = "\"origin-tag\""; - const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; - const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; - const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; - - struct DispatchingTestProvider; - - struct RangeAwareHttpClient { - stub: StubHttpClient, - } - - impl RangeAwareHttpClient { - fn new() -> Self { - Self { - stub: StubHttpClient::new(), - } - } - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RangeAwareHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - if request.request.headers().contains_key(header::RANGE) { - self.stub.push_response_with_headers( - 206, - b"partial".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("content-range", "bytes 0-18/39"), - ], - ); - } else { - self.stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - } - self.stub.send(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.stub.send_async(request).await - } + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.stub.send_async(request).await + } async fn select( &self, @@ -12675,6 +12568,28 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + } + + fn settings_with_disabled_auction() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = false\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled auction") + } + + fn settings_without_creative_opportunities() -> Settings { + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without creative opportunities") + } fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -12734,12 +12649,27 @@ mod tests { } fn queue_cacheable_html_response(stub: &StubHttpClient) { + queue_html_response_with_cache_control(stub, "public, max-age=300"); + } + + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { + queue_html_response_with_status_and_cache_control(stub, 200, cache_control); + } + + fn queue_html_response_with_status_and_cache_control( + stub: &StubHttpClient, + status: u16, + cache_control: &'static str, + ) { stub.push_response_with_headers( - 200, + status, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", cache_control), ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), @@ -12750,14 +12680,40 @@ mod tests { ); } + fn non_regulated_consent() -> crate::consent::ConsentContext { + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + } + } + async fn run_with_slots( settings: &Settings, services: &RuntimeServices, slots: &[CreativeOpportunitySlot], req: Request, + ) -> PublisherResponse { + run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) + .await + } + + async fn run_with_slots_and_consent( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, ) -> PublisherResponse { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - run_with_orchestrator(settings, services, &orchestrator, slots, req).await + run_with_orchestrator_and_consent( + settings, + services, + &orchestrator, + slots, + req, + consent, + ) + .await } async fn run_with_orchestrator( @@ -12767,10 +12723,25 @@ mod tests { slots: &[CreativeOpportunitySlot], req: Request, ) -> PublisherResponse { - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; + run_with_orchestrator_and_consent( + settings, + services, + orchestrator, + slots, + req, + non_regulated_consent(), + ) + .await + } + + async fn run_with_orchestrator_and_consent( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, + ) -> PublisherResponse { let mut ec_context = EcContext::new_for_test(None, consent); handle_publisher_request( @@ -12847,7 +12818,7 @@ mod tests { .headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "eligible HTML response should be private and non-storable" ); for header_name in [ @@ -12954,7 +12925,98 @@ mod tests { ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), + (header::CACHE_CONTROL, "private, max-age=60"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "publisher response without matched slots should preserve {header_name}" + ); + } + } + + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let mut settings = settings_with_disabled_ad_templates(); + settings.proxy.allowed_domains = + vec!["*.example".to_string(), "*.example.com".to_string()]; + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let response = buffer_publisher_response_async( + response, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should buffer disabled-template response"); + let (response_head, body) = response.into_parts(); + let body = String::from_utf8( + body.into_bytes() + .expect("should return an in-memory publisher body") + .to_vec(), + ) + .expect("should return UTF-8 publisher HTML"); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabled server-side ad templates should not bypass the origin cache" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "disabled server-side ad templates should use the private browser cache policy" + ); + assert!( + !body.contains(".adSlots=JSON.parse"), + "disabled server-side ad templates should not inject ad-slot state" + ); + for (header_name, expected) in [ (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -12977,10 +13039,321 @@ mod tests { assert_eq!( response_head .headers - .get(&header_name) + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); + } + } + + #[tokio::test] + async fn disabled_auction_uses_private_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_auction(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "disabled auction should use the private browser cache policy" + ); + } + + #[tokio::test] + async fn navigation_without_matched_slots_replaces_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "inactive server-side ad templates should replace origin {cache_control} policy" + ); + } + } + + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve private origin {cache_control} policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "inactive server-side ad templates should preserve {header_name}" + ); + } + } + } + + #[tokio::test] + async fn request_scoped_ad_stack_suppression_preserves_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let slots = [article_slot()]; + let mut bot_request = conditional_navigation_request(); + bot_request.headers_mut().insert( + "user-agent", + HeaderValue::from_static("Mozilla/5.0 (compatible; Googlebot/2.1)"), + ); + let mut prefetch_request = conditional_navigation_request(); + prefetch_request + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for (skip_reason, request, consent) in [ + ("bot", bot_request, non_regulated_consent()), + ("prefetch", prefetch_request, non_regulated_consent()), + ( + "consent denied", + conditional_navigation_request(), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots_and_consent(&settings, &services, &slots, request, consent) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "{skip_reason} should retain the origin cache policy" + ); + } + } + + #[tokio::test] + async fn absent_creative_opportunities_use_short_browser_cache_policy() { + // Arrange + let settings = settings_without_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "absent creative opportunities should use the private inactive-stack cache policy" + ); + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_non_ok_response_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for status in [206, 404, 500, 503] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve origin policy on {status}" + ); + } + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_gpt_diagnostics_cache_privacy() { + // Arrange + let mut settings = settings_with_disabled_ad_templates(); + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) + .expect("should enable GPT diagnostics"); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article?ts_console=1") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build GPT diagnostics request"); + + // Act + let response = run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "active GPT diagnostics should retain cache privacy when server-side ad templates are inactive" + ); + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_non_get_and_non_document_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for request in [ + HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build non-GET document request"), + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "empty") + .body(EdgeBody::empty()) + .expect("should build non-document request"), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some(expected), - "publisher response without matched slots should preserve {header_name}" + Some("no-cache"), + "inactive server-side ad templates should preserve non-document request policy" ); } } @@ -13044,7 +13417,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "eligible origin 304 should return an explicitly non-storable response" ); assert!( @@ -13094,7 +13467,7 @@ mod tests { } #[tokio::test] - async fn noneligible_origin_304_preserves_conditional_response_metadata() { + async fn inactive_ad_stack_304_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); @@ -13129,10 +13502,17 @@ mod tests { assert_eq!( response.status(), StatusCode::NOT_MODIFIED, - "noneligible origin 304 should preserve its status" + "inactive server-side ad templates should preserve the 304 status" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "inactive server-side ad templates should apply the browser policy on revalidation" ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -13150,7 +13530,7 @@ mod tests { .get(&header_name) .and_then(|value| value.to_str().ok()), Some(expected), - "noneligible origin 304 should preserve {header_name}" + "inactive server-side ad templates should preserve {header_name} on revalidation" ); } assert_eq!( @@ -13173,6 +13553,222 @@ mod tests { "noneligible publisher request should preserve If-Modified-Since" ); } + + #[tokio::test] + async fn inactive_ad_stack_304_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![("cache-control", cache_control), ("etag", ORIGIN_ETAG)], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve origin {cache_control} policy on revalidation" + ); + } + } + } + + #[tokio::test] + async fn publisher_asset_cache_policy_applies_to_non_html_response() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, s-maxage=31536000, immutable"), + "matched publisher asset should receive immutable browser and edge policy" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + None, + "S-maxage fallback should keep the edge TTL in Cache-Control" + ); + } + + #[tokio::test] + async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { mut response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + crate::response_privacy::apply_response_headers_with_cache_privacy( + &settings, + &mut response, + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "publisher asset with Set-Cookie must become private after finalization" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "publisher asset with Set-Cookie must not retain a shared-cache header" + ); + } + + #[tokio::test] + async fn publisher_asset_cache_policy_skips_html_response() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "broad-publisher-path" + enabled = true + path_glob = "/news/*.html" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + fingerprint_style = "hex" + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"news".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/news/story.0123abcd.html") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let response = match response { + PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { + response + } + PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { + panic!("should classify HTML response for processing") + } + }; + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=60"), + "asset policy must not apply shared asset caching to publisher HTML" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "HTML response must not receive a shared-cache header" + ); } #[tokio::test] @@ -13608,7 +14204,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "suppressed HTML should be private and non-storable" ); assert!( @@ -13653,7 +14249,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "suppressed HTML should use the exact synthesized-HTML policy" ); } @@ -13825,41 +14421,70 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + let enabled_config = ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: true, + }; assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" + should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), + "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), "requests without required consent should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: false, + }, + ), "disabled [auction].enabled kill switch should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: false, + auction_enabled: true, + }, + ), + "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" + ); } - #[tokio::test] async fn body_close_hold_loop_processes_close_tail_before_reading_post_body_chunks() { let settings = create_test_settings(); @@ -16799,6 +17424,7 @@ mod tests { fn make_config() -> CreativeOpportunitiesConfig { CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index c053c375e..c1a70a01a 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -9,33 +9,148 @@ //! cache such as Cloudflare would otherwise serve an operator/origin //! `Cache-Control: public` on a cookie-bearing response as-is. -use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::cache_policy::{ - CacheControlPolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, - is_edge_cache_header_name, remove_edge_cache_headers, + cache_control_headers_are_private_or_no_store, is_edge_cache_header_name, + remove_edge_cache_headers, }; use crate::settings::Settings; -/// Marks a response whose `no-store, private` policy belongs to Trusted Server. +/// Marks a response whose `private, no-store` policy belongs to Trusted Server. /// /// Platform terminal hooks use this out-of-band marker to re-enforce the policy after /// late integrations run without rewriting unrelated origin-private responses. #[derive(Clone, Copy, Debug)] pub struct TerminalPrivateResponse; -fn cache_control_is_private_or_no_store(response: &Response) -> bool { - cache_control_headers_are_private_or_no_store(response.headers()) +/// CDN-targeted cache headers stripped from every cookie-bearing response. +/// +/// A single source of truth so the adapter copies of the privacy downgrade +/// cannot drift apart. +pub const CDN_CACHE_HEADERS: &[&str] = &[ + "surrogate-control", + "fastly-surrogate-control", + "cdn-cache-control", + "cloudflare-cdn-cache-control", +]; + +const INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL: &str = "private, max-age=60"; + +#[derive(Debug, Clone, Copy)] +struct GeneratedInactiveAdStackBrowserCachePolicy; + +fn cache_control_segment_has_directive(segment: &[u8], target: &[u8]) -> bool { + let name_end = segment + .iter() + .position(|byte| *byte == b'=') + .unwrap_or(segment.len()); + segment[..name_end] + .trim_ascii() + .eq_ignore_ascii_case(target) +} + +fn cache_control_value_has_directive(value: &[u8], target: &[u8]) -> bool { + let mut segment_start = 0; + let mut in_quotes = false; + let mut escaped = false; + + for (index, byte) in value.iter().copied().enumerate() { + if in_quotes { + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + in_quotes = false; + } + } else if byte == b'"' { + in_quotes = true; + } else if byte == b',' { + if cache_control_segment_has_directive(&value[segment_start..index], target) { + return true; + } + segment_start = index + 1; + } + } + + cache_control_segment_has_directive(&value[segment_start..], target) +} + +fn cache_control_has_directive(headers: &HeaderMap, target: &str) -> bool { + headers + .get_all(header::CACHE_CONTROL) + .iter() + .any(|value| cache_control_value_has_directive(value.as_bytes(), target.as_bytes())) +} + +/// Returns whether `Cache-Control` prohibits storage by shared caches. +pub(crate) fn cache_control_forbids_shared_storage(headers: &HeaderMap) -> bool { + cache_control_has_directive(headers, "private") + || cache_control_has_directive(headers, "no-store") +} + +fn has_generated_inactive_ad_stack_browser_cache_policy(response: &Response) -> bool { + response + .extensions() + .get::() + .is_some() + && response + .headers() + .get(header::CACHE_CONTROL) + .is_some_and(|value| { + value + .as_bytes() + .eq_ignore_ascii_case(INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL.as_bytes()) + }) +} + +/// Applies the browser-only cache policy for structurally inactive ad templates. +pub fn apply_inactive_ad_stack_browser_cache_policy(response: &mut Response) { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static(INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL), + ); + response + .extensions_mut() + .insert(GeneratedInactiveAdStackBrowserCachePolicy); +} + +fn strip_cdn_cache_headers(response: &mut Response) { + for name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*name); + } +} + +/// Whether `Cache-Control` already forbids shared caching. +/// +/// Extracted because both arms of the cookie-privacy net below need it. +/// +/// `publisher::template_cache_bypass_reason` deliberately does **not** call this and keeps its own +/// copy: it additionally treats `no-cache` as non-shareable, because "revalidate before +/// reuse" is correct for an HTTP cache and too permissive for a spike-owned one. The +/// duplicate is the stricter of the two, so consolidating them would loosen the shared +/// template-cache gate rather than tidy it. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub fn is_private_or_no_store(headers: &HeaderMap) -> bool { + cache_control_headers_are_private_or_no_store(headers) } /// Reassert the terminal privacy invariant for a synthesized per-reader response. /// /// Call this after every configurable response mutation. It deliberately overwrites -/// `Cache-Control` and strips validators, expiry metadata, and runtime edge-cache +/// `Cache-Control` and strips validators, expiry metadata, and CDN-specific cache /// directives so a later integration cannot turn an assembled document into C3. pub fn enforce_private_no_store(response: &mut Response) { - CacheControlPolicy::NoStorePrivate - .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); for name in [ header::ETAG.as_str(), header::LAST_MODIFIED.as_str(), @@ -44,6 +159,7 @@ pub fn enforce_private_no_store(response: &mut Response) { ] { response.headers_mut().remove(name); } + strip_cdn_cache_headers(response); } /// Marks a Trusted Server response as terminal-private and applies its cache policy. @@ -55,8 +171,8 @@ pub(crate) fn enforce_terminal_private_cache_privacy(response: &mut Response) { /// Forces synthesized HTML to be private and non-storable. /// /// Use this exact policy whenever Trusted Server changes an origin HTML -/// representation with request-specific content: force `no-store, private`, -/// remove origin validators, and remove all runtime edge-cache directives. +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { enforce_terminal_private_cache_privacy(response); } @@ -67,7 +183,7 @@ pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { /// `no-store` directive cannot coexist with an independently authoritative edge /// cache header. pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { - if cache_control_is_private_or_no_store(response) { + if is_private_or_no_store(response.headers()) { remove_edge_cache_headers(response.headers_mut()); } } @@ -78,27 +194,33 @@ pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { /// must never be shared-cached, or a shared cache could replay one user's /// `Set-Cookie` to others. /// -/// Idempotent: a response already marked `private`/`no-store` keeps its stricter -/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a -/// `no-store` cookie response can never retain shared cacheability. +/// Origin `private`/`no-store` policies remain unchanged, but the generated +/// inactive-stack browser policy is downgraded to `private, max-age=0`. CDN +/// cache headers are always stripped so a cookie response cannot retain shared +/// cacheability. pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Edge-cache headers must come off every cookie-bearing response, even one - // already carrying a stricter `no-store`/`private` directive — they are + // Shared-cache control headers must come off every cookie-bearing response, even + // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - remove_edge_cache_headers(response.headers_mut()); - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = cache_control_is_private_or_no_store(response); - if !already_uncacheable { + strip_cdn_cache_headers(response); + // Cookie privacy takes precedence over the generated inactive-stack browser + // policy, while unrelated origin private/no-store policies remain unchanged. + let already_forbids_shared_storage = cache_control_forbids_shared_storage(response.headers()); + let has_generated_inactive_browser_policy = + has_generated_inactive_ad_stack_browser_cache_policy(response); + if !already_forbids_shared_storage || has_generated_inactive_browser_policy { response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static("private, max-age=0"), ); } + response + .extensions_mut() + .remove::(); } /// Applies operator-configured `settings.response_headers` with cookie-privacy @@ -107,10 +229,10 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { /// First downgrades cookie-bearing responses via /// [`enforce_set_cookie_cache_privacy`], then applies operator headers — but on /// an uncacheable (`private`/`no-store`) response the cache-controlling headers -/// (`Cache-Control` and runtime edge-cache headers) are skipped so operators +/// (`Cache-Control` and the surrogate cache headers) are skipped so operators /// cannot re-enable shared caching for per-user payloads. After the operator /// headers are applied the cookie-privacy downgrade runs once more, so a -/// configured `Set-Cookie` combined with public edge-cache headers cannot +/// configured `Set-Cookie` combined with public/surrogate cache headers cannot /// produce a shared-cacheable cookie-bearing response. /// /// Invalid header names/values are logged and skipped rather than panicking, so @@ -118,7 +240,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = cache_control_is_private_or_no_store(response); + let response_is_uncacheable = cache_control_forbids_shared_storage(response.headers()); enforce_uncacheable_cache_privacy(response); for (key, value) in &settings.response_headers { @@ -148,9 +270,9 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: enforce_uncacheable_cache_privacy(response); // Operator headers can themselves introduce Set-Cookie (alongside public - // edge-cache headers) onto a previously cookieless response, which the - // pre-apply pass could not see. Re-run the downgrade so the final response - // can never pair Set-Cookie with shared cacheability. + // or surrogate cache headers) onto a previously cookieless response, which + // the pre-apply pass could not see. Re-run the downgrade so the final + // response can never pair Set-Cookie with shared cacheability. enforce_set_cookie_cache_privacy(response); } @@ -158,9 +280,8 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: mod tests { use super::*; - use edgezero_core::http::response_builder; - use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; + use edgezero_core::http::response_builder; fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( @@ -189,7 +310,7 @@ mod tests { } #[test] - fn synthesized_html_is_forced_no_store_without_validators_or_edge_headers() { + fn synthesized_html_is_forced_no_store_without_validators_or_cdn_headers() { let mut response = response_builder() .header(header::CACHE_CONTROL, "private, max-age=600") .header(header::ETAG, "\"origin\"") @@ -205,7 +326,7 @@ mod tests { assert_eq!( response.headers()[header::CACHE_CONTROL], - "no-store, private", + "private, no-store", "synthesized HTML should always be non-storable" ); assert!( @@ -217,7 +338,7 @@ mod tests { ); for header_name in [header::ETAG.as_str(), header::LAST_MODIFIED.as_str()] .into_iter() - .chain(EDGE_CACHE_HEADER_NAMES.iter().copied()) + .chain(CDN_CACHE_HEADERS.iter().copied()) { assert!( !response.headers().contains_key(header_name), @@ -226,12 +347,90 @@ mod tests { } } + #[test] + fn identifies_cache_control_that_forbids_shared_storage() { + for (cache_control, expected) in [ + ("public, max-age=60", false), + ("no-cache", false), + ("public, x=notprivate", false), + ("public, x=\"private, no-store\"", false), + (r#"public, x="private\", no-store", max-age=60"#, false), + (r#"public, x="private\\", no-store"#, true), + ("Private=\"set-cookie\", max-age=60", true), + ("No-Store", true), + ] { + let response = response_builder() + .header(header::CACHE_CONTROL, cache_control) + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + assert_eq!( + cache_control_forbids_shared_storage(response.headers()), + expected, + "should classify {cache_control} shared-storage policy" + ); + } + } + + #[test] + fn downgrades_inactive_browser_cache_policy_on_cookie_response() { + let mut response = response_builder() + .header("surrogate-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + apply_inactive_ad_stack_browser_cache_policy(&mut response); + response + .headers_mut() + .insert(header::SET_COOKIE, HeaderValue::from_static("id=abc")); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "inactive browser cache policy should yield to cookie privacy" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "cookie privacy should strip CDN cache headers" + ); + } + + #[test] + fn preserves_identical_origin_private_policy_on_cookie_response() { + let mut response = response_builder() + .header(header::SET_COOKIE, "id=abc") + .header(header::CACHE_CONTROL, "private, max-age=60") + .header("surrogate-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "origin private policy should remain unchanged" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "cookie privacy should still strip CDN cache headers" + ); + } + #[test] fn downgrades_public_cache_control_on_cookie_response() { let settings = settings_with_response_headers(&[("cache-control", "public, max-age=600")]); let mut response = response_builder() .header(header::SET_COOKIE, "id=abc") .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) @@ -247,14 +446,12 @@ mod tests { Some("private, max-age=0"), "operator public Cache-Control must not override cookie privacy downgrade" ); - assert!( - !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), - "edge cache headers must be stripped on cookie responses" - ); + for header_name in CDN_CACHE_HEADERS { + assert!( + !response.headers().contains_key(*header_name), + "CDN cache header {header_name} must be stripped on cookie responses" + ); + } } #[test] @@ -265,8 +462,9 @@ mod tests { ("set-cookie", "operator=abc"), ("cache-control", "public, max-age=600"), ("surrogate-control", "max-age=600"), - ("cdn-cache-control", "max-age=600"), - ("cloudflare-cdn-cache-control", "max-age=600"), + ("fastly-surrogate-control", "max-age=600"), + ("cdn-cache-control", "public, max-age=600"), + ("cloudflare-cdn-cache-control", "public, max-age=600"), ]); let mut response = response_builder() .body(edgezero_core::body::Body::empty()) @@ -282,14 +480,12 @@ mod tests { Some("private, max-age=0"), "operator Set-Cookie plus public Cache-Control must be re-downgraded to private" ); - assert!( - !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), - "edge cache headers must be stripped when operator headers add Set-Cookie" - ); + for header_name in CDN_CACHE_HEADERS { + assert!( + !response.headers().contains_key(*header_name), + "CDN cache header {header_name} must be stripped when operator headers add Set-Cookie" + ); + } assert!( response.headers().contains_key(header::SET_COOKIE), "the operator Set-Cookie itself should still be applied" @@ -315,7 +511,7 @@ mod tests { response .headers() .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()), + .and_then(|value| value.to_str().ok()), Some("private, max-age=0"), "pseudo-directives must not prevent the cookie privacy downgrade" ); @@ -375,7 +571,7 @@ mod tests { "private, no-store", "operator cache headers must not weaken an existing private response" ); - for header_name in EDGE_CACHE_HEADER_NAMES { + for header_name in CDN_CACHE_HEADERS { assert!( !response.headers().contains_key(*header_name), "operator headers must not restore shared caching through {header_name}" @@ -383,6 +579,80 @@ mod tests { } } + #[test] + fn terminal_private_stamp_removes_every_cache_and_validator_header() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "public, s-maxage=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") + .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") + .header(header::AGE, "30") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "public, max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_private_no_store(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + "surrogate-control", + "cdn-cache-control", + ] { + assert!( + !response.headers().contains_key(name), + "terminal private stamp must strip {name}" + ); + } + } + + #[test] + fn applies_operator_headers_on_cookieless_response() { + let settings = settings_with_response_headers(&[("x-operator", "value")]); + let mut response = response_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("value"), + "operator headers should still apply to cacheable responses" + ); + } + + #[test] + fn uncacheable_response_rejects_operator_cdn_cache_headers() { + let settings = settings_with_response_headers(&[ + ("cdn-cache-control", "public, max-age=600"), + ("cloudflare-cdn-cache-control", "public, max-age=600"), + ]); + let mut response = response_builder() + .header(header::CACHE_CONTROL, "private, no-store") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + for header_name in ["cdn-cache-control", "cloudflare-cdn-cache-control"] { + assert!( + !response.headers().contains_key(header_name), + "operator headers must not restore shared caching through {header_name}" + ); + } + } + #[test] fn strips_edge_headers_from_uncacheable_cookieless_response() { let settings = settings_with_response_headers(&[ @@ -436,58 +706,4 @@ mod tests { "final guard should remove every edge-cache header" ); } - - #[test] - fn terminal_private_stamp_removes_every_cache_and_validator_header() { - let mut response = response_builder() - .header(header::CACHE_CONTROL, "public, s-maxage=600") - .header(header::ETAG, "\"origin\"") - .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") - .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") - .header(header::AGE, "30") - .header("surrogate-control", "max-age=600") - .header("cdn-cache-control", "public, max-age=600") - .body(edgezero_core::body::Body::empty()) - .expect("should build response"); - - enforce_private_no_store(&mut response); - - assert_eq!( - response.headers()[header::CACHE_CONTROL], - "no-store, private" - ); - for name in [ - header::ETAG.as_str(), - header::LAST_MODIFIED.as_str(), - header::EXPIRES.as_str(), - header::AGE.as_str(), - ] - .into_iter() - .chain(EDGE_CACHE_HEADER_NAMES.iter().copied()) - { - assert!( - !response.headers().contains_key(name), - "terminal private stamp must strip {name}" - ); - } - } - - #[test] - fn applies_operator_headers_on_cookieless_response() { - let settings = settings_with_response_headers(&[("x-operator", "value")]); - let mut response = response_builder() - .body(edgezero_core::body::Body::empty()) - .expect("should build response"); - - apply_response_headers_with_cache_privacy(&settings, &mut response); - - assert_eq!( - response - .headers() - .get("x-operator") - .and_then(|v| v.to_str().ok()), - Some("value"), - "operator headers should still apply to cacheable responses" - ); - } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 1e3444ff0..22e5ec906 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -5,11 +5,13 @@ use glob::{MatchOptions, Pattern}; use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; +use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; use std::time::Duration; +use subtle::ConstantTimeEq as _; use url::Url; use validator::{Validate, ValidationError}; @@ -2596,6 +2598,108 @@ pub struct TesterCookieConfig { pub enabled: bool, } +/// Authenticated forwarding configuration for a trusted client IP header. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = validate_trusted_client_ip))] +pub struct TrustedClientIpConfig { + /// Header containing the client IP address supplied by the trusted edge. + pub ip_header: String, + /// Header containing the shared-secret authentication value. + pub auth_header: String, + /// Shared secret required before accepting the forwarded client IP address. + #[validate(custom(function = validate_redacted_not_empty))] + pub shared_secret: Redacted, +} + +impl TrustedClientIpConfig { + /// Placeholder shared secrets shipped in the example configuration and docs. + pub const SHARED_SECRET_PLACEHOLDERS: &[&str] = &["replace-with-a-random-shared-secret"]; + + /// Minimum accepted `shared_secret` length. + /// + /// Matches `Ec::MIN_PASSPHRASE_LENGTH`. This secret is the only gate on + /// forging the client address that geolocation, EC identity derivation, and + /// bot protection consume, so it is held to the same strength as the EC + /// passphrase. + const MIN_SHARED_SECRET_LENGTH: usize = Ec::MIN_PASSPHRASE_LENGTH; + + /// Returns `true` if `shared_secret` matches a known placeholder value + /// (case-insensitive). + #[must_use] + pub fn is_placeholder_shared_secret(shared_secret: &str) -> bool { + Self::SHARED_SECRET_PLACEHOLDERS + .iter() + .any(|p| p.eq_ignore_ascii_case(shared_secret)) + } + + /// Returns whether `candidate` exactly matches the configured shared secret. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::redacted::Redacted; + /// use trusted_server_core::settings::TrustedClientIpConfig; + /// + /// let config = TrustedClientIpConfig { + /// ip_header: "fastly-client-ip".to_owned(), + /// auth_header: "x-trusted-client-auth".to_owned(), + /// shared_secret: Redacted::new("fictional-shared-secret-0123456789".to_owned()), + /// }; + /// + /// assert!(config.authenticates("fictional-shared-secret-0123456789")); + /// assert!(!config.authenticates("fictional-wrong-secret")); + /// ``` + #[must_use] + pub fn authenticates(&self, candidate: &str) -> bool { + let configured_digest = Sha256::digest(self.shared_secret.expose().as_bytes()); + let candidate_digest = Sha256::digest(candidate.as_bytes()); + + configured_digest.ct_eq(&candidate_digest).into() + } +} + +fn validate_trusted_client_ip(config: &TrustedClientIpConfig) -> Result<(), ValidationError> { + let ip_header = http::HeaderName::from_bytes(config.ip_header.as_bytes()) + .map_err(|_| ValidationError::new("invalid_trusted_client_ip_header"))?; + let auth_header = http::HeaderName::from_bytes(config.auth_header.as_bytes()) + .map_err(|_| ValidationError::new("invalid_trusted_client_ip_auth_header"))?; + + if ip_header == auth_header { + return Err(ValidationError::new("identical_trusted_client_ip_headers")); + } + + for header in [&ip_header, &auth_header] { + if matches!(header.as_str(), "x-ts-tls-protocol" | "x-ts-tls-cipher") { + return Err(ValidationError::new("reserved_trusted_client_ip_header")); + } + } + + if ip_header.as_str() != "fastly-client-ip" && !ip_header.as_str().starts_with("x-") { + return Err(ValidationError::new("unsafe_trusted_client_ip_header")); + } + if !auth_header.as_str().starts_with("x-") { + return Err(ValidationError::new("unsafe_trusted_client_ip_auth_header")); + } + + let shared_secret = config.shared_secret.expose(); + if shared_secret.len() < TrustedClientIpConfig::MIN_SHARED_SECRET_LENGTH { + return Err(ValidationError::new( + "short_trusted_client_ip_shared_secret", + )); + } + if !shared_secret + .bytes() + .all(|byte| matches!(byte, b'!'..=b'~')) + { + return Err(ValidationError::new( + "invalid_trusted_client_ip_shared_secret", + )); + } + + Ok(()) +} + #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] pub struct Settings { @@ -2603,6 +2707,10 @@ pub struct Settings { pub publisher: Publisher, #[serde(default)] pub tester_cookie: TesterCookieConfig, + /// Optional authenticated trusted client IP forwarding configuration. + #[serde(default)] + #[validate(nested)] + pub trusted_client_ip: Option, #[serde(default)] #[validate(nested)] pub ec: Ec, @@ -2789,13 +2897,14 @@ impl Settings { Ok(()) } - /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( &self, ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { self.creative_opportunities .as_ref() + .filter(|co| co.enabled) .map(|co| co.slot.as_slice()) .unwrap_or(&[]) } @@ -2815,6 +2924,13 @@ impl Settings { if Publisher::is_placeholder_proxy_secret(self.publisher.proxy_secret.expose()) { insecure_fields.push("publisher.proxy_secret".to_owned()); } + if let Some(trusted_client_ip) = &self.trusted_client_ip + && TrustedClientIpConfig::is_placeholder_shared_secret( + trusted_client_ip.shared_secret.expose(), + ) + { + insecure_fields.push("trusted_client_ip.shared_secret".to_owned()); + } for partner in &self.ec.partners { if EcPartner::is_placeholder_api_token(partner.api_token.expose()) { insecure_fields.push(format!("ec.partners[{}].api_token", partner.source_domain)); @@ -3353,6 +3469,446 @@ mod tests { use crate::redacted::Redacted; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; + fn trusted_client_ip_toml(ip_header: &str, auth_header: &str, shared_secret: &str) -> String { + format!( + "{}\n[trusted_client_ip]\nip_header = \"{ip_header}\"\nauth_header = \"{auth_header}\"\nshared_secret = \"{shared_secret}\"\n", + crate_test_settings_str() + ) + } + + #[test] + fn trusted_client_ip_is_absent_by_default() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without trusted client IP configuration"); + + assert!( + settings.trusted_client_ip.is_none(), + "should leave trusted client IP configuration disabled by default" + ); + } + + #[test] + fn trusted_client_ip_parses_and_redacts_shared_secret_in_debug_output() { + let settings = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + )) + .expect("should parse valid trusted client IP configuration"); + let config = settings + .trusted_client_ip + .expect("should retain trusted client IP configuration"); + + assert_eq!(config.ip_header, "fastly-client-ip"); + assert_eq!(config.auth_header, "x-trusted-client-auth"); + let debug = format!("{config:?}"); + assert!( + debug.contains("[REDACTED]"), + "should redact trusted client IP shared secret in debug output" + ); + assert!( + !debug.contains("fictional-shared-secret-0123456789"), + "should not expose trusted client IP shared secret in debug output" + ); + } + + #[test] + fn trusted_client_ip_accepts_x_prefixed_ip_header() { + let settings = Settings::from_toml(&trusted_client_ip_toml( + "x-trusted-client-ip", + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + )) + .expect("should accept an x-prefixed trusted client IP header"); + let config = settings + .trusted_client_ip + .expect("should retain trusted client IP configuration"); + + assert_eq!( + config.ip_header, "x-trusted-client-ip", + "should retain the x-prefixed trusted client IP header" + ); + } + + #[test] + fn trusted_client_ip_authentication_requires_an_exact_match() { + let settings = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + "fictional-shared-secret-0123456789", + )) + .expect("should parse valid trusted client IP configuration"); + let config = settings + .trusted_client_ip + .expect("should retain trusted client IP configuration"); + + assert!( + config.authenticates("fictional-shared-secret-0123456789"), + "should authenticate an exact shared secret match" + ); + assert!( + !config.authenticates("fictional-wrong-secret"), + "should reject a different shared secret" + ); + assert!( + !config.authenticates(" fictional-shared-secret-0123456789"), + "should reject a leading-whitespace shared secret" + ); + assert!( + !config.authenticates("fictional-shared-secret-0123456789 "), + "should reject a trailing-whitespace shared secret" + ); + } + + #[test] + fn trusted_client_ip_rejects_identical_header_names() { + for (ip_header, auth_header) in [ + ("x-trusted-client", "x-trusted-client"), + ("X-Trusted-Client", "x-trusted-client"), + ] { + let error = Settings::from_toml(&trusted_client_ip_toml( + ip_header, + auth_header, + "fictional-shared-secret-0123456789", + )) + .expect_err("should reject identical trusted client IP header names"); + + assert!( + format!("{error:?}").contains("identical_trusted_client_ip_headers"), + "should identify duplicate trusted client IP header names" + ); + } + } + + #[test] + fn trusted_client_ip_rejects_unsafe_header_names() { + for (ip_header, auth_header, expected_code) in [ + ( + "host", + "x-trusted-client-auth", + "unsafe_trusted_client_ip_header", + ), + ( + "fastly-client-ip", + "authorization", + "unsafe_trusted_client_ip_auth_header", + ), + ] { + let error = Settings::from_toml(&trusted_client_ip_toml( + ip_header, + auth_header, + "fictional-shared-secret-0123456789", + )) + .expect_err("should reject unsafe trusted client IP header names"); + let message = format!("{error:?}"); + + assert!( + message.contains(expected_code), + "should identify unsafe trusted client IP header names" + ); + assert!( + !message.contains("fictional-shared-secret-0123456789"), + "should not include the shared secret in validation errors" + ); + } + } + + #[test] + fn trusted_client_ip_rejects_reserved_tls_bridge_headers() { + for (ip_header, auth_header) in [ + ("x-ts-tls-protocol", "x-trusted-client-auth"), + ("x-ts-tls-cipher", "x-trusted-client-auth"), + ("fastly-client-ip", "x-ts-tls-protocol"), + ("fastly-client-ip", "x-ts-tls-cipher"), + ] { + let error = Settings::from_toml(&trusted_client_ip_toml( + ip_header, + auth_header, + "fictional-shared-secret-0123456789", + )) + .expect_err("should reject reserved TLS bridge headers"); + + assert!( + format!("{error:?}").contains("reserved_trusted_client_ip_header"), + "should identify reserved TLS bridge headers" + ); + } + } + + #[test] + fn trusted_client_ip_rejects_empty_secret_malformed_names_and_incomplete_sections() { + let empty_secret = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + "", + )); + assert!( + empty_secret.is_err(), + "should reject an empty trusted client IP shared secret" + ); + + for (ip_header, auth_header, expected_code) in [ + ( + "invalid header", + "x-trusted-client-auth", + "invalid_trusted_client_ip_header", + ), + ( + "fastly-client-ip", + "invalid header", + "invalid_trusted_client_ip_auth_header", + ), + ] { + let error = Settings::from_toml(&trusted_client_ip_toml( + ip_header, + auth_header, + "fictional-shared-secret-0123456789", + )) + .expect_err("should reject malformed trusted client IP header names"); + assert!( + format!("{error:?}").contains(expected_code), + "should identify malformed trusted client IP header names" + ); + } + + for section in [ + "[trusted_client_ip]\nauth_header = \"x-trusted-client-auth\"\nshared_secret = \"fictional-shared-secret-0123456789\"", + "[trusted_client_ip]\nip_header = \"fastly-client-ip\"\nshared_secret = \"fictional-shared-secret-0123456789\"", + "[trusted_client_ip]\nip_header = \"fastly-client-ip\"\nauth_header = \"x-trusted-client-auth\"", + "[trusted_client_ip]\nip_header = \"fastly-client-ip\"\nauth_header = \"x-trusted-client-auth\"\nshared_secret = \"fictional-shared-secret-0123456789\"\nunknown_field = true", + ] { + let result = + Settings::from_toml(&format!("{}\n{section}\n", crate_test_settings_str())); + assert!( + result.is_err(), + "should reject incomplete or unknown trusted client IP configuration" + ); + } + } + + #[test] + fn trusted_client_ip_rejects_control_byte_auth_header_without_exposing_secret() { + let mut settings = serde_json::to_value( + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse base settings for JSON validation"), + ) + .expect("should serialize base settings for JSON validation"); + settings["trusted_client_ip"] = json!({ + "ip_header": "fastly-client-ip", + "auth_header": "x-trusted\u{0000}client-auth", + "shared_secret": "fictional-control-byte-secret-0123", + }); + + let error = Settings::from_json_value(settings) + .expect_err("should reject a control byte in the trusted client IP auth header"); + let message = format!("{error:?}"); + + assert!( + message.contains("invalid_trusted_client_ip_auth_header"), + "should identify the malformed trusted client IP auth header" + ); + assert!( + !message.contains("fictional-control-byte-secret-0123"), + "should not expose the trusted client IP shared secret in validation errors" + ); + } + + #[test] + fn trusted_client_ip_rejects_a_31_byte_shared_secret_without_exposing_it() { + let shared_secret = "1234567890123456789012345678901"; + let error = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + shared_secret, + )) + .expect_err("should reject a shared secret below the minimum length"); + let message = format!("{error:?}"); + + assert!( + message.contains("short_trusted_client_ip_shared_secret"), + "should identify the undersized trusted client IP shared secret" + ); + assert!( + !message.contains(shared_secret), + "should not expose the undersized trusted client IP shared secret" + ); + } + + #[test] + fn trusted_client_ip_accepts_an_exactly_32_byte_ascii_graphic_shared_secret() { + let shared_secret = "0123456789abcdef0123456789ABCDEF"; + let settings = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + shared_secret, + )) + .expect("should accept an exactly 32-byte ASCII graphic shared secret"); + let config = settings + .trusted_client_ip + .expect("should retain trusted client IP configuration"); + + assert_eq!( + config.shared_secret.expose(), + shared_secret, + "should retain the accepted shared secret" + ); + } + + #[test] + fn trusted_client_ip_rejects_a_non_ascii_shared_secret_without_exposing_it() { + let shared_secret = "ascii-graphic-secret-0123456789é"; + let error = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + shared_secret, + )) + .expect_err("should reject a non-ASCII shared secret that exceeds 32 bytes"); + let message = format!("{error:?}"); + + assert!( + message.contains("invalid_trusted_client_ip_shared_secret"), + "should identify the non-header-safe trusted client IP shared secret" + ); + assert!( + !message.contains(shared_secret), + "should not expose the non-ASCII trusted client IP shared secret" + ); + } + + #[test] + fn trusted_client_ip_rejects_a_shared_secret_with_an_embedded_space_without_exposing_it() { + let shared_secret = "valid-shared-secret-with space-012345"; + let error = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + shared_secret, + )) + .expect_err("should reject a shared secret containing an ASCII space"); + let message = format!("{error:?}"); + + assert!( + message.contains("invalid_trusted_client_ip_shared_secret"), + "should identify the non-header-safe trusted client IP shared secret" + ); + assert!( + !message.contains(shared_secret), + "should not expose the shared secret containing an ASCII space" + ); + } + + #[test] + fn trusted_client_ip_rejects_a_shared_secret_with_an_embedded_tab_without_exposing_it() { + let shared_secret = "valid-shared-secret-with\t-tab-012345"; + let mut settings = serde_json::to_value( + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse base settings for JSON validation"), + ) + .expect("should serialize base settings for JSON validation"); + settings["trusted_client_ip"] = json!({ + "ip_header": "fastly-client-ip", + "auth_header": "x-trusted-client-auth", + "shared_secret": shared_secret, + }); + + let error = Settings::from_json_value(settings) + .expect_err("should reject a shared secret containing a horizontal tab"); + let message = format!("{error:?}"); + + assert!( + message.contains("invalid_trusted_client_ip_shared_secret"), + "should identify the non-header-safe trusted client IP shared secret" + ); + assert!( + !message.contains(shared_secret), + "should not expose the shared secret containing a horizontal tab" + ); + } + + #[test] + fn trusted_client_ip_rejects_a_shared_secret_with_del_without_exposing_it() { + let shared_secret = "valid-shared-secret-with\u{007f}-del-012345"; + let mut settings = serde_json::to_value( + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse base settings for JSON validation"), + ) + .expect("should serialize base settings for JSON validation"); + settings["trusted_client_ip"] = json!({ + "ip_header": "fastly-client-ip", + "auth_header": "x-trusted-client-auth", + "shared_secret": shared_secret, + }); + + let error = Settings::from_json_value(settings) + .expect_err("should reject a shared secret containing DEL"); + let message = format!("{error:?}"); + + assert!( + message.contains("invalid_trusted_client_ip_shared_secret"), + "should identify the non-header-safe trusted client IP shared secret" + ); + assert!( + !message.contains(shared_secret), + "should not expose the shared secret containing DEL" + ); + } + + #[test] + fn trusted_client_ip_rejects_a_shared_secret_with_a_control_byte_without_exposing_it() { + let shared_secret = "valid-shared-secret-with\u{0001}-control-012345"; + let mut settings = serde_json::to_value( + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse base settings for JSON validation"), + ) + .expect("should serialize base settings for JSON validation"); + settings["trusted_client_ip"] = json!({ + "ip_header": "fastly-client-ip", + "auth_header": "x-trusted-client-auth", + "shared_secret": shared_secret, + }); + + let error = Settings::from_json_value(settings) + .expect_err("should reject a shared secret containing a control byte"); + let message = format!("{error:?}"); + + assert!( + message.contains("invalid_trusted_client_ip_shared_secret"), + "should identify the non-header-safe trusted client IP shared secret" + ); + assert!( + !message.contains(shared_secret), + "should not expose the shared secret containing a control byte" + ); + } + + #[test] + fn trusted_client_ip_rejects_placeholder_shared_secrets() { + for placeholder in TrustedClientIpConfig::SHARED_SECRET_PLACEHOLDERS { + assert!( + TrustedClientIpConfig::is_placeholder_shared_secret(placeholder), + "should detect placeholder shared secret '{placeholder}'" + ); + assert!( + TrustedClientIpConfig::is_placeholder_shared_secret(&placeholder.to_uppercase()), + "should detect placeholder shared secret case-insensitively" + ); + + let settings = Settings::from_toml(&trusted_client_ip_toml( + "fastly-client-ip", + "x-trusted-client-auth", + placeholder, + )) + .expect("should parse a placeholder trusted client IP shared secret"); + let error = settings + .reject_placeholder_secrets() + .expect_err("should reject a placeholder trusted client IP shared secret"); + + assert!( + format!("{error:?}").contains("trusted_client_ip.shared_secret"), + "should name the placeholder trusted client IP shared secret field" + ); + } + } + #[test] fn auction_debug_comment_options_default_matches_serde_defaults() { let opts = AuctionDebugCommentOptions::default(); @@ -6568,6 +7124,10 @@ formats = [{ width = 300, height = 250 }] let co = settings .creative_opportunities .expect("should have creative_opportunities"); + assert!( + co.enabled, + "creative-opportunity templates should default to enabled" + ); assert_eq!(co.gam_network_id, "21765378893"); assert_eq!(co.auction_timeout_ms, Some(500)); assert_eq!( @@ -6577,6 +7137,45 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_disables_creative_opportunity_slots_when_configured_off() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"21765378893\"\n\n[[creative_opportunities.slot]]\nid = \"atf\"\npage_patterns = [\"/\"]\nformats = [{{ width = 300, height = 250 }}]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse disabled templates"); + assert!( + settings.creative_opportunity_slots().is_empty(), + "disabled template delivery should expose no runtime slots" + ); + } + + #[test] + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", + crate_test_settings_str() + ); + let env_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_var(env_key, Some("false"), || { + let settings = Settings::from_toml_and_env(&toml) + .expect("should parse template enabled environment override"); + assert!( + !settings + .creative_opportunities + .expect("should have creative opportunities") + .enabled, + "environment override should disable template delivery" + ); + }); + } + #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts index 6c0301e2c..1604f2076 100644 --- a/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts @@ -4,6 +4,8 @@ import { expect, test, type Page } from "@playwright/test"; import { runtimeUrl } from "../../helpers/state.js"; const RUNNER_URL = "https://client.aps.amazon-adsystem.com/prebid-creative.js"; +const PUBLISHER_CORE_URL = + "https://client.aps.amazon-adsystem.com/trusted-server-core.js"; const IFRAME_CREATIVE_URL = "https://creative.example/iframe"; const SCRIPT_CREATIVE_URL = "https://creative.example/script.js"; const SANDBOX = @@ -155,6 +157,9 @@ const FAKE_RUNNER = `(function(){ if (bid.ext.tagtype === 'iframe') { var frame = document.createElement('iframe'); frame.setAttribute('sandbox', 'allow-scripts allow-same-origin'); + frame.width = String(bid.w); + frame.height = String(bid.h); + frame.style.border = '0'; frame.src = bid.ext.creativeurl; document.body.appendChild(frame); } else { @@ -192,7 +197,7 @@ const SCRIPT_CREATIVE = `(function(){ }, '*'); })();`; -test.describe("APS opaque renderer", () => { +test.describe("APS rendering", () => { test("renders a trustedServer adapter bid using Prebid's generated GAM ad ID", async ({ page, }) => { @@ -956,4 +961,161 @@ parent.postMessage(JSON.stringify({ 1, ); }); + + test("renders publisher-native mode through the injected friendly-frame runner", async ({ + page, + }) => { + const publisherOrigin = "https://publisher.example"; + const auctionUrl = `${publisherOrigin}/auction`; + const testUrl = `${publisherOrigin}/aps-publisher-native-test`; + const renderer = descriptor("iframe"); + const coreBundle = readFileSync(clientAuctionBundlePaths().core, "utf8"); + let runnerRequests = 0; + + await page.route(PUBLISHER_CORE_URL, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/javascript", + body: coreBundle, + }); + }); + await page.route(RUNNER_URL, async (route) => { + runnerRequests += 1; + await route.fulfill({ + status: 200, + contentType: "application/javascript", + body: FAKE_RUNNER, + }); + }); + await page.route(IFRAME_CREATIVE_URL, async (route) => { + await route.fulfill({ + status: 200, + contentType: "text/html", + body: IFRAME_CREATIVE, + }); + }); + await page.route(auctionUrl, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + id: "fictional-native-auction", + seatbid: [ + { + seat: "aps", + bid: [ + { + id: renderer.bidId, + impid: "publisher-native-slot", + price: 1.23, + w: renderer.width, + h: renderer.height, + ext: { trusted_server: { renderer } }, + }, + ], + }, + ], + ext: {}, + }), + }); + }); + await page.route(testUrl, async (route) => { + await route.fulfill({ + status: 200, + contentType: "text/html", + headers: { + "Content-Security-Policy": + "default-src 'none'; script-src https://client.aps.amazon-adsystem.com https://creative.example; connect-src 'self'; frame-src https://creative.example", + }, + body: ` +
existing publisher content
`, + }); + }); + + await page.goto(testUrl); + await page.evaluate(async (scriptUrl) => { + await new Promise((resolveScript, rejectScript) => { + const script = document.createElement("script"); + script.setAttribute( + "data-ts-aps-rendering-mode", + "publisher_native", + ); + script.src = scriptUrl; + script.addEventListener("load", () => resolveScript(), { + once: true, + }); + script.addEventListener( + "error", + () => rejectScript(new Error("TSJS core failed to load")), + { once: true }, + ); + document.head.appendChild(script); + }); + }, PUBLISHER_CORE_URL); + await page.evaluate(() => { + const tsjs = ( + window as unknown as { + tsjs: { + addAdUnits(units: Array>): void; + requestAds(): void; + }; + } + ).tsjs; + tsjs.addAdUnits([ + { + code: "publisher-native-slot", + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [], + }, + ]); + tsjs.requestAds(); + }); + + await expect.poll(() => runnerRequests).toBe(1); + const frame = page.locator("#publisher-native-slot > iframe"); + await expect(frame).toHaveCount(1); + await expect(frame).toBeVisible(); + expect(await frame.getAttribute("sandbox")).toBeNull(); + await expect( + frame + .contentFrame() + .locator(`iframe[src="${IFRAME_CREATIVE_URL}"]`), + ).toHaveCount(1); + await expect( + page.locator("#publisher-native-slot .existing"), + ).toHaveCount(0); + expect( + await frame.evaluate((element: HTMLIFrameElement) => { + const document = element.contentDocument!; + const creative = document.body.querySelector("iframe")!; + return { + bodyMargin: getComputedStyle(document.body).margin, + bodyPadding: getComputedStyle(document.body).padding, + creativeDisplay: getComputedStyle(creative).display, + clientWidth: document.documentElement.clientWidth, + clientHeight: document.documentElement.clientHeight, + scrollWidth: document.documentElement.scrollWidth, + scrollHeight: document.documentElement.scrollHeight, + }; + }), + ).toEqual({ + bodyMargin: "0px", + bodyPadding: "0px", + creativeDisplay: "block", + clientWidth: 300, + clientHeight: 250, + scrollWidth: 300, + scrollHeight: 250, + }); + expect( + await page + .locator("#publisher-native-slot") + .evaluate( + (slot) => + slot.querySelectorAll( + 'iframe[src*="/integrations/aps/renderer"]', + ).length, + ), + ).toBe(0); + }); }); diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index 5d8e41971..b2c4e41e1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -36,10 +36,10 @@ api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; // Defensive defaults: the edge injects adSlots (head-open) and bids (before -// ) only when the server-side ad stack runs for the request. When it -// is gated off (kill switch, consent fail-closed, bots, prefetch), page code -// reading window.tsjs.bids / window.tsjs.adSlots must still see defined -// values instead of throwing. Injected scripts overwrite these wholesale. +// ) only when server-side ad templates run for the request. When template +// delivery is disabled or gated off (auction/consent, bots, prefetch), page code +// reading window.tsjs.bids / window.tsjs.adSlots must still see defined values +// instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; // Point global tsjs diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index df3fee6a6..c05070a9d 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -1,5 +1,5 @@ // Request orchestration for tsjs: unified auction endpoint with iframe-based creative rendering. -import { renderApsCreative } from '../integrations/aps/render'; +import { dispatchApsRendering, renderApsCreative } from '../integrations/aps/render'; import { buildAdRequest, sendAuction } from './auction'; import { collectContext } from './context'; @@ -52,7 +52,13 @@ export function requestAds( for (const bid of bids) { if (!bid.impid) continue; if (bid.renderer) { - renderApsCreative({ slotId: bid.impid, renderer: bid.renderer }); + void Promise.resolve( + dispatchApsRendering({ + slotId: bid.impid, + renderer: bid.renderer, + trustedServer: (renderer) => renderApsCreative({ slotId: bid.impid, renderer }), + }) + ); continue; } if (!bid.adm) { diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index 85f17adf9..377713ba3 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -1,7 +1,12 @@ import { log } from '../../core/log'; +import { findSlot } from '../../core/render'; import type { ApsPrebidRendererEntry, ApsRendererV1, TsjsApi } from '../../core/types'; export const APS_RENDERER_PATH = '/integrations/aps/renderer'; +export const APS_RENDERING_MODE_ATTRIBUTE_NAME = 'data-ts-aps-rendering-mode'; +export const APS_PREBID_CREATIVE_RUNNER_URL = + 'https://client.aps.amazon-adsystem.com/prebid-creative.js'; +export const APS_NATIVE_RENDERER_TIMEOUT_MS = 10_000; export const APS_RENDERER_SANDBOX = 'allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation'; export const APS_UNIVERSAL_CREATIVE_RENDERER_VERSION = 4; @@ -38,6 +43,92 @@ type ValidatedRendererCacheEntry = { renderer: ApsRendererV1; }; const validatedRendererCache = new WeakMap(); +const nativeDispatches = new Map(); +const publisherNativeRendering = + typeof document !== 'undefined' && + document.currentScript?.getAttribute(APS_RENDERING_MODE_ATTRIBUTE_NAME) === 'publisher_native'; + +function releaseNativeDispatch(slotId: string, dispatch: symbol): boolean { + if (nativeDispatches.get(slotId) !== dispatch) return false; + nativeDispatches.delete(slotId); + return true; +} + +function sourceBelongsToElement( + source: MessageEventSource | null | undefined, + element: HTMLElement +): boolean { + return source + ? Array.from(element.querySelectorAll('iframe')).some( + (iframe) => iframe.contentWindow === source + ) + : false; +} + +function sourceMatchedCandidates( + candidates: HTMLElement[], + source?: MessageEventSource | null +): HTMLElement[] { + if (!source) return candidates; + const sourceMatches = candidates.filter((element) => sourceBelongsToElement(source, element)); + return sourceMatches.length > 0 ? sourceMatches : candidates; +} + +function dynamicSlotCandidates( + divIdPrefix: string, + source?: MessageEventSource | null +): HTMLElement[] { + const candidates = Array.from(document.querySelectorAll('[id]')).filter( + (element) => element.id.startsWith(divIdPrefix) && !element.id.endsWith('-container') + ); + return sourceMatchedCandidates(candidates, source); +} + +function uniqueSlotCandidate(candidates: HTMLElement[]): HTMLElement | null { + return candidates.length === 1 ? candidates[0]! : null; +} + +function findApsContainer(slotId: string, source?: MessageEventSource | null): HTMLElement | null { + try { + const mapping = window.tsjs?.divToSlotId ?? {}; + const mappedCandidates = sourceMatchedCandidates( + Object.entries(mapping) + .filter(([, mappedSlotId]) => mappedSlotId === slotId) + .map(([divId]) => findSlot(divId)) + .filter((element): element is HTMLElement => element !== null), + source + ); + const mapped = uniqueSlotCandidate(mappedCandidates); + if (mapped) return mapped; + + if (slotId.endsWith('-container')) { + const inner = findSlot(slotId.slice(0, -'-container'.length)); + if (inner) return inner; + } + + const direct = findSlot(slotId); + if (direct && !direct.id.endsWith('-container')) return direct; + + const configuredDivId = window.tsjs?.adSlots?.find((slot) => slot.id === slotId)?.div_id; + if (configuredDivId) { + const configured = findSlot(configuredDivId); + if (configured) return configured; + + const dynamic = uniqueSlotCandidate(dynamicSlotCandidates(configuredDivId, source)); + if (dynamic) return dynamic; + } + + const dynamic = uniqueSlotCandidate(dynamicSlotCandidates(slotId, source)); + return dynamic ?? direct; + } catch { + return null; + } +} + +function cancelPendingApsRendering(slotId: string, source?: MessageEventSource | null): void { + const container = findApsContainer(slotId, source); + if (container) pendingFrameCancels.get(container)?.(); +} function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -277,6 +368,202 @@ export function consumeApsPrebidRenderer(adId: string, expected: ApsPrebidRender return true; } +export interface DispatchApsRenderingOptions { + slotId: string; + renderer: unknown; + source?: MessageEventSource | null; + /** Existing Trusted Server owner, invoked only in the default mode. */ + trustedServer: (renderer: ApsRendererV1) => boolean; +} + +/** + * Dispatch a validated APS descriptor to exactly one configured rendering owner. + * + * Native mode loads APS's fixed Prebid creative runner in a publisher-origin friendly + * frame. Superseded attempts are cancelled and never fall back to the opaque renderer. + */ +export function dispatchApsRendering({ + slotId, + renderer: input, + source, + trustedServer, +}: DispatchApsRenderingOptions): boolean | Promise { + // Every attempt supersedes a pending frame for this slot, including an invalid + // replacement that fails before a new frame can be created. + cancelPendingApsRendering(slotId, source); + const dispatch = Symbol(slotId); + nativeDispatches.set(slotId, dispatch); + + const renderer = validateApsRenderer(input); + if (!renderer) { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS renderer: rejected descriptor'); + return false; + } + if (!publisherNativeRendering) { + try { + return trustedServer(renderer); + } finally { + releaseNativeDispatch(slotId, dispatch); + } + } + + let rendering: Promise; + try { + rendering = renderApsPublisherNative({ slotId, renderer, source }); + } catch { + releaseNativeDispatch(slotId, dispatch); + log.warn('APS native renderer: failed to start publisher-origin frame'); + return Promise.resolve(false); + } + + return rendering.then((accepted) => { + if (!releaseNativeDispatch(slotId, dispatch)) { + if (accepted) log.warn('APS native renderer: ignored stale completion'); + return false; + } + return accepted; + }); +} + +interface RenderApsPublisherNativeOptions { + slotId: string; + renderer: unknown; + source?: MessageEventSource | null; +} + +function prepareApsRunnerDocument( + frameWindow: Window & typeof globalThis, + frameDocument: Document +): void { + for (const element of [frameDocument.documentElement, frameDocument.body]) { + element.style.margin = '0px'; + element.style.padding = '0px'; + } + + const normalizeFrame = (node: Node): void => { + if ( + node instanceof frameWindow.HTMLIFrameElement && + node.parentElement === frameDocument.body + ) { + node.style.display = 'block'; + } + }; + Array.from(frameDocument.body.children).forEach(normalizeFrame); + new frameWindow.MutationObserver((records) => { + for (const record of records) record.addedNodes.forEach(normalizeFrame); + }).observe(frameDocument.body, { childList: true }); +} + +/** Render the exact selected response through APS's fixed runner in a friendly iframe. */ +function renderApsPublisherNative({ + slotId, + renderer: input, + source, +}: RenderApsPublisherNativeOptions): Promise { + const renderer = validateApsRenderer(input); + const container = findApsContainer(slotId, source); + if (!renderer || !container) { + log.warn( + renderer ? 'APS native renderer: slot not found' : 'APS renderer: rejected descriptor' + ); + return Promise.resolve(false); + } + + // Keep an already committed creative visible until the replacement runner loads. + pendingFrameCancels.get(container)?.(); + const iframe = document.createElement('iframe'); + iframe.title = 'Ad content'; + iframe.width = String(renderer.width); + iframe.height = String(renderer.height); + iframe.style.border = '0'; + iframe.style.display = 'none'; + activeFrames.set(container, iframe); + + return new Promise((resolve) => { + let settled = false; + let runner: HTMLScriptElement | undefined; + + const cleanup = (): void => { + window.clearTimeout(timeoutId); + runner?.removeEventListener('load', commit); + runner?.removeEventListener('error', fail); + }; + const finish = (accepted: boolean, warning?: string): void => { + if (settled) return; + settled = true; + cleanup(); + if (pendingFrameCancels.get(container) === cancel) pendingFrameCancels.delete(container); + + if (!accepted || activeFrames.get(container) !== iframe || !iframe.isConnected) { + if (activeFrames.get(container) === iframe) activeFrames.delete(container); + iframe.remove(); + if (warning) log.warn(warning); + resolve(false); + return; + } + + for (const child of Array.from(container.children)) { + if (child !== iframe) child.remove(); + } + iframe.style.display = ''; + resolve(true); + }; + const cancel = (): void => finish(false); + function fail(): void { + finish(false, 'APS native renderer: creative runner failed'); + } + function commit(): void { + finish(true); + } + + const timeoutId = window.setTimeout( + () => finish(false, 'APS native renderer: creative runner timed out'), + APS_NATIVE_RENDERER_TIMEOUT_MS + ); + pendingFrameCancels.set(container, cancel); + container.appendChild(iframe); + + try { + const frameWindow = iframe.contentWindow as + | (Window & + typeof globalThis & { + _aps: Map> }>; + }) + | null; + const frameDocument = iframe.contentDocument; + if (!frameWindow || !frameDocument) { + fail(); + return; + } + + frameDocument.open(); + frameDocument.write( + '' + ); + frameDocument.close(); + prepareApsRunnerDocument(frameWindow, frameDocument); + frameWindow._aps = new Map(); + frameWindow._aps.set(renderer.accountId, { + queue: [ + new frameWindow.CustomEvent('prebid/creative/render', { + detail: { aaxResponse: renderer.aaxResponse, seatBidId: renderer.bidId }, + }), + ], + store: new Map([['listeners', new Map()]]), + }); + + runner = frameDocument.createElement('script'); + runner.src = APS_PREBID_CREATIVE_RUNNER_URL; + runner.addEventListener('load', commit, { once: true }); + runner.addEventListener('error', fail, { once: true }); + frameDocument.head.appendChild(runner); + } catch { + fail(); + } + }); +} + function createNonce(): string | undefined { if (typeof crypto === 'undefined' || typeof crypto.getRandomValues !== 'function') return undefined; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8b61c8b73..89b480c6f 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -11,6 +11,7 @@ import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, apsRendererUrl, + dispatchApsRendering, consumeApsPrebidRenderer, getApsPrebidRenderer, validateApsRenderer, @@ -1117,8 +1118,8 @@ export function installTsAdInit(): void { ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. + // response (template switch, auction gate, or consent denial) returns no + // slots, so the loops above leave these empty. const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page @@ -1426,10 +1427,10 @@ export function installSpaAuctionHook(): void { // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; - // An empty page-bids response (auction kill switch or consent gate) carries - // no TS slots. Only run adInit() when there are slots to apply or prior TS - // state to sweep — otherwise a consent-denied or kill-switched navigation - // must not enter the GPT command queue and risk activating services. + // An empty page-bids response (template switch, auction, or consent gate) + // carries no TS slots. Only run adInit() when there are slots to apply or + // prior TS state to sweep — otherwise a gated navigation must not enter + // the GPT command queue and risk activating services. const hasPriorTsState = (ts.prevGptSlots?.length ?? 0) > 0 || Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || @@ -1699,29 +1700,50 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; - if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; + if (!renderer || !hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); - port.postMessage( - JSON.stringify({ - message: 'Prebid Response', - adId, - renderer: APS_UNIVERSAL_CREATIVE_RENDERER, - rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, - apsRenderer: renderer, - width: renderer.width, - height: renderer.height, - }) - ); - - try { - prebidRendererEntry.markUsed(); - } catch (err) { - log.warn(`[tsjs-gpt] APS Prebid markUsed callback threw for '${adId}'`, err); + const markUsed = (): void => { + try { + prebidRendererEntry.markUsed(); + } catch (err) { + log.warn(`[tsjs-gpt] APS Prebid markUsed callback threw for '${adId}'`, err); + } + }; + const dispatched = dispatchApsRendering({ + slotId: prebidRendererEntry.adUnitCode, + renderer, + source: e.source, + trustedServer: (validatedRenderer) => { + const rendererUrl = apsRendererUrl(); + if (!rendererUrl) return false; + try { + port.postMessage( + JSON.stringify({ + message: 'Prebid Response', + adId, + renderer: APS_UNIVERSAL_CREATIVE_RENDERER, + rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + rendererUrl, + apsRenderer: validatedRenderer, + width: validatedRenderer.width, + height: validatedRenderer.height, + }) + ); + return true; + } catch (err) { + log.warn(`[tsjs-gpt] APS Prebid response post failed for '${adId}'`, err); + return false; + } + }, + }); + if (typeof dispatched === 'boolean') { + if (dispatched) markUsed(); + } else { + void dispatched.then((accepted) => { + if (accepted) markUsed(); + }); } return; } @@ -1750,19 +1772,35 @@ export function installTsRenderBridge(): void { e.stopImmediatePropagation(); if (consumedServerApsBySlot.get(slotId) === adId) return; const renderer = validateApsRenderer(matchedBid.renderer); - const rendererUrl = apsRendererUrl(); - if (!renderer || !rendererUrl) return; + if (!renderer) return; consumedServerApsBySlot.set(slotId, adId); - port.postMessage( - JSON.stringify({ - message: 'Prebid Response', - adId, - renderer: APS_UNIVERSAL_CREATIVE_RENDERER, - rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, - rendererUrl, - apsRenderer: renderer, - width: renderer.width, - height: renderer.height, + void Promise.resolve( + dispatchApsRendering({ + slotId, + renderer, + source: e.source, + trustedServer: (validatedRenderer) => { + const rendererUrl = apsRendererUrl(); + if (!rendererUrl) return false; + try { + port.postMessage( + JSON.stringify({ + message: 'Prebid Response', + adId, + renderer: APS_UNIVERSAL_CREATIVE_RENDERER, + rendererVersion: APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + rendererUrl, + apsRenderer: validatedRenderer, + width: validatedRenderer.width, + height: validatedRenderer.height, + }) + ); + return true; + } catch (err) { + log.warn(`[tsjs-gpt] APS server response post failed for '${slotId}'`, err); + return false; + } + }, }) ); return; diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index dc17c9e87..e88ed520c 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { AdUnit } from '../../src/core/types'; +import { + APS_PREBID_CREATIVE_RUNNER_URL, + APS_RENDERING_MODE_ATTRIBUTE_NAME, +} from '../../src/integrations/aps/render'; import envelope from '../fixtures/aps-renderer-v1.json'; async function flushRequestAds(): Promise { @@ -137,6 +141,73 @@ describe('request.requestAds', () => { expect(document.querySelector('#slot1 span')).toBeNull(); }); + it('contract test: renders a direct APS bid through the injected native runner', async () => { + const apsBid = envelope.seatbid[0].bid[0]; + const renderer = { + type: 'aps' as const, + version: 1 as const, + accountId: 'example-account-id', + bidId: apsBid.id, + tagType: apsBid.ext.tagtype as 'iframe', + creativeUrl: apsBid.ext.creativeurl, + aaxResponse: btoa(JSON.stringify(envelope)), + width: apsBid.w, + height: apsBid.h, + }; + const publisherScript = document.createElement('script'); + publisherScript.setAttribute(APS_RENDERING_MODE_ATTRIBUTE_NAME, 'publisher_native'); + const currentScriptSpy = vi + .spyOn(document, 'currentScript', 'get') + .mockReturnValue(publisherScript); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + seatbid: [ + { + seat: 'aps', + bid: [{ impid: 'slot1', ext: { trusted_server: { renderer } } }], + }, + ], + }), + }); + + try { + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + currentScriptSpy.mockRestore(); + document.body.innerHTML = '
existing
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } }); + + requestAds(); + await flushRequestAds(); + const frame = document.querySelector('#slot1 iframe')!; + const runner = frame.contentDocument?.querySelector('script'); + expect(runner).not.toBeNull(); + const frameWindow = frame.contentWindow as unknown as { + _aps: Map>> }>; + }; + const queued = frameWindow._aps.get(renderer.accountId)?.queue[0]; + + expect(frame.getAttribute('sandbox')).toBeNull(); + expect(runner!.src).toBe(APS_PREBID_CREATIVE_RUNNER_URL); + expect(queued?.type).toBe('prebid/creative/render'); + expect(queued?.detail).toEqual({ + aaxResponse: renderer.aaxResponse, + seatBidId: renderer.bidId, + }); + expect(document.querySelector('#slot1 span')).not.toBeNull(); + + runner!.dispatchEvent(new Event('load')); + await Promise.resolve(); + expect(document.querySelector('#slot1 span')).toBeNull(); + expect(frame.style.display).toBe(''); + } finally { + currentScriptSpy.mockRestore(); + } + }); + it('does not mutate the slot for an invalid APS descriptor', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index eae60c90e..4157e492b 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -4,11 +4,15 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; import type { ApsRendererV1 } from '../../../src/core/types'; import { log } from '../../../src/core/log'; import { + APS_NATIVE_RENDERER_TIMEOUT_MS, + APS_PREBID_CREATIVE_RUNNER_URL, APS_RENDERER_PATH, APS_RENDERER_SANDBOX, + APS_RENDERING_MODE_ATTRIBUTE_NAME, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, apsRendererUrl, + dispatchApsRendering as dispatchDefaultApsRendering, getApsPrebidRenderer, parseApsRendererDescriptor, registerApsPrebidRenderer, @@ -16,6 +20,20 @@ import { validateApsRenderer, } from '../../../src/integrations/aps/render'; +function nativeRunnerState(frame: HTMLIFrameElement): { + runner: HTMLScriptElement; + event: CustomEvent<{ aaxResponse: string; seatBidId: string }>; +} { + const runner = frame.contentDocument?.querySelector('script'); + const frameWindow = frame.contentWindow as unknown as { + _aps: Map> }>; + }; + const account = frameWindow._aps.get('example-account-id'); + expect(runner).not.toBeNull(); + expect(account?.queue).toHaveLength(1); + return { runner: runner!, event: account!.queue[0] }; +} + function encodeBytes(bytes: Uint8Array): string { let binary = ''; for (const byte of bytes) binary += String.fromCharCode(byte); @@ -261,6 +279,254 @@ describe('Prebid APS renderer registry', () => { }); }); +describe('APS rendering-mode authorization', () => { + it('ignores mode markers and duplicate script tags injected after module initialization', () => { + document.body.innerHTML = '
'; + document.head.insertAdjacentHTML( + 'beforeend', + '' + + '' + ); + const trustedServer = vi.fn(() => true); + + expect( + dispatchDefaultApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer, + }) + ).toBe(true); + expect(trustedServer).toHaveBeenCalledOnce(); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + + document.head + .querySelectorAll( + 'meta[name="trusted-server-aps-rendering-mode"], script[data-ts-aps-rendering-mode]' + ) + .forEach((element) => element.remove()); + document.body.innerHTML = ''; + }); +}); + +describe('publisher-native APS runner contract tests', () => { + let dispatchApsRendering: typeof dispatchDefaultApsRendering; + + beforeEach(async () => { + vi.resetModules(); + document.body.innerHTML = '
existing
'; + const publisherScript = document.createElement('script'); + publisherScript.setAttribute(APS_RENDERING_MODE_ATTRIBUTE_NAME, 'publisher_native'); + const currentScriptSpy = vi + .spyOn(document, 'currentScript', 'get') + .mockReturnValue(publisherScript); + ({ dispatchApsRendering } = await import('../../../src/integrations/aps/render')); + currentScriptSpy.mockRestore(); + }); + + afterEach(() => { + delete window.tsjs; + vi.restoreAllMocks(); + document.body.innerHTML = ''; + }); + + it('queues the exact selected response for the fixed APS runner and commits on load', async () => { + const trustedServer = vi.fn(() => true); + const accepted = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer, + }); + const slot = document.getElementById('fictional-slot')!; + const frame = slot.querySelector('iframe')!; + const { runner, event } = nativeRunnerState(frame); + + expect(frame.getAttribute('sandbox')).toBeNull(); + expect(frame.style.display).toBe('none'); + expect(runner.src).toBe(APS_PREBID_CREATIVE_RUNNER_URL); + expect(event.type).toBe('prebid/creative/render'); + expect(event.detail).toEqual({ + aaxResponse: descriptor().aaxResponse, + seatBidId: descriptor().bidId, + }); + expect(slot.querySelector('span')).not.toBeNull(); + expect(trustedServer).not.toHaveBeenCalled(); + + const runnerDocument = frame.contentDocument!; + expect(runnerDocument.documentElement.style.margin).toBe('0px'); + expect(runnerDocument.documentElement.style.padding).toBe('0px'); + expect(runnerDocument.body.style.margin).toBe('0px'); + expect(runnerDocument.body.style.padding).toBe('0px'); + const creativeFrame = runnerDocument.createElement('iframe'); + runnerDocument.body.appendChild(creativeFrame); + await vi.waitFor(() => expect(creativeFrame.style.display).toBe('block')); + + runner.dispatchEvent(new Event('load')); + await expect(accepted).resolves.toBe(true); + expect(slot.querySelector('span')).toBeNull(); + expect(frame.style.display).toBe(''); + }); + + it('fails closed when the runner fails without clearing publisher content', async () => { + const trustedServer = vi.fn(() => true); + const accepted = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer, + }); + const frame = document.querySelector('#fictional-slot iframe')!; + const { runner } = nativeRunnerState(frame); + + runner.dispatchEvent(new Event('error')); + + await expect(accepted).resolves.toBe(false); + expect(trustedServer).not.toHaveBeenCalled(); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot span')).not.toBeNull(); + }); + + it('cancels a pending runner when a newer dispatch replaces it', async () => { + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + const firstFrame = document.querySelector('#fictional-slot iframe')!; + + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + const secondFrame = document.querySelector('#fictional-slot iframe')!; + + expect(firstFrame.isConnected).toBe(false); + expect(secondFrame).not.toBe(firstFrame); + await expect(first).resolves.toBe(false); + nativeRunnerState(secondFrame).runner.dispatchEvent(new Event('load')); + await expect(second).resolves.toBe(true); + }); + + it('lets an invalid replacement cancel an older pending runner', async () => { + const first = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + const second = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor({ aaxResponse: 'invalid' }), + trustedServer: () => true, + }); + + expect(second).toBe(false); + await expect(first).resolves.toBe(false); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot span')).not.toBeNull(); + }); + + it('resolves a logical GPT slot through the injected div mapping', async () => { + document.body.innerHTML = '
existing
'; + window.tsjs = { divToSlotId: { 'div-header': 'homepage_header' } } as typeof window.tsjs; + + const accepted = dispatchApsRendering({ + slotId: 'homepage_header', + renderer: descriptor(), + trustedServer: () => true, + }); + const frame = document.querySelector('#div-header iframe')!; + nativeRunnerState(frame).runner.dispatchEvent(new Event('load')); + + await expect(accepted).resolves.toBe(true); + expect(document.querySelector('#div-header span')).toBeNull(); + }); + + it('renders inside the inner slot when Prebid uses its container ID', async () => { + document.body.innerHTML = + '
'; + const source = document.querySelector('#div-header > iframe')!.contentWindow; + + const accepted = dispatchApsRendering({ + slotId: 'div-header-container', + renderer: descriptor(), + source, + trustedServer: () => true, + }); + const frame = Array.from( + document.querySelectorAll('#div-header > iframe') + ).find((candidate) => candidate.title === 'Ad content')!; + nativeRunnerState(frame).runner.dispatchEvent(new Event('load')); + + await expect(accepted).resolves.toBe(true); + expect(document.getElementById('div-header-container')).not.toBeNull(); + expect(document.getElementById('div-header')).not.toBeNull(); + expect(document.querySelectorAll('#div-header > iframe')).toHaveLength(1); + }); + + it('uses the requesting frame to resolve a dynamic slot prefix', async () => { + document.body.innerHTML = + '
' + + '
'; + const source = document.querySelector( + '#div-header-second > iframe' + )!.contentWindow; + + const accepted = dispatchApsRendering({ + slotId: 'div-header-', + renderer: descriptor(), + source, + trustedServer: () => true, + }); + const frame = Array.from( + document.querySelectorAll('#div-header-second > iframe') + ).find((candidate) => candidate.title === 'Ad content')!; + nativeRunnerState(frame).runner.dispatchEvent(new Event('load')); + + await expect(accepted).resolves.toBe(true); + expect(document.querySelector('#div-header-first > iframe')).not.toBeNull(); + expect(document.querySelectorAll('#div-header-second > iframe')).toHaveLength(1); + }); + + it('contains throwing publisher slot mappings without falling back', async () => { + const tsjs = {} as NonNullable; + Object.defineProperty(tsjs, 'divToSlotId', { + get: () => { + throw new Error('fictional mapping lookup failure'); + }, + }); + window.tsjs = tsjs; + const trustedServer = vi.fn(() => true); + + await expect( + dispatchApsRendering({ + slotId: 'logical-slot', + renderer: descriptor(), + trustedServer, + }) + ).resolves.toBe(false); + expect(trustedServer).not.toHaveBeenCalled(); + expect(document.querySelector('iframe')).toBeNull(); + }); + + it('times out an unacknowledged runner without clearing publisher content', async () => { + vi.useFakeTimers(); + try { + const result = dispatchApsRendering({ + slotId: 'fictional-slot', + renderer: descriptor(), + trustedServer: () => true, + }); + await vi.advanceTimersByTimeAsync(APS_NATIVE_RENDERER_TIMEOUT_MS); + + await expect(result).resolves.toBe(false); + expect(vi.getTimerCount()).toBe(0); + expect(document.querySelector('#fictional-slot iframe')).toBeNull(); + expect(document.querySelector('#fictional-slot span')).not.toBeNull(); + } finally { + vi.useRealTimers(); + } + }); +}); + describe('direct APS rendering', () => { beforeEach(() => { document.body.innerHTML = '
existing
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index fb9cdecf0..b7186518b 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -7,6 +7,42 @@ import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vites import envelope from '../../fixtures/aps-renderer-v1.json'; import type { AuctionBidData, TsjsApi } from '../../../src/core/types'; +import { + APS_PREBID_CREATIVE_RUNNER_URL, + APS_RENDERING_MODE_ATTRIBUTE_NAME, +} from '../../../src/integrations/aps/render'; + +let publisherNativeScript: HTMLScriptElement | undefined; + +function enablePublisherNativeMode(): { remove(): void } { + publisherNativeScript = document.createElement('script'); + publisherNativeScript.setAttribute(APS_RENDERING_MODE_ATTRIBUTE_NAME, 'publisher_native'); + return { + remove: () => { + publisherNativeScript = undefined; + }, + }; +} + +function nativeRunnerIn(divId: string): { + frame: HTMLIFrameElement; + runner: HTMLScriptElement; + event: CustomEvent<{ aaxResponse: string; seatBidId: string }>; +} { + const container = document.getElementById(divId)!; + const frame = Array.from(container.querySelectorAll('iframe')).find( + (candidate) => candidate.title === 'Ad content' + ); + expect(frame).not.toBeUndefined(); + const runner = frame!.contentDocument?.querySelector('script'); + const frameWindow = frame!.contentWindow as unknown as { + _aps: Map> }>; + }; + const event = Array.from(frameWindow._aps.values())[0]?.queue[0]; + expect(runner?.src).toBe(APS_PREBID_CREATIVE_RUNNER_URL); + expect(event).not.toBeUndefined(); + return { frame: frame!, runner: runner!, event }; +} function apsRenderer() { const bid = envelope.seatbid[0].bid[0]; @@ -2940,6 +2976,7 @@ describe('installTsRenderBridge', () => { beforeEach(() => { vi.resetModules(); + publisherNativeScript = undefined; // Remove ALL accumulated 'message' handlers from previous test module imports // to prevent stale bridge listeners from intercepting our test event. for (const handler of allMessageHandlers) { @@ -3000,6 +3037,9 @@ describe('installTsRenderBridge', () => { async function captureBridgeListener(): Promise<(e: MessageEvent) => unknown> { let bridgeListener: ((e: MessageEvent) => unknown) | undefined; const origAdd = window.addEventListener.bind(window); + const currentScriptSpy = publisherNativeScript + ? vi.spyOn(document, 'currentScript', 'get').mockReturnValue(publisherNativeScript) + : undefined; const addSpy = vi .spyOn(window, 'addEventListener') .mockImplementation( @@ -3014,6 +3054,7 @@ describe('installTsRenderBridge', () => { ); await import('../../../src/integrations/gpt/index'); addSpy.mockRestore(); + currentScriptSpy?.mockRestore(); expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); return bridgeListener!; @@ -3289,6 +3330,78 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('contract test: renders a server APS owner with the injected runner and no Universal Creative response', async () => { + const renderer = apsRenderer(); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: renderer.bidId, + renderer, + }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: renderer.bidId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + bridgeListener(request); + const native = nativeRunnerIn('div-header'); + expect(native.event.type).toBe('prebid/creative/render'); + expect(native.event.detail).toEqual({ + aaxResponse: renderer.aaxResponse, + seatBidId: renderer.bidId, + }); + native.runner.dispatchEvent(new Event('load')); + await Promise.resolve(); + await Promise.resolve(); + + expect(native.frame.style.display).toBe(''); + expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + + it('contract test: fails a server APS runner without a Universal Creative response or fallback', async () => { + const renderer = apsRenderer(); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: renderer.bidId, + renderer, + }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: renderer.bidId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + nativeRunnerIn('div-header').runner.dispatchEvent(new Event('error')); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(portMessages).toEqual([]); + expect(document.querySelector('iframe[title="Ad content"]')).toBeNull(); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + it('serves a registered Prebid APS renderer when its generated ad ID differs from the APS bid ID', async () => { const renderer = apsRenderer(); const prebidAdId = 'prebid-generated-ad-id'; @@ -3344,6 +3457,137 @@ describe('installTsRenderBridge', () => { foreignIframe.remove(); }); + it('contract test: fails a registered APS runner without a Universal Creative response or markUsed', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-prebid-decline-ad-id'; + const markUsed = vi.fn(); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + nativeRunnerIn('div-header').runner.dispatchEvent(new Event('error')); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(markUsed).not.toHaveBeenCalled(); + expect(portMessages).toEqual([]); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + expect(document.querySelector('iframe[title="Ad content"]')).toBeNull(); + expect(document.querySelector('iframe[src*="/integrations/aps/renderer"]')).toBeNull(); + } finally { + marker.remove(); + } + }); + + it('contract test: consumes a registered APS capability and marks it used only after runner load', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-prebid-ad-id'; + const markUsed = vi.fn(); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + const marker = enablePublisherNativeMode(); + + try { + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const request = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request); + expect(markUsed).not.toHaveBeenCalled(); + const native = nativeRunnerIn('div-header'); + native.runner.dispatchEvent(new Event('load')); + await Promise.resolve(); + await Promise.resolve(); + bridgeListener(request); + + expect(native.frame.style.display).toBe(''); + expect(markUsed).toHaveBeenCalledOnce(); + expect(portMessages).toEqual([]); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + } finally { + marker.remove(); + } + }); + + it('uses the requesting frame to resolve a registered APS dynamic slot prefix', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'native-dynamic-prebid-ad-id'; + const markUsed = vi.fn(); + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-native-', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed, + }, + }; + const marker = enablePublisherNativeMode(); + const firstSource = createTrustedSlotIframe('div-native-first'); + const source = createTrustedSlotIframe('div-native-second'); + + try { + const bridgeListener = await captureBridgeListener(); + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: vi.fn() }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + const native = nativeRunnerIn('div-native-second'); + native.runner.dispatchEvent(new Event('load')); + await Promise.resolve(); + await Promise.resolve(); + + expect(native.frame.style.display).toBe(''); + expect(markUsed).toHaveBeenCalledOnce(); + expect( + Array.from(document.querySelectorAll('#div-native-first iframe')).some( + (frame) => frame.contentWindow === firstSource + ) + ).toBe(true); + } finally { + marker.remove(); + document.getElementById('div-native-first')?.remove(); + document.getElementById('div-native-second')?.remove(); + } + }); + it('still serves the APS renderer when markUsed throws', async () => { const renderer = apsRenderer(); const prebidAdId = 'throwing-mark-used-ad-id'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index ab6d646f2..77dc6faca 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -33,6 +33,7 @@ interface MockGoogleTag { pubads: () => unknown; enableServices: () => void; display: (divId: string) => void; + getConfig?: (key: string) => Record; setConfig?: (config: Record) => void; } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 7127efcb4..314348fa8 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -216,9 +216,9 @@ describe('installSpaAuctionHook', () => { }); it('skips adInit on an empty page-bids response with no prior TS state', async () => { - // A gated page-bids response (auction kill switch or consent denial) returns - // no slots. With no prior TS state to sweep, the hook must not call adInit() - // so a consent-denied navigation cannot activate the publisher's GPT setup. + // A gated page-bids response (template switch, auction gate, or consent + // denial) returns no slots. With no prior TS state to sweep, the hook must + // not call adInit() so a gated navigation cannot activate publisher GPT. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index a47dbf155..26a0d5e3b 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -757,8 +757,8 @@ timeout_ms = 500 # Mediator budget (called after providers) ### Environment Variable Overrides The typed `ts config validate`, `ts config diff`, and `ts config push` flows can -override auction values that already exist in the TOML. EdgeZero v0.0.4 does -not create missing leaves, so existing configs must add **both** +override auction values that already exist in the TOML. EdgeZero's env overlay +does not create missing leaves, so existing configs must add **both** `rewrite_creatives = true` and `sanitize_creatives = false` under `[auction]` before relying on the corresponding environment overrides — an override for a missing leaf is silently ignored. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index b6829895e..1b6d95327 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -76,6 +76,67 @@ Trusted Server settings JSON. This blob model is intentional because full Trusted Server configs can exceed Fastly limits when split into one config-store entry per setting. +Reclaim orphaned chunk entries leaked from prior oversized pushes: + +```bash +ts config gc --adapter fastly +``` + +Without `--yes`, `config gc` only previews: it reports what it would delete and +deletes nothing. `--dry-run` states that intent explicitly and conflicts with +`--yes`. To actually delete, pass `--yes` together with `--older-than ` +(`s`/`m`/`h`/`d` suffixes, e.g. `7d`; a bare number means seconds): + +```bash +ts config gc --adapter fastly --yes --older-than 7d +``` + +`config gc` sweeps every root in the selected physical store, so `--older-than` +is a safety assertion about the whole store: nothing in it changed within the +window and no writer is targeting it. Unlike the other `config` subcommands, +`gc` never loads the typed app config; its `--no-env` flag instead ignores +`EDGEZERO__STORES__CONFIG____NAME` when resolving which physical store to +sweep. On a destructive run, check the store id `gc` reports before passing +`--yes`. + +### Diagnose ad-template configuration + +The static `ts config ad-templates` commands evaluate local configuration +without launching a browser: + +| Command | Purpose | +| ---------------------------------------- | ------------------------------------------------------------------------- | +| `lint` | Summarize configuration and report invalid slot page patterns. | +| `match [--details]` | List matching slots; `--details` includes divs, paths, formats/providers. | +| `check --expected-slot ID` | Assert the exact matching slot set; repeat `--expected-slot`. | +| `check --expect-no-slots` | Assert that no slots match. | +| `explain ` | Print every runtime ad-stack gate and its final yes/no verdict. | + +`check --allow-extra-slots` permits matches beyond the repeated +`--expected-slot` values. It conflicts with `--expect-no-slots`. + +`explain` models a GET navigation with consent allowed by default. Use +`--method `, `--non-navigation`, `--prefetch`, `--bot`, or +`--consent-denied` to model another request. Provider configuration is printed +as a separate advisory; it does not change the runtime gate verdict. + +Every `ts config ad-templates ...` and `ts audit ad-templates ...` command +accepts the same config-location flags: + +| Flag | Behavior | +| --------------------- | ----------------------------------------------------------------------------- | +| `--app-config ` | Read this app config instead of deriving `.toml` from the manifest. | +| `--manifest ` | Read this manifest; defaults to `edgezero.toml`. | +| `--no-env` | Disable `TRUSTED_SERVER__...` overlays for read-only commands. | + +The mutating audit generator always edits file-backed values and never writes +environment-only overlays into TOML, including during `--dry-run`. + +For CI-oriented assertions, exit code 0 means the assertion passed, 1 means the +command ran and found drift (`config ad-templates check` or audit verification +with `--strict`), and 2 means argument parsing, configuration, browser launch, +or another tool operation failed. + ## Lifecycle commands Lifecycle commands delegate to the selected EdgeZero adapter: @@ -98,7 +159,7 @@ Chrome or Chromium must be installed locally. The command checks common PATH names and standard macOS/Linux install locations. ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` By default, the command writes: @@ -118,13 +179,13 @@ ts config validate If a config already exists, avoid overwriting it: ```bash -ts audit https://publisher.example --no-config +ts audit generate https://publisher.example --no-config ``` Use custom output paths when reviewing artifacts first: ```bash -ts audit https://publisher.example \ +ts audit generate https://publisher.example \ --js-assets audit/js-assets.toml \ --config audit/trusted-server.toml ``` @@ -132,9 +193,284 @@ ts audit https://publisher.example \ Use `--force` only when replacing existing output files is intentional: ```bash -ts audit https://publisher.example --force +ts audit generate https://publisher.example --force +``` + +The legacy `ts audit ` form remains a compatibility alias for artifact +generation. New automation should use `ts audit generate `. + +## Generate ad-template slots from a live site + +`ts audit ad-templates generate ` discovers the publisher's ad slots and +rewrites the `[creative_opportunities]` slot array in `trusted-server.toml` in +place, preserving every other section and comment. + +```bash +ts audit ad-templates generate https://publisher.example/ +``` + +It samples the site rather than a single page. Ad slots repeat per site +section, so the crawl is sized by the publisher's taxonomy — a dozen sections — +not its catalogue: + +1. Load the requested page and read its links and, from `robots.txt`, its + sitemap. +2. Group both into candidate sections, keeping one landing page and one article + per section. +3. Load those pages, recording each slot's div, sizes, and GAM ad-unit path. +4. Reconcile every slot across the pages it appeared on. +5. Infer a `{section}` ad-unit template if the evidence proves one. +6. Verify the result loads, then write it. + +### What it writes + +Given a site whose ad units track the section, the run produces: + +```toml +[creative_opportunities] +gam_network_id = "99999" +section_root = "homepage" +section_segment = 0 + +[[creative_opportunities.slot]] +id = "ad-header-0" +div_id = "ad-header-0" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/deals", "/deals/*", "/news", "/news/*"] +formats = [{ width = 728, height = 90 }] +``` + +Each section contributes **two** patterns. `*` crosses `/` in this glob +dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing +page; emitting only the star form would drop the landing page from the slot. + +Sizes are unioned across pages, so a format that renders only on articles +survives alongside the homepage's. + +### When it keeps literal paths, and when it refuses + +A wrong ad-unit template makes the publisher bid against inventory that does not +exist, so the command prefers a narrow literal path over a plausible guess. + +| Situation | Result | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted; the note lists the ad-unit paths it used and says none generalized. | +| No crawled page lacked a section segment, so `section_root` is unwitnessed | No template is written and the reason names the crawl gap. A slot that merely never appears on the root (a sidebar, an in-article unit) still templates, borrowing the `section_root` another slot witnessed; a note says so. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | +| Several live elements normalize onto one div-id prefix | The whole group is omitted, on every page of the crawl. A prefix resolves to at most one element and the exact ids change per render; the prefix is named in a note. | +| A per-render token sits before the placement part of a div id | The slot is omitted from a single observation and the family prefix is named in a note; no stable prefix identifies one element. | + +Every run checks that the config it produced still loads before replacing the +file, and `--dry-run` runs the same check — a clean preview is evidence the +config loads, not just that it parses. Dry-run stdout is a zero-context unified +diff containing only the managed creative-opportunity fields; notes and refusal +reasons go to stderr, so unrelated config and secrets are not printed. Crawl +progress also goes to stderr, one line per phase and page — for example +`Auditing desktop [2/17]: /news`. Progress renders the path only, never the +origin, userinfo, query, or fragment, and there is no flag to suppress it. A +`--dry-run` that changes nothing says so on stderr too, leaving stdout an empty +diff. + +### Bounding and steering the crawl + +```bash +# Cover more of a large site. +ts audit ad-templates generate https://publisher.example/ --max-sections 20 --max-pages 41 + +# Audit exactly one page, as earlier releases did. +ts audit ad-templates generate https://publisher.example/ --max-pages 1 + +# Set the patterns yourself; this disables pattern inference entirely unless a +# slot's template had to borrow section_root from another slot. +ts audit ad-templates generate https://publisher.example/ \ + --page-pattern '/' --page-pattern '/news' --page-pattern '/news/*' + +# Preview without writing. +ts audit ad-templates generate https://publisher.example/ --dry-run +``` + +Re-running merges into the existing slots: a slot seen again keeps its +hand-tuned fields and gains this run's patterns and newly observed formats, and a hand-written +`gam_unit_path` template is preserved. `--replace` discards existing slots +instead, which also discards any template you wrote by hand. + +A slot that never appeared without a section segment can borrow a +`section_root` witnessed by another slot only while its patterns are derived +from the paths where it was observed. If `--page-pattern` would override those +patterns, generation fails and names the affected slots; remove the explicit +patterns so the safe per-slot patterns can be derived. + +A merge refuses to change the section policy that preserved `{section}` slots +were written against. If the config has a non-empty `section_root`, an inferred +root or segment mismatch fails and asks for `--replace` as an explicit +migration. An explicitly configured `section_segment` is preserved even when +`section_root` is unset. When the root is unset and the segment is either unset +or agrees with inference, the first merge adopts the inferred root and makes +the otherwise unloadable `{section}` config valid. + +Locale-prefixed sites are inferred at their observed section depth. Only real +ISO 639-1 language codes are read as a locale prefix, so a two-letter _section_ +root such as `/tv` or `/us` keeps sections at the first segment. For +example, `/en/news/story` can produce `section_segment = 1`; generated patterns +retain the locale prefix (`/en/news` and `/en/news/*`). The crawler never +invents an unwitnessed locale or section. + +Behind bot protection, pass a valid clearance cookie. The crawl reuses one +browser session, so clearance earned on the first page carries to the rest, and +`--page-delay-ms` spaces the requests — an unpaced crawl is both discourteous to +the origin and likelier to be challenged partway through: + +```bash +ts audit ad-templates generate https://publisher.example/ \ + --cookie '=' --page-delay-ms 1500 ``` +Some origins refuse a headless browser outright regardless of the cookie. +`--headful` runs a visible one, which is also the quickest way to _see_ whether +a challenge is being shown: + +```bash +ts audit ad-templates generate https://publisher.example/ --headful +``` + +### Sites behind a consent platform + +Publishers gate slot definition behind their consent platform, and the audit +runs in a throwaway browser profile with no consent cookie. Left alone, such a +site defines no slots at all and looks identical to a site with no ad stack. + +The crawl therefore answers the two IAB interfaces every compliant platform +exposes — TCF v2 and US Privacy — as a consenting, out-of-scope reader, before +any page script runs. This changes only what the audit browser sees; it does not +affect the publisher's own readers. Pass `--no-assume-consent` to observe the +un-consented page instead. + +When a page still yields no slots, the run reports GPT's observable state — +whether the library reached `apiReady`, how many queued commands never drained, +how many scripts ran. An empty slot registry has several very different causes, +and that line distinguishes them. + +### Auditing a production hostname served locally + +`ts dev proxy` serves a production hostname from a local Trusted Server. +Auditing through it keeps the page's origin, cookie scope, and any origin checks +in the ad stack matching production rather than `localhost`: + +```bash +ts dev proxy --map www.publisher.example=127.0.0.1:7676 --upstream-plaintext --rewrite-host + +ts audit ad-templates generate https://www.publisher.example/ \ + --browser-proxy 127.0.0.1:18080 --danger-accept-invalid-certs +``` + +`--danger-accept-invalid-certs` covers the proxy's MITM certificate when the +throwaway browser profile does not trust its CA; installing that CA +(`ts dev proxy ca`) is preferable. Against a real origin the flag is dangerous — +the audit sends any `--cookie` session upstream and treats the response as +evidence, so an invalid certificate could mean an impersonator is both +harvesting the session and fabricating the result. + +Note that a local Trusted Server injects its own configured slots into the page, +so a run through the proxy can rediscover config it already has. Slot ids that +are absent from the current config are the publisher's own. + +### Slots that change div id on every render + +Some ad stacks build div ids from a per-render token, so one placement arrives +under a new id on every page. Those ids match nothing at runtime, so the run +declines to write them and reports the group instead: + +```text +note: skipped 3 slot(s) that look like one placement under a per-render div id + on `/123456789/publisher/overlay` (ex_slot_a1_overlay_1, …); + they share the prefix `ex_slot`. Add it once by hand with a div_id prefix + that is stable across renders +``` + +The detection is by evidence, not by recognising token shapes: candidates share +an ad-unit path and formats, and what separates a fragmented placement from two +legitimate siblings on one unit is co-occurrence — real siblings appear together +on a page, fragments never do. The suggested prefix is a starting point only, not +written as a `div_id`, because it reaches only as far as the observed tokens +happen to agree. + +### Checking for a device split + +Publishers often serve a different ad unit per device +(`/network/desktop/news` against `/network/mobile/news`). A desktop-only crawl +cannot see that — it infers a template correct for desktop and silently wrong +for every mobile impression. + +```bash +ts audit ad-templates generate https://publisher.example/ --profiles desktop,mobile +``` + +Each page is loaded once per profile. Where the profiles disagree, the slot is +omitted and the diagnostic explains the conflicting paths. The generator does +not fall back to a fabricated default ad unit. + +### Deploy ordering for templated config + +> **A config containing `section_root` or `section_segment` is not +> rollback-safe.** These keys are rejected outright by a Trusted Server binary +> that predates ad-unit templating, and the rejection fails the _entire_ +> configuration load — not just the ad-template section — so every route serves +> an error. This is a full-site outage, not a degraded ad stack. + +When a run reports that it wrote a `{section}` template: + +1. Deploy the template-aware binary **first**. +2. Then `ts config push`. +3. Do **not** roll that binary back while the config is live. + +A run that did not template writes neither key, and leaves the config exactly as +rollback-safe as it was. + +### Audit safety defaults + +Every `ts audit` browser session validates TLS certificates. This matters +because `--cookie` sends a real session to the origin and the page's own +response becomes the audit's evidence, so a certificate-invalid host could both +harvest the session and fabricate what the audit reports. Override only for a +host you control with a known self-signed certificate: + +```bash +ts audit page https://staging.publisher.example --danger-accept-invalid-certs +``` + +`ts audit ad-templates verify` matches configured slots against the +**post-redirect** path, so it refuses a redirect that leaves the requested +origin rather than accepting another site's evidence as verification. Allow it +for a known redirect between your own properties (for example apex to `www`): + +```bash +ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-redirect +``` + +Verification accepts multiple URLs and reuses one browser/profile. Add +`--strict` to return exit 1 when a confirmable slot is missing or partially +confirmed, and `--json` for the stable machine-readable report. Video- and +native-only slots are reported as `unconfirmable`; that records a checker +limitation and does not fail strict mode. A live out-of-page slot with no sizes +against banner-configured formats is reported `partial` and does fail strict +mode. `--scroll` enables the optional second evidence phase and labels evidence +first seen after the deterministic scroll. + +Browser-backed ad-template generation and verification share `--chrome`, +`--headful`, `--browser-proxy`, `--no-assume-consent`, +`--settle-quiet-ms`, `--settle-max-ms`, and +`--danger-accept-invalid-certs`. Verification also accepts +`--browser-profile desktop|mobile`; generation uses +`--profiles desktop,mobile` to compare both profiles. `--cookie NAME=VALUE` is +repeatable and creates host-only, root-path cookies; HTTPS targets also mark +them Secure. Verification refuses cookies when URLs span multiple origins. The quiet settle window +must not exceed the maximum. + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 1240dce20..af77cb409 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -66,17 +66,18 @@ fail and the service will return its startup-error response. ## Key Sections -| Section | Purpose | -| ------------------- | -------------------------------------------- | -| `[publisher]` | Domain, origin, proxy settings | -| `[ec]` | Edge Cookie (EC) ID generation | -| `[tester_cookie]` | Optional tester-cookie endpoint | -| `[proxy]` | Proxy SSRF allowlist and asset routes | -| `[cache]` | Static/rehosted asset cache policy rules | -| `[image_optimizer]` | Reusable Image Optimizer profile sets | -| `[request_signing]` | Ed25519 request signing | -| `[auction]` | Auction orchestration | -| `[integrations.*]` | Partner integrations (Prebid, Next.js, etc.) | +| Section | Purpose | +| --------------------- | -------------------------------------------- | +| `[publisher]` | Domain, origin, proxy settings | +| `[trusted_client_ip]` | Authenticated client-IP forwarding | +| `[ec]` | Edge Cookie (EC) ID generation | +| `[tester_cookie]` | Optional tester-cookie endpoint | +| `[proxy]` | Proxy SSRF allowlist and asset routes | +| `[cache]` | Static/rehosted asset cache policy rules | +| `[image_optimizer]` | Reusable Image Optimizer profile sets | +| `[request_signing]` | Ed25519 request signing | +| `[auction]` | Auction orchestration | +| `[integrations.*]` | Partner integrations (Prebid, Next.js, etc.) | ## Example: Production Setup @@ -115,7 +116,7 @@ base TOML configuration by `ts config validate`, `ts config diff`, and stored in the app-config blob. Changing an environment variable requires rerunning validation and pushing the resolved config, not rebuilding the binary. -EdgeZero v0.0.4 only overrides leaves that already exist in the parsed TOML; it +EdgeZero's env overlay only overrides leaves that already exist in the parsed TOML; it does not create missing fields. Add newly introduced defaulted fields to an existing config before relying on their environment overrides. Pass `--no-env` to use file values without the overlay. @@ -353,6 +354,74 @@ a zero-byte cap fails every non-empty publisher response. TRUSTED_SERVER__PUBLISHER__MAX_BUFFERED_BODY_BYTES=16777216 ``` +## Trusted Client IP Configuration + +Use this optional section when a trusted CDN service forwards requests to the +Fastly service running Trusted Server. It lets Trusted Server use the reader's +address instead of the immediate fronting edge node's address. Only the Fastly +adapter honours this section; the Cloudflare, Spin, and Axum adapters validate +it but keep using their own runtime client address. + +### `[trusted_client_ip]` + +| Field | Type | Required | Description | +| --------------- | ------ | -------- | --------------------------------------------------------------------------------- | +| `ip_header` | String | Yes | Header containing exactly one reader IP address | +| `auth_header` | String | Yes | Header containing exactly one shared-secret value | +| `shared_secret` | String | Yes | Secret shared with the trusted front door, 32+ ASCII graphic bytes, no whitespace | + +All three fields are required when the section exists. When the section is +absent, Trusted Server continues to use the immediate peer address. + +```toml +[trusted_client_ip] +ip_header = "fastly-client-ip" +auth_header = "x-ts-client-ip-auth" +shared_secret = "replace-with-a-random-shared-secret" +``` + +The front door must overwrite both headers on every request. Trusted Server +accepts the forwarded address only when the request has exactly one +`auth_header` value that matches `shared_secret` byte-for-byte and exactly one +`ip_header` value that parses directly as IPv4 or IPv6. Values are not trimmed +or normalized. Missing, empty, duplicate, non-UTF-8, mismatched, or malformed +values do not reject the request; Trusted Server safely falls back to the +immediate peer address. Both configured headers are removed before routing. + +Header names are validated case-insensitively. `ip_header` must be +`fastly-client-ip` or start with `x-`, while `auth_header` must start with `x-`. +The names must differ. Neither field may use the reserved +`x-ts-tls-protocol` or `x-ts-tls-cipher` header. These restrictions exclude +standard sensitive headers such as `Host`, `Content-Length`, `Cookie`, and +`Authorization`, as well as Trusted Server's TLS bridge headers. Choose +dedicated `x-` names that no other application or routing logic uses, because +Trusted Server removes the configured headers before routing. + +Generate `shared_secret` with a cryptographically secure random generator, +encode it as hex or base64url, store the same value only in the front door and +Trusted Server configuration, and never commit it. The value is redacted from +configuration debug output. Configuration requires at least 32 ASCII graphic +bytes (`!` through `~`) with no whitespace, controls, DEL, or non-ASCII bytes, +and startup fails when the value is still the documented placeholder. + +Redaction protects debug output and validation errors; it does not move the +value into a platform secret store. `ts config push` serializes the value in the +Trusted Server application-config blob, so restrict access to that configuration +store. Every adapter removes the configured IP and authentication headers before +routing, although only Fastly uses them for client-IP resolution. + +**Environment Overrides**: + +```bash +TRUSTED_SERVER__TRUSTED_CLIENT_IP__IP_HEADER=fastly-client-ip +TRUSTED_SERVER__TRUSTED_CLIENT_IP__AUTH_HEADER=x-ts-client-ip-auth +TRUSTED_SERVER__TRUSTED_CLIENT_IP__SHARED_SECRET=replace-with-a-random-shared-secret +``` + +Because the typed environment overlay cannot create a missing section, add +`[trusted_client_ip]` and all three fields to the TOML before using these +overrides. + ## Tester Cookie Configuration Settings for the optional tester-cookie endpoints. This feature is disabled by @@ -1429,7 +1498,7 @@ remove that field's non-default value (and any environment override), run `ts config validate`, push the resulting default-compatible blob, and only then roll back the binary. -**Environment overlays:** EdgeZero v0.0.4 overlays cannot create missing TOML +**Environment overlays:** EdgeZero's env overlays cannot create missing TOML leaves. Existing configs must add **both** leaves under `[auction]` (`rewrite_creatives` and `sanitize_creatives`) before `TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES` / @@ -1484,8 +1553,37 @@ Defines the ad slots the trusted server offers on a page: which pages each slot appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad unit it maps to (`gam_unit_path`). +`enabled` is the dedicated server-side ad-template switch. It defaults to `true` +for compatibility with existing configurations. Set it to `false` to stop +publisher HTML and SPA page-bids template delivery while retaining the slot +configuration and direct `POST /auction` endpoint. + +#### Publisher document cache policy + +For a successful GET publisher document, Trusted Server applies the +browser-only `Cache-Control: private, max-age=60` policy from +[#1007](https://github.com/IABTechLab/trusted-server/issues/1007) when the +server-side ad stack is structurally inactive. Trusted Server also applies this +policy to a subsequent `304 Not Modified` response so revalidation cannot +restore the origin freshness policy. This includes an absent +`[creative_opportunities]` section, `enabled = false`, no slot matching the +path, or a disabled auction. The `private` directive prevents shared caches +that use `Cache-Control` from storing the document. The policy replaces the +origin browser cache policy except when the origin sends `private` or +`no-store`, which are preserved. Bot, prefetch, and consent-denied requests +also retain the origin policy because they can produce a request-specific +representation for the same URL. Error responses and non-document requests +retain the origin policy. + +Trusted Server leaves origin validators and CDN-specific cache headers +unchanged. Those headers continue to control supporting CDNs independently of +the browser-only policy. If a response using the generated inactive-stack +policy later carries `Set-Cookie`, cookie privacy finalization replaces it with +`Cache-Control: private, max-age=0` and removes the CDN-specific cache headers. + ```toml [creative_opportunities] +enabled = true # set to false to disable server-side ad templates gam_network_id = "123456789" price_granularity = "dense" @@ -1504,6 +1602,16 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` +The same switch can be overridden through the typed CLI environment overlay. +Because EdgeZero only replaces TOML leaves that already exist, first add +`enabled = true` to the `[creative_opportunities]` block in the base config +before using this override. See [Environment Variable Overrides (Typed +CLI)](#environment-variable-overrides-typed-cli) for the general overlay rules. + +```bash +TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false +``` + ### Shared template assembly (`assembly_mode = "esi"`) This configuration is an experimental validation spike scoped to @@ -1685,8 +1793,9 @@ publisher-specific. Startup fails if `{section}` is used without a valid `section_root`. Startup rejects a blank `gam_network_id` only when an absent path/default or a `{network_id}` template consumes it; static paths and templates without `{network_id}` do not consume it. A -`[creative_opportunities]` block with no slots is disabled, so its -`gam_network_id` is not checked. +`[creative_opportunities]` block with `enabled = false` or no slots is +inactive, so no publisher templates are delivered and its `gam_network_id` is +not checked when no slot uses it. Both knobs are config-driven, so the URL→section convention stays with the publisher: `section_segment` selects which segment names the section, and @@ -1903,7 +2012,7 @@ trusted-server.dev.toml # Development overrides **Environment Variables Not Applied**: - Run the override through `ts config validate`, `ts config diff`, or `ts config push` -- Verify the target leaf already exists in `trusted-server.toml`; EdgeZero v0.0.4 does not create missing fields +- Verify the target leaf already exists in `trusted-server.toml`; the env overlay does not create missing fields - Verify prefix: `TRUSTED_SERVER__` - Check separator: `__` (double underscore) - Confirm the variable is exported: `echo $VARIABLE_NAME` diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index 20faf1995..6a37a263a 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -72,6 +72,84 @@ When you're ready to use your own domain: - Fastly Compute **only accepts client traffic via TLS** (HTTPS) - Origins and backends can be non-TLS if needed +## CDN-fronted Client IP + +When another CDN or Fastly service fronts Trusted Server, the Fastly Compute +client address identifies the immediate edge node rather than the original +reader. Trusted Server can instead consume an authenticated reader-IP header, +but only after the public front door is configured to overwrite both the IP and +authentication headers on every backend request. Preserving values supplied by +the browser is unsafe because a caller could choose the IP used for geolocation +and other request processing. + +For a dedicated VCL-to-Compute [service chain](https://www.fastly.com/documentation/guides/getting-started/services/service-chaining/), +where every request from the VCL service goes to Trusted Server, overwrite +[`Fastly-Client-IP`](https://www.fastly.com/documentation/reference/http/http-headers/Fastly-Client-IP/) +from the initial [`client.ip`](https://www.fastly.com/documentation/reference/vcl/variables/client-connection/client-ip/) +and set a dedicated authentication header in the fronting service. Read the +secret from a private (write-only) edge dictionary instead of placing it in the +VCL source. For example: + +```vcl +sub vcl_recv { + if (fastly.ff.visits_this_service == 0 && req.restarts == 0) { + unset req.http.Fastly-Client-IP; + unset req.http.X-TS-Client-IP-Auth; + + set req.http.Fastly-Client-IP = client.ip; + set req.http.X-TS-Client-IP-Auth = + table.lookup(ts_private_config, "trusted_client_ip_secret"); + } +} +``` + +In this example, attach an edge dictionary named `ts_private_config` to the +fronting VCL service and store the shared secret under the key +`trusted_client_ip_secret`. If the service also routes to other backends, wrap +the Trusted Server header setup in the same host or path condition that selects +Trusted Server. Strip any client-supplied authentication header on the other +routes, and do not send the dictionary value to unrelated backends. + +The `unset` before each `set` matters. Trusted Server ignores a forwarded +address whenever either trust header carries more than one value, so a client +that sends its own copy of either header could otherwise force the fallback and +keep its real address out of geolocation and bot protection. + +Use a cryptographically random secret of at least 32 ASCII graphic bytes in +production, encoded as hex or base64url with no whitespace. Keep the fronting +copy in a private edge dictionary rather than inlining it in VCL, where it is +readable by anyone with service-configuration access and preserved in every +version diff. Configure the identical header names and secret in Trusted Server: + +```toml +[trusted_client_ip] +ip_header = "fastly-client-ip" +auth_header = "x-ts-client-ip-auth" +shared_secret = "replace-with-a-random-shared-secret" +``` + +The `shared_secret` value above is an intentionally invalid placeholder. Replace +it with the exact value stored in the fronting service's edge dictionary. + +`Fastly-Client-IP` is not protected from modification when it first enters +Fastly, which is why overwriting it and authenticating the handoff are both +required. Trusted Server removes both trust headers before routing. Direct +requests and requests with missing, invalid, or duplicated trust headers remain +available and use the immediate peer address instead. + +The VCL example assumes that the reader connects directly to the fronting +Fastly service. If another CDN is in front, `client.ip` identifies that CDN's +node instead. In that topology, restrict direct access to the Fastly front door, +derive the IP header from the upstream CDN's protected reader-IP value, and +still overwrite both trust headers before the request enters Trusted Server. + +::: warning No-code request routing limitation +Fastly no-code request routing does not provide a point to inject these headers. +If that routing path does not preserve the original reader IP, Trusted Server +cannot recover it with this mechanism. Use a fronting service that can overwrite +both headers before forwarding the request. +::: + ## Create Config and Secret Stores For features like request signing, you'll need to create Fastly stores: diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9314f983b..29a32f8be 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -127,7 +127,7 @@ ts config init To bootstrap from a public publisher page, run an audit first: ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` The audit command writes `js-assets.toml` plus a draft `trusted-server.toml`. diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 658f5d5bc..00cf94c44 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -36,6 +36,8 @@ debug = false # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" allow_script_creatives = false +# Default. Set publisher_native only for the controlled friendly-frame experiment below. +rendering_mode = "trusted_server" [auction] enabled = true @@ -49,6 +51,28 @@ timeout_ms = 2000 `allow_script_creatives` defaults to `false`. While disabled, APS script bids are rejected before per-impression reduction, floors, mediation, and winner selection. Enable it only for a controlled cohort after the browser-security checks in [Rollout](#rollout) pass. +`rendering_mode` is a strict enum: `trusted_server` (the default) retains the opaque static renderer route, and `publisher_native` disables that route and adds `data-ts-aps-rendering-mode="publisher_native"` to the server-generated TSJS bundle tag. TSJS captures this server-owned attribute when the bundle executes, so markup added later cannot change the mode. The attribute works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. + +### Publisher-native runner experiment + +`publisher_native` is an opt-in browser experiment, **not** general APS compatibility proof. No public `apstag` API was found that accepts an externally selected OpenRTB `aaxResponse`. In controlled browser testing, `apstag.renderImp(document, bidId)` did not render the Trusted Server bid because that bid was absent from the SDK's browser-auction state. Trusted Server therefore does not call `apstag`, `fetchBids`, or `setDisplayBids`, mutate the publisher's APS SDK, or start a second auction. Instead, this mode reuses the same `prebid/creative/render` runner contract already used by `trusted_server` mode, but inside a publisher-origin frame; that observed vendor contract still requires APS account-team validation. + +No publisher JavaScript change is required. After validating and freezing the exact selected descriptor, Trusted Server JS: + +1. resolves the direct-auction slot or its injected GAM div mapping; +2. creates a hidden, publisher-origin friendly iframe sized to the winner; +3. initializes only that fresh frame's account-scoped `_aps` event queue; +4. queues `prebid/creative/render` with the selected `aaxResponse` and bid ID; and +5. loads the fixed `https://client.aps.amazon-adsystem.com/prebid-creative.js` runner. + +The existing publisher content remains visible until the runner script loads. A runner error, a blocked script, a missing slot, a superseding dispatch, or a load taking longer than 10 seconds removes the pending frame and visibly declines the bid. It never falls back to `/integrations/aps/renderer` or sends a Universal Creative renderer response. Trusted Server treats runner load as successful handoff; the runner owns subsequent creative completion and resource loading. + +Unlike `trusted_server` mode, this friendly frame deliberately has no opaque-origin sandbox. The fixed APS runner and its creative execute with the behavior of a publisher-origin integration, so `publisher_native` has a larger security surface—especially when `allow_script_creatives = true`. Use only a controlled cohort, and ensure publisher CSP permits the APS runner and required creative resources. + +For a client-side Prebid APS capability, Trusted Server consumes the one-shot capability before starting the runner and calls `markWinningBidAsUsed` only after the runner loads. For server/GPT ownership, it similarly claims the slot/ad ID first. This prevents native and Trusted Server rendering from both owning the same response. + +Disable or coordinate existing publisher-native APS demand for every `publisher_native` cohort. Otherwise the publisher's normal APS auction and this server-selected bid can duplicate demand. Validate the exact account, inventory, CSP, iframe/script creative behavior, impression reporting, and click-through behavior with the APS account team before any production rollout. + Set `inventory_domain` and `inventory_page_origin` together only when the public deployment hostname differs from the inventory identity authorized by APS. The domain becomes `site.domain`. The HTTPS page origin replaces the current page's scheme and host while preserving its path; query and fragment data are removed before forwarding. The origin must be the inventory domain or one of its subdomains and cannot include credentials, a port, path, query, or fragment. These values come only from operator configuration; Trusted Server never accepts APS inventory identity from the client auction payload. APS uses ordinary auction slot IDs and banner formats. Legacy creative-opportunity APS `slot_id` configuration is accepted for compatibility but ignored, and `bidders.aps.slotID` is not required. Remove both during migration. @@ -150,7 +174,7 @@ Trusted Server does not insert APS creative markup into the publisher document. Seats, `impid`, markup, notifications, user-sync data, sibling bids, losing seats, and unknown fields are not exposed. The browser decodes this envelope and cross-checks the ID, dimensions, URL, and tag type before any DOM mutation or message suppression. -Both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. +In `trusted_server` mode, both rendering paths use `GET /integrations/aps/renderer`, a static Trusted Server document with its own restrictive CSP. The document initializes the account-keyed APS queue and then loads only the fixed runner at `https://client.aps.amazon-adsystem.com/prebid-creative.js`. The outer iframe uses these sandbox permissions: @@ -167,11 +191,13 @@ It deliberately omits `allow-same-origin`, so APS and bidder execution remains b ### Direct `/auction` -The TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. +In `trusted_server` mode, the TSJS auction client validates the typed renderer descriptor, creates the opaque renderer iframe, and sends the minimized envelope after the frame loads. In `publisher_native` mode it creates the injected friendly iframe and queues the response for the fixed APS Prebid creative runner. Ordinary non-APS `adm` continues through the existing sanitizer and generic creative iframe. ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid`, validates the complete envelope, and returns a static dynamic-renderer program that creates the same opaque renderer iframe. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid` and validates the complete envelope. In `trusted_server` mode it returns a static dynamic-renderer program that creates the same opaque renderer iframe. In `publisher_native` mode it instead resolves the publisher div and starts the friendly-frame runner without sending a Universal Creative renderer response. + +After the native runner loads, Trusted Server replaces the existing children of the resolved publisher div with the friendly frame. This removes the GAM or Universal Creative iframe when it is inside that div. If the runner fails, the existing iframe remains, but its Universal Creative request receives no response because Trusted Server has already claimed the selected bid. This one-owner behavior avoids a second render path, but GAM impression and viewability reporting must be validated with the APS account team for the controlled cohort. For client-side `trustedServer` adapter auctions, Prebid generates its own `hb_adid`. Trusted Server binds that generated ID to the validated APS descriptor in a bounded, expiring browser registry before GAM refresh. The bridge verifies that the requesting Universal Creative iframe belongs to the same ad unit, consumes the capability once, and passes the APS bid ID separately to the Amazon runner. @@ -209,17 +235,20 @@ This release is a direct protocol cutover: There is no legacy runtime switch. Roll back by disabling `[integrations.aps]`, restoring native APS for the cohort, or deploying the prior binary. +Changing `rendering_mode` does not update pages that are already loaded or stored in an HTML cache. A cached `trusted_server` page can continue requesting `/integrations/aps/renderer` after a native-mode deployment removes that route. A cached `publisher_native` page continues using its captured native mode after rollback. Coordinate the mode change with HTML cache expiry or purge and reload active test sessions before judging the result. + ## Rollout Use fictional values in source-controlled configuration and fixtures. Supply controlled account details out of band. 1. Obtain APS account-team confirmation for edge-originated OpenRTB traffic. 2. Enable Trusted Server APS only for an isolated cohort and disable native APS demand there. -3. Keep `allow_script_creatives = false` and observe iframe bids through direct and GAM paths. +3. Keep the default `trusted_server` mode and `allow_script_creatives = false`; observe iframe bids through direct and GAM paths. 4. Confirm outbound privacy fields, aggregate diagnostics, decoded-price competition, line-item targeting, dimensions, click-throughs, and opaque-origin isolation. -5. Run the restrictive-CSP browser proof for script behavior. -6. Only then enable script creatives for the isolated cohort and validate them in a real browser. -7. Expand traffic only after APS confirmation and successful controlled validation. +5. In a still-smaller cohort, set `rendering_mode = "publisher_native"` and confirm the fixed runner request, friendly-frame dimensions, iframe creatives, impression reporting, and click-throughs without a request to `/integrations/aps/renderer`. +6. Purge or expire cached HTML and reload active test sessions when changing modes. Confirm the publisher CSP permits the runner but does not need to permit inline Trusted Server scripts. +7. Only after reviewing the friendly-frame security tradeoff, enable script creatives for the isolated native cohort and validate them in a real browser. +8. Expand traffic only after APS confirmation and successful controlled validation. ## Troubleshooting @@ -235,12 +264,12 @@ Use fictional values in source-controlled configuration and fixtures. Supply con ### Winner targets but does not render -- Confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`. -- Confirm publisher CSP permits `frame-src 'self'`. +- In `trusted_server` mode, confirm `GET /integrations/aps/renderer` returns HTML with its CSP and `Referrer-Policy: no-referrer`, and that publisher CSP permits `frame-src 'self'`. +- In `publisher_native` mode, confirm the mode `` marker is present, the slot receives a hidden friendly iframe, and `https://client.aps.amazon-adsystem.com/prebid-creative.js` is not blocked by CSP. The static renderer route is intentionally absent in this mode. - Confirm the GAM creative uses the supported Prebid Universal Creative bridge and the winning `hb_adid`. - For client-side `trustedServer` adapter auctions, confirm Prebid's `bidResponse` contains a generated `adId` and that the corresponding capability appears briefly in `window.tsjs.apsPrebidRenderers` before rendering. -- Ensure no native APS path is trying to handle the same cohort. -- Keep script creatives disabled while diagnosing iframe rendering. +- Ensure no publisher APS auction is trying to handle the same cohort. +- Keep script creatives disabled while diagnosing either rendering mode. ## Verification diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 0c2d8ae6c..87b3083eb 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -68,30 +68,30 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| ### Configuration options -| Option | Type | Default | Description | -| -------------------------------------- | ------- | -------------------------------- | ----------------------------------------------------------------------- | -| `enabled` | boolean | `false` | Enable the DataDome integration | -| `sdk_origin` | string | `https://js.datadome.co` | DataDome SDK origin URL for `tags.js` | -| `api_origin` | string | `https://api-js.datadome.co` | DataDome signal collection API origin URL for `/js/*` | -| `cache_ttl_seconds` | integer | `3600` | Cache TTL for `tags.js` | -| `rewrite_sdk` | boolean | `true` | Rewrite DataDome script URLs in HTML to first-party paths | -| `enable_protection` | boolean | `false` | Call the Protection API before route matching | -| `server_side_key_secret_store` | string | `ts_secrets` | Runtime secret store containing the DataDome server-side key | -| `server_side_key_secret_name` | string | `datadome_server_side_key` | Secret name containing the DataDome server-side key | -| `protection_api_origin` | string | `https://api-fastly.datadome.co` | Protection API origin | -| `timeout_ms` | integer | `1500` | Dynamic backend first-byte timeout for Protection API calls | -| `protection_excluded_methods` | array | `["OPTIONS"]` | HTTP methods skipped before the Protection API call | -| `protection_excluded_asns` | array | `[]` | Client autonomous system numbers skipped before the Protection API call | -| `protection_excluded_ip_cidrs` | array | `[]` | Inline client IP CIDR ranges skipped before the Protection API call | -| `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | -| `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | -| `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | -| `protection_test_bypass` | object | omitted | Staging-only fixed-header bypass; secret must contain at least 32 bytes | -| `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | -| `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | -| `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | -| `client_side_tag_url` | string | `/integrations/datadome/tags.js` | Root-relative or HTTPS script URL used by auto-injection | -| `client_side_configuration` | object | `{ ajaxListenerPath = true }` | Options assigned to `window.ddoptions` | +| Option | Type | Default | Description | +| -------------------------------------- | ------- | -------------------------------- | ------------------------------------------------------------------------------ | +| `enabled` | boolean | `false` | Enable the DataDome integration | +| `sdk_origin` | string | `https://js.datadome.co` | DataDome SDK origin URL for `tags.js` | +| `api_origin` | string | `https://api-js.datadome.co` | DataDome signal collection API origin URL for `/js/*` | +| `cache_ttl_seconds` | integer | `3600` | Cache TTL for `tags.js` | +| `rewrite_sdk` | boolean | `true` | Rewrite DataDome script URLs in HTML to first-party paths | +| `enable_protection` | boolean | `false` | Call the Protection API before route matching | +| `server_side_key_secret_store` | string | `ts_secrets` | Runtime secret store containing the DataDome server-side key | +| `server_side_key_secret_name` | string | `datadome_server_side_key` | Secret name containing the DataDome server-side key | +| `protection_api_origin` | string | `https://api-fastly.datadome.co` | Protection API origin | +| `timeout_ms` | integer | `1500` | Dynamic backend first-byte timeout for Protection API calls | +| `protection_excluded_methods` | array | `["OPTIONS"]` | HTTP methods skipped before the Protection API call | +| `protection_excluded_asns` | array | `[]` | Client autonomous system numbers skipped before the Protection API call | +| `protection_excluded_ip_cidrs` | array | `[]` | Inline client IP CIDR ranges skipped before the Protection API call | +| `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | +| `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | +| `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | +| `protection_test_bypass` | object | omitted | Configuration-gated fixed-header bypass; secret must contain at least 32 bytes | +| `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | +| `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | +| `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | +| `client_side_tag_url` | string | `/integrations/datadome/tags.js` | Root-relative or HTTPS script URL used by auto-injection | +| `client_side_configuration` | object | `{ ajaxListenerPath = true }` | Options assigned to `window.ddoptions` | ## Client-side setup @@ -169,20 +169,19 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. -8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. +8. The request does not contain a matching enabled `protection_test_bypass` credential. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. -### Staging test bypass +### Test bypass For short-lived browser automation on an access-controlled staging site, you can configure a static header credential that skips only the server-side Protection API: ```toml -# Runtime activation also requires FASTLY_IS_STAGING=1. [integrations.datadome.protection_test_bypass] enabled = true credential_secret_store = "ts_secrets" @@ -190,25 +189,20 @@ credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. -`FASTLY_IS_STAGING` is supplied at runtime by Fastly (`1` in staging and `0` in -production); it is not compiled into or promoted with the Wasm artifact. Verify -staging through the `X-TS-ENV: staging` response signal and the integration -activation log, and verify production omits that response signal. A retained -section cannot bypass protection in a production or other non-staging runtime. -Store a randomly generated credential containing at least 32 bytes of -high-entropy material in the configured Secret Store, configure this section -only while needed, protect the site with an outer access control such as Basic -Auth, and remove the section when testing finishes. +when omitted or when its `enabled` field is false. Store a randomly generated +credential containing at least 32 bytes of high-entropy material in the +configured Secret Store, configure this section only while needed, and protect +the site with an outer access control such as Basic Auth. Remove the section +when testing finishes. Whenever the enabled DataDome request filter runs on the Fastly adapter, the fixed `x-ts-datadome-bypass` header is removed before configuration or credential checks. It therefore cannot reach DataDome or the publisher origin -through that path when the bypass is absent, disabled, inactive, or invalid. -Active credentials are compared in constant time and never logged. Duplicate -header values fail closed. Scope the header to the staging origin; do not attach -it to every request in a browser context because that can disclose the -credential to third-party origins. With Playwright: +through that path when the bypass is absent, disabled, or invalid. Active +credentials are compared in constant time and never logged. Duplicate header +values fail closed. Scope the header to the test origin; do not attach it to +every request in a browser context because that can disclose the credential to +third-party origins. With Playwright: ```ts await context.route('https://staging.example.com/**', async (route) => { @@ -233,7 +227,7 @@ This behavior applies to: - `protection_excluded_ip_cidr_sources`; - structured `ip_cidr` rules; - structured `ip_cidr_source` rules; and -- a matching enabled `protection_test_bypass` credential in a staging runtime. +- a matching enabled `protection_test_bypass` credential. Method, ASN, path, query-parameter, static-asset, and internal-route exclusions alone do not suppress the client-side tag. However, a simultaneous matching IP diff --git a/docs/package-lock.json b/docs/package-lock.json index e31c9db5e..e6003067a 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -14,7 +14,7 @@ "eslint": "^10", "mermaid": "^11.12.3", "prettier": "^3.4.2", - "typescript-eslint": "8.57", + "typescript-eslint": "8.66", "vitepress": "^1.5.0", "vitepress-plugin-mermaid": "^2.0.17" } @@ -180,7 +180,6 @@ "integrity": "sha512-Jc360x4yqb3eEg4OY4KEIdGePBxZogivKI+OGIU8aLXgAYPTECvzeOBc90312yHA1hr3AeRlAFl0rIc8lQaIrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@algolia/client-common": "5.50.0", "@algolia/requester-browser-xhr": "5.50.0", @@ -1886,7 +1885,6 @@ "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1914,20 +1912,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.2.tgz", - "integrity": "sha512-NZZgp0Fm2IkD+La5PR81sd+g+8oS6JwJje+aRWsDocxHkjyRw0J5L5ZTlN3LI1LlOcGL7ph3eaIUmTXMIjLk0w==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.66.0.tgz", + "integrity": "sha512-p088eaGrzYz1s+7cov0aMOCkNGTJlVxF4jgubf28c8L0Cv9Rloj8YBHnv4hXLq6IIEE1AsjNWavO+k+8kP2Y0A==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/type-utils": "8.57.2", - "@typescript-eslint/utils": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/type-utils": "8.66.0", + "@typescript-eslint/utils": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1937,15 +1935,15 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.57.2", + "@typescript-eslint/parser": "^8.66.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", "dev": true, "license": "MIT", "engines": { @@ -1953,17 +1951,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.2.tgz", - "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.66.0.tgz", + "integrity": "sha512-X6ypGChaWYk6PBtUg2BwuTZEFFcHJAtGTVJ9/lCTOufhZ4i9fNolQNnktq+kkMCwMj7V8Svsq7+TxSDslmhE0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3" }, "engines": { @@ -1975,18 +1972,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.2.tgz", - "integrity": "sha512-FuH0wipFywXRTHf+bTTjNyuNQQsQC3qh/dYzaM4I4W0jrCqjCVuUh99+xd9KamUfmCGPvbO8NDngo/vsnNVqgw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.66.0.tgz", + "integrity": "sha512-7MthGPTt4BP69lSryqpqq8HQqxuzynssckL/jyDyk3+TNMQ3y2jFWkptCrktWvBrP+EH787Nl5N5Qpw7WZg+5g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.57.2", - "@typescript-eslint/types": "^8.57.2", + "@typescript-eslint/tsconfig-utils": "^8.66.0", + "@typescript-eslint/types": "^8.66.0", "debug": "^4.4.3" }, "engines": { @@ -1997,18 +1994,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.2.tgz", - "integrity": "sha512-snZKH+W4WbWkrBqj4gUNRIGb/jipDW3qMqVJ4C9rzdFc+wLwruxk+2a5D+uoFcKPAqyqEnSb4l2ULuZf95eSkw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.66.0.tgz", + "integrity": "sha512-8TGcH25j9zqJ/IULB/ppyhRvxA8QYfFEZ7nfbg6/BN9spDgb8fPWQXlE5l8TWBL50EtUx007uZ1o9VOwrq2/9g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2" + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2019,9 +2016,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.2.tgz", - "integrity": "sha512-3Lm5DSM+DCowsUOJC+YqHHnKEfFh5CoGkj5Z31NQSNF4l5wdOwqGn99wmwN/LImhfY3KJnmordBq/4+VDe2eKw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.66.0.tgz", + "integrity": "sha512-9D5gLYZG4rOjcoag8MQ/fWI8WqA9wcPDyOGyWtWFhvM1lHRbliqUSPIY5J3zqCU1tvSwzXxnnjhQhz5Ne7mJ4g==", "dev": true, "license": "MIT", "engines": { @@ -2032,21 +2029,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.2.tgz", - "integrity": "sha512-Co6ZCShm6kIbAM/s+oYVpKFfW7LBc6FXoPXjTRQ449PPNBY8U0KZXuevz5IFuuUj2H9ss40atTaf9dlGLzbWZg==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.66.0.tgz", + "integrity": "sha512-LG2dWfjZQQp0ADtAu/EWJVayefGL2UEZ3CDeI44D9v3rXB/WYUqE/jpO28KrEKul5AySrmI+Zh1v6v+xW2U9+g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/utils": "8.66.0", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2057,13 +2054,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.2.tgz", - "integrity": "sha512-/iZM6FnM4tnx9csuTxspMW4BOSegshwX5oBDznJ7S4WggL7Vczz5d2W11ecc4vRrQMQHXRSxzrCsyG5EsPPTbA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.66.0.tgz", + "integrity": "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ==", "dev": true, "license": "MIT", "engines": { @@ -2075,21 +2072,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.2.tgz", - "integrity": "sha512-2MKM+I6g8tJxfSmFKOnHv2t8Sk3T6rF20A1Puk0svLK+uVapDZB/4pfAeB7nE83uAZrU6OxW+HmOd5wHVdXwXA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.66.0.tgz", + "integrity": "sha512-8/x4INiiQb10jGgXYD7116/zQ+OL84ZIFn0za68wwFHCanT/VLbBEroWht8RV8fn0/ZCAoazHLQgwUC0UQcDfg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.57.2", - "@typescript-eslint/tsconfig-utils": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/project-service": "8.66.0", + "@typescript-eslint/tsconfig-utils": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2099,20 +2096,20 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.2.tgz", - "integrity": "sha512-krRIbvPK1ju1WBKIefiX+bngPs+odIQUtR7kymzPfo1POVw3jlF+nLkmexdSSd4UCbDcQn+wMBATOOmpBbqgKg==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.66.0.tgz", + "integrity": "sha512-jasearZPolBw5NJNYGMwxzHMF83niVWmMU1VdHzG1CyfI2VS7f7nZltnKtHcg20hW+7Uo5GfK4MeDPoU3qI8EA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2" + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2123,17 +2120,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.2.tgz", - "integrity": "sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.66.0.tgz", + "integrity": "sha512-dkKR8q+lKciskj1Y3vthHktl+3cMLWGyVUP23bRiPZ5O9BRT++4EqDDV+TVeIKBL1VXVEqrJlz8MYbcnvJcAlg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", + "@typescript-eslint/types": "8.66.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -2433,7 +2430,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2474,7 +2470,6 @@ "integrity": "sha512-yE5I83Q2s8euVou8Y3feXK08wyZInJWLYXgWO6Xti9jBUEZAGUahyeQ7wSZWkifLWVnQVKEz5RAmBlXG5nqxog==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@algolia/abtesting": "1.16.0", "@algolia/client-abtesting": "5.50.0", @@ -2567,7 +2562,6 @@ "integrity": "sha512-opLQzEVriiH1uUQ4Kctsd49bRoFDXGGSC4GUqj7pGyxM3RehRhvTlZJc1FL/Flew2p5uwxa1tUDWKzI4wNM8pg==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@chevrotain/cst-dts-gen": "11.1.2", "@chevrotain/gast": "11.1.2", @@ -2672,7 +2666,6 @@ "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10" } @@ -3107,7 +3100,6 @@ "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -3363,7 +3355,6 @@ "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -3613,7 +3604,6 @@ "integrity": "sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "tabbable": "^6.4.0" } @@ -3979,7 +3969,6 @@ "integrity": "sha512-fEnci+Immw6lKMFI8sqzjlATTyjLkRa6axrEgLV2yHTfv8r+h1wjFbV6xeRtd4rUV1cS4EpR9rwp3Rci7TRWDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", @@ -4300,12 +4289,11 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4552,9 +4540,9 @@ "peer": true }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { @@ -4688,14 +4676,14 @@ } }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -4767,16 +4755,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.57.2.tgz", - "integrity": "sha512-VEPQ0iPgWO/sBaZOU1xo4nuNdODVOajPnTIbog2GKYr31nIlZ0fWPoCQgGfF3ETyBl1vn63F/p50Um9Z4J8O8A==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.66.0.tgz", + "integrity": "sha512-QlEbBPz/RuJ1XUHj29nm3t0F/O/cSlEnntozqPOYHnnTGAXFamnMBu5i9Vn6vhUPHGAjR+Vl+5J8vPN/BMUrJw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.57.2", - "@typescript-eslint/parser": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2" + "@typescript-eslint/eslint-plugin": "8.66.0", + "@typescript-eslint/parser": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/utils": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4787,7 +4775,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/ufo": { @@ -4937,7 +4925,6 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -4998,7 +4985,6 @@ "integrity": "sha512-+2ym1/+0VVrbhNyRoFFesVvBvHAVMZMK0rw60E3X/5349M1GuVdKeazuksqopEdvkKwKGs21Q729jX81/bkBJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@docsearch/css": "3.8.2", "@docsearch/js": "3.8.2", @@ -5110,7 +5096,6 @@ "integrity": "sha512-iV/sU9SzOlmA/0tygSmjkEN6Jbs3nPoIPFhCMLD2STrjgOU8DX7ZtzMhg4ahVwf5Rp9KoFzcXeB1ZrVbLBp5/Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.31", "@vue/compiler-sfc": "3.5.31", diff --git a/docs/package.json b/docs/package.json index f74991a4c..df1bd66d0 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,7 +20,7 @@ "eslint": "^10", "mermaid": "^11.12.3", "prettier": "^3.4.2", - "typescript-eslint": "8.57", + "typescript-eslint": "8.66", "vitepress": "^1.5.0", "vitepress-plugin-mermaid": "^2.0.17" } diff --git a/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md new file mode 100644 index 000000000..df781e518 --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md @@ -0,0 +1,2142 @@ +# Server-Side Ad Template CLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the unified `ts` CLI support for server-side ad-template static diagnostics and browser-backed verification described in `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. + +**Architecture:** Keep the CLI host-only and thin: Clap parsing stays in `run.rs` / command adapter modules, shared app-config loading moves to `app_config.rs`, pure ad-template logic lives under `ad_templates/`, and Chrome/Chromium collection lives under `audit/`. Runtime gate rules are extracted into a small pure helper in `trusted-server-core` so the CLI does not duplicate server behavior. + +**Tech Stack:** Rust 2024 workspace, host-target `trusted-server-cli`, `clap`, EdgeZero typed app-config loader, `serde`/`serde_json` for stable JSON, `chromiumoxide` for browser-backed audit collection, local HTML fixture tests, and existing `trusted-server-core::creative_opportunities` matching. + +--- + +## Current State + +- Branch: `feature/ts-cli-ad-templates`. +- Static ad-template commands already exist in `crates/trusted-server-cli/src/config_ad_templates.rs`. +- The current branch does not contain #800 audit files. Port useful #800 pieces into the current #799 code shape; do not resurrect stale `args.rs` or `config_command.rs`. +- The spec was updated after review and is the source of truth: + `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. +- Keep `.env` and operator-owned `trusted-server.toml` out of commits. + +## File Map + +### New files + +| File | Responsibility | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-cli/src/app_config.rs` | Shared effective app-config loader and shared `AppConfigArgs`. | +| `crates/trusted-server-cli/src/ad_templates/mod.rs` | Re-export focused ad-template CLI modules. | +| `crates/trusted-server-cli/src/ad_templates/expected.rs` | Path/URL normalization and expected-slot projection from runtime slot matching. | +| `crates/trusted-server-cli/src/ad_templates/compare.rs` | Pure DOM/GPT/APS evidence comparison, statuses, warnings, runtime gate output, strict failure decisions. | +| `crates/trusted-server-cli/src/ad_templates/output.rs` | Human and JSON rendering for static diagnostics and browser verification. | +| `crates/trusted-server-cli/src/audit/mod.rs` | Audit namespace entry point. | +| `crates/trusted-server-cli/src/audit/page.rs` | Generic page audit command ported from #800. | +| `crates/trusted-server-cli/src/audit/collector.rs` | Browser collector trait plus collected page/evidence structs. | +| `crates/trusted-server-cli/src/audit/browser.rs` | Chromiumoxide-backed browser collector, init scripts, optional scroll, page-level collection errors. | +| `crates/trusted-server-cli/src/audit/ad_templates.rs` | `ts audit ad-templates verify` orchestration. | +| `crates/trusted-server-cli/src/audit/ad_template_collector.js` | Read-only init script for GPT/APS/DOM evidence collection, included via `include_str!`. | + +### Modified files + +| File | Change summary | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `Cargo.toml` | Add workspace dependencies missing from this branch: `chromiumoxide`, `serde`, and `serde_json` if not present. | +| `crates/trusted-server-cli/Cargo.toml` | Add host-only CLI dependencies for browser audit and JSON output. | +| `crates/trusted-server-cli/src/lib.rs` | Register new `app_config`, `ad_templates`, and `audit` modules under `cfg(not(target_arch = "wasm32"))`. | +| `crates/trusted-server-cli/src/run.rs` | Add `Audit` command namespace, parser tests, and dispatch. | +| `crates/trusted-server-cli/src/config_ad_templates.rs` | Shrink to Clap adapter using shared loader/expected/output modules. | +| `crates/trusted-server-core/src/creative_opportunities.rs` | Add pure runtime gate helper types/functions shared by runtime and CLI. | +| `crates/trusted-server-core/src/publisher.rs` | Route existing server-side ad-stack gate through the shared helper without changing behavior. | + +## Implementation Rules + +- Use TDD for each task: write a failing test first, run it, implement the minimal code, re-run, then commit. +- Commit after each task using repo style: sentence case, imperative, no semantic prefix. +- Keep `trusted-server-cli` host-only. Do not introduce `tokio`, `chromiumoxide`, or filesystem/browser dependencies into core runtime or wasm adapter crates. +- Do not write real publisher domains or secrets in tests. Use `example.com`, `publisher.example`, and fictional IDs only. +- Prefer pure module tests over browser tests. Browser-backed fixture tests should use local HTML only and no GPT/APS network. + +## Task 0: Baseline And Branch Hygiene + +**Files:** + +- Verify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` +- Verify: `docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md` + +- [ ] **Step 1: Confirm branch and working tree** + + Run: + + ```bash + git status --short --branch + git log --oneline --decorate -5 + ``` + + Expected: on `feature/ts-cli-ad-templates`; no unrelated modified files besides the approved spec/plan docs. + +- [ ] **Step 2: Run docs format check before code work** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: `All matched files use Prettier code style!` + +- [ ] **Step 3: Commit reviewed spec and plan** + + Run: + + ```bash + git add docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md + git commit -m "Add server-side ad-template CLI implementation plan" + ``` + + Expected: docs-only commit. If the spec commit already exists separately, commit only the plan. + +## Task 1: Share Runtime Ad-Stack Gate Logic + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Write failing core tests for the shared gate helper** + + Add tests near the existing `creative_opportunities` tests: + + ```rust + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert!(result.blocking_gates().is_empty()); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!(result.blocking_gates().contains(&AdStackGateName::AuctionEnabled)); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u8..64 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 1 == 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + ``` + +- [ ] **Step 2: Run the focused test and verify it fails** + + Run: + + ```bash + # NOTE: trusted-server-core links the `fastly` crate and CANNOT build for the host + # triple — run core tests on the DEFAULT target (wasm32-wasip1 + viceroy runner), + # i.e. no `--target`. Only the host-only `trusted-server-cli` uses `--target `. + cargo test -p trusted-server-core creative_opportunities::tests::ad_stack_gate + ``` + + Expected: compile failure because `AdStackGateInput` / `evaluate_ad_stack_gate` do not exist. + +- [ ] **Step 3: Implement pure gate types and helper** + + Add public, serde-free types to `creative_opportunities.rs`: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum RuntimeAdStackExpected { + Yes, + No, + Unknown, + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum AdStackGateName { + MethodGet, + Navigation, + NotPrefetch, + NotBot, + MatchedSlots, + ConsentAllowsAuction, + AuctionEnabled, + } + + #[derive(Debug, Clone, Copy)] + pub struct AdStackGateInput { + pub method_get: bool, + pub navigation: bool, + pub prefetch: bool, + pub bot: bool, + pub matched_slots: bool, + pub consent_allows_auction: Option, + pub auction_enabled: bool, + } + + #[derive(Debug, Clone, Eq, PartialEq)] + pub struct AdStackGateResult { + pub expected: RuntimeAdStackExpected, + blocking_gates: Vec, + } + + impl AdStackGateResult { + pub fn blocking_gates(&self) -> &[AdStackGateName] { + &self.blocking_gates + } + } + ``` + + Implement `evaluate_ad_stack_gate(input)` so any known blocking boolean gate returns `No`, all known pass plus `Some(true)` consent returns `Yes`, and all known pass plus `None` consent returns `Unknown`. + + Mind the gate polarity, mirroring `should_run_server_side_ad_stack`: `method_get`, + `navigation`, `matched_slots`, and `auction_enabled` block when **false**, while + `prefetch` and `bot` block when **true** (their gate names `NotPrefetch` / `NotBot` + pass when the input bool is false). `consent_allows_auction` is the only tri-state + input: `Some(false)` blocks (No), `Some(true)` passes, `None` yields Unknown only + when no other gate already blocks. + +- [ ] **Step 4: Route `publisher.rs` through the helper** + + Replace the body of `should_run_server_side_ad_stack` with a call to `evaluate_ad_stack_gate`, preserving the existing function signature for low-risk runtime compatibility: + + ```rust + crate::creative_opportunities::evaluate_ad_stack_gate( + crate::creative_opportunities::AdStackGateInput { + method_get: is_get, + navigation: is_navigation, + prefetch: is_prefetch, + bot: is_bot, + matched_slots: has_matched_slots, + consent_allows_auction: Some(consent_allows_auction), + auction_enabled, + }, + ) + .expected + == crate::creative_opportunities::RuntimeAdStackExpected::Yes + ``` + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + # Core tests run on the default wasm target via viceroy (no --target). + cargo test -p trusted-server-core publisher::tests + cargo test -p trusted-server-core creative_opportunities + ``` + + Expected: all focused tests pass (including the existing `should_run_server_side_ad_stack` truth-table tests in `publisher::tests`). + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs + git commit -m "Share server-side ad stack gate evaluation" + ``` + +## Task 2: Extract Shared CLI App Config Loader + +**Files:** + +- Create: `crates/trusted-server-cli/src/app_config.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing loader tests** + + Move the existing temp-project helpers from `config_ad_templates.rs` tests into `app_config.rs` tests and add: + + ```rust + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `app_config` module is not registered. + +- [ ] **Step 3: Implement `app_config.rs`** + + Move these items out of `config_ad_templates.rs`: + - `AppConfigArgs` + - `LoadedSettings` + - `load_settings` + - `resolve_app_config_path` + + Make the API explicit: + + ```rust + #[derive(Clone, Debug, Args)] + pub struct AppConfigArgs { + #[arg(long)] + pub app_config: Option, + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + #[arg(long)] + pub no_env: bool, + } + + pub struct LoadedSettings { + pub app_config_path: PathBuf, + pub settings: Settings, + } + + pub fn load_settings(args: &AppConfigArgs) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = !args.no_env; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) + } + + fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, + ) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } + } + ``` + + Include the same top-level imports currently used by these helpers: + `std::path::{Path, PathBuf}`, `clap::Args`, + `edgezero_core::app_config::{self, AppConfigLoadOptions}`, + `edgezero_core::manifest::ManifestLoader`, + `trusted_server_core::config::TrustedServerAppConfig`, and + `trusted_server_core::settings::Settings`. + +- [ ] **Step 4: Register module and update imports** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod app_config; + ``` + + In `config_ad_templates.rs`, import: + + ```rust + use crate::app_config::{load_settings, AppConfigArgs}; + ``` + +- [ ] **Step 5: Run focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: existing static command behavior remains unchanged. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/app_config.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Extract shared CLI app config loader" + ``` + +## Task 3: Add Expected-Slot Model + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/mod.rs` +- Create: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing expected-slot tests** + + Add a test-only dependency to `crates/trusted-server-cli/Cargo.toml` so tests can + deserialize core slot config instead of constructing `CreativeOpportunitySlot` with + its `pub(crate)` `compiled_patterns` cache: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] + toml = { workspace = true } + ``` + + In `expected.rs`, add tests for path normalization, full URL normalization, config-order preservation, resolved div ID, resolved GAM unit path, provider names, and matching page patterns: + + ```rust + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + r#" + gam_network_id = "123" + auction_timeout_ms = 500 + price_granularity = "dense" + + [[slot]] + id = "atf" + gam_unit_path = "/123/news/atf" + div_id = "ad-atf-" + page_patterns = [{page_patterns}] + formats = [{{ width = 300, height = 250 }}] + floor_price = 0.50 + targeting = {{ zone = "atf" }} + + [slot.providers.prebid] + bidders = {{}} + "# + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(["/news/*", "/"].as_slice()); + let expected = expected_slots_for_path("/news/story", &config) + .expect("should build expected slots"); + + assert_eq!(expected.path, "/news/story"); + assert_eq!(expected.slots.iter().map(|slot| slot.id.as_str()).collect::>(), ["atf"]); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!(expected.slots[0].gam_unit_path, "/123/news/atf"); + assert_eq!(expected.slots[0].providers, ["prebid"]); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!(normalize_path_or_url("https://www.example.com/news/story?x=1#top").expect("should normalize"), "/news/story"); + assert_eq!(normalize_path_or_url("news/story?x=1").expect("should normalize"), "/news/story"); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because module/types do not exist. + +- [ ] **Step 3: Implement expected-slot structs** + + Define pure structs that own strings and are stable for output: + + ```rust + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlots { + pub path: String, + pub slots: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlot { + pub id: String, + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + pub page_patterns: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + // Mirrors `MediaType` rendered as a stable string (`"banner"`, `"video"`, `"native"`). + pub media_type: String, + } + ``` + + `div_id` and `gam_unit_path` are resolved (non-optional) strings. The core + `CreativeOpportunitySlot` stores `div_id` / `gam_unit_path` as `Option` + and the GAM unit path is composed with the configured GAM network ID; mirror the + existing `format_slot` resolution in `config_ad_templates.rs` so the CLI does not + invent a second resolution rule. Use + `trusted_server_core::creative_opportunities::match_slots`. Do not compile globs in CLI. + +- [ ] **Step 4: Register `ad_templates` and update static commands** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod ad_templates; + ``` + + Rewire `config_ad_templates.rs` onto the shared module, and remove the now-duplicated + local code so there is no name collision or dead `normalize_path_or_url`: + - delete the private `fn normalize_path_or_url` (currently `config_ad_templates.rs:448`) + and add `use crate::ad_templates::expected::{expected_slots_for_path, normalize_path_or_url};`; + - the existing `config_ad_templates::tests::normalizes_path_or_url_like_runtime_request_path` + test (currently `:661`) calls the local fn via `super::*` — either delete it (Task 3 + Step 1 already adds normalization tests in `expected.rs`) or repoint it at + `crate::ad_templates::expected::normalize_path_or_url`. Pick one so the test crate + still compiles at this commit. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/expected.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Add shared ad-template expected slot model" + ``` + +## Task 4: Add Stable Output And JSON Types + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Add CLI JSON dependencies** + + The CLI crate has **no plain `[dependencies]` table** — every runtime dep lives + under `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` (the workspace + default build target is `wasm32-wasip1` per `.cargo/config.toml`). Add the new deps + to that existing table; do **not** create a `[dependencies]` table, or they compile + for wasm and leak host-only crates into the wasm build: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/etc ... + serde = { workspace = true } + serde_json = { workspace = true } + ``` + + Add workspace dependency `chromiumoxide = "0.9.1"` in `Cargo.toml` only in Task 7 when browser code is introduced. + +- [ ] **Step 2: Write failing JSON output tests** + + In `output.rs`, add tests that construct an in-memory verification result and assert exact JSON values: + + ```rust + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + } + + // Pins the spec §8 navigation_failed shape: error present, runtime/gates/ + // matched_slot_count keys ABSENT (skipped), final_url/path null. + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!(page.get("runtime_ad_stack_expected").is_none(), "runtime field absent on error page"); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!(page.get("matched_slot_count").is_none(), "matched_slot_count absent on error page"); + assert_eq!(value["ok"], false); + } + ``` + +- [ ] **Step 3: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because output model does not exist. + +- [ ] **Step 4: Implement serializable output types** + + Model the **entire** `--json` wire tree from spec §8 (this is the single source of + truth for field names and ordering). Use owned `String` / `Vec` fields and + `#[serde(rename_all = "snake_case")]` so output is stable. Leaf enums: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum SlotStatus { Confirmed, Partial, Missing } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum RuntimeAdStackExpectedJson { Yes, No, Unknown } + + impl From for RuntimeAdStackExpectedJson { + fn from(value: trusted_server_core::creative_opportunities::RuntimeAdStackExpected) -> Self { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected as Core; + match value { + Core::Yes => Self::Yes, + Core::No => Self::No, + Core::Unknown => Self::Unknown, + } + } + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum GateState { Pass, Fail, Unknown } + ``` + + Top-level tree (field names and nesting must match §8 exactly): + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct VerificationReport { + pub ok: bool, + pub strict: bool, + pub pages: Vec, + pub warnings: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct PageJson { + pub url: String, + pub final_url: Option, + pub requested_path: String, + pub path: Option, + // Field ORDER matters: serde serializes in declaration order. Spec §8 places + // `error` immediately after `path` on the navigation_failed shape, so it must + // be declared here (not last). On normal pages `error` is None and skipped, so + // the runtime/gates/slots run in §8 order; on error pages the runtime/gates/ + // matched_slot_count are None and skipped, leaving url..path, error, slots, + // extra_evidence, warnings — exactly the §8 error shape. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + pub slots: Vec, + pub extra_evidence: Vec, + pub warnings: Vec, + } + + // One field per gate name from spec §5.2 / §8, each a GateState. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct Gates { + pub method_get: GateState, + pub navigation: GateState, + pub not_prefetch: GateState, + pub not_bot: GateState, + pub matched_slots: GateState, + pub auction_enabled: GateState, + pub consent_allows_auction: GateState, + } + + // Serialize for output JSON; Deserialize because the browser collector payload + // (Task 8) carries warning objects decoded into `BrowserAdEvidence.warnings`. + #[derive(Debug, Clone, Eq, PartialEq, Serialize, serde::Deserialize)] + pub struct Warning { + pub code: String, + pub message: String, + } + ``` + + Define the remaining nested JSON structs **explicitly** — do not serialize the + compare-module types directly. The compare types (`SlotResult`, `SlotEvidence`, + `GptSlotEvidence`, `ExtraEvidence`) carry a `phase` field and are not `Serialize`; + spec §8's `evidence.gpt` has **no** `phase` key and `configured` excludes `id` + and `page_patterns`. Mismatched reuse would emit extra keys. Wire structs: + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotJson { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhaseJson, + pub configured: ConfiguredJson, + pub evidence: SlotEvidenceJson, + pub warnings: Vec, + } + + // §8 `configured`: div_id, gam_unit_path, formats, providers — NO id/page_patterns. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ConfiguredJson { + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct FormatJson { + pub width: u32, + pub height: u32, + pub media_type: String, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotEvidenceJson { + pub dom_id: Option, + pub gpt: Option, + } + + // §8 `evidence.gpt`: gam_unit_path, div_id, sizes — NO phase. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct GptEvidenceJson { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<[u32; 2]>, + } + + #[derive(Debug, Clone, Copy, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhaseJson { InitialLoad, Scroll } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ExtraEvidenceJson { + pub kind: String, + pub phase: EvidencePhaseJson, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<[u32; 2]>, + pub reason: String, + } + ``` + + Note `sizes` serialize as `[[300,250]]` (arrays of two ints), matching §8 — use + `[u32; 2]` here even though the compare module uses `(u32, u32)` tuples; the + Task 9 assembly maps tuple → `[w, h]`. The conversion from the compare + `SlotResult`/`SlotEvidence`/`ExtraEvidence` to these JSON types (dropping `phase` + from `gpt`, dropping `id`/`page_patterns` from `configured`) lives in Task 9 Step 7. + `Warning` is the single warning type for the whole CLI; defined here and re-exported + from `ad_templates::mod` so `compare.rs` reuses it (plain data, not JSON logic). + Keep `example_confirmed_with_extra_evidence()` and similar fixtures behind + `#[cfg(test)]`. + +- [ ] **Step 5: Add verification human-render helpers** + + Add only the **browser-verification** page summary writers here (used by + `audit::ad_templates` in Task 9), writing to `&mut dyn Write`; no `println!` / + `eprintln!`. Do **not** add static match/check/explain writers in this task — + those are the existing `write_match_result`/`format_slot`/etc. functions that + Task 6 Step 3 **moves** out of `config_ad_templates.rs`. Keeping the static + relocation solely in Task 6 avoids two competing copies of the same helpers in + `output.rs`. The verification writers added here may be unused until Task 9 (a + warn-level `dead_code` lint that does not fail `cargo test`); add + `#[allow(dead_code)]` if clippy is run between Task 4 and Task 9. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 7: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Add ad-template CLI output models" + ``` + +## Task 5: Add Pure Evidence Comparison + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Write failing comparison tests** + + Cover every spec status and warning case without launching Chrome. Define small + test constructors so tests do not couple to the full `BrowserAdEvidence` field + list (`page_bids` and `warnings` default empty, evidence items default to + `EvidencePhase::InitialLoad`): + + ```rust + fn dom(id: &str) -> DomEvidence { + DomEvidence { dom_id: id.to_string(), phase: EvidencePhase::InitialLoad } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { slot_id: slot_id.to_string(), sizes: sizes.to_vec(), phase: EvidencePhase::InitialLoad } + } + + // Non-banner format helper for the unsupported-format test. + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: vec![ExpectedFormat { width: 0, height: 0, media_type: "video".to_string() }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + fn evidence(doms: Vec, gpts: Vec, aps: Vec) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "GPT path+div+size overlap should confirm"); + assert!(result.slots[0].warnings.is_empty(), "confirmed slot should carry no warnings"); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only evidence should be partial"); + assert!( + result.slots[0].warnings.iter().any(|w| w.code == "dom_without_gpt"), + "DOM-only slot should warn dom_without_gpt" + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Missing, "no DOM/GPT evidence should be missing"); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot("header", "ad-header-0-", "/123/homepage/header", &[(728, 90)], &[]); + // First candidate ends with `-container` and must be skipped; the framework-suffixed ID resolves. + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].evidence.dom_id.as_deref(), Some("ad-header-0-_R_abc123"), "prefix match should skip -container"); + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only prefix match is partial without GPT"); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot("/123/publisher/right-rail", "ad-right-rail-0", &[(300, 250)]), + ], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "matched slot still confirms"); + assert_eq!(result.extra_evidence.len(), 1, "unmatched GPT slot becomes extra evidence"); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!(!result.strict_failed(), "extra evidence alone must not fail strict"); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::auction_disabled()); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No, "auction disabled should set No"); + assert_eq!(result.slots[0].status, SlotStatus::Missing, "static status is still reported"); + assert!(!result.strict_failed(), "missing slot must not fail strict when ad stack expected is No"); + } + + // §5.4: GPT path+div match but no numeric size overlap -> partial + warning. + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "no size overlap should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "incompatible_sizes")); + } + + // §5.4/§5.6: matched slot with only non-banner formats -> partial + unsupported_format. + #[test] + fn non_banner_only_slot_is_partial() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "non-banner-only should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "unsupported_format")); + } + + // §5.4: GPT element ID may be `${resolved_dom_id}-container` and still confirm. + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot("/123/news/atf", "ad-atf-0-container", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "container element id is a valid GPT div match"); + } + + // §5.4: out-of-page GPT slot is partial (so it fails strict) plus a warning. + #[test] + fn out_of_page_gpt_slot_warns_and_is_partial() { + let expected = expected_slot("interstitial", "ad-oop-", "/123/news/oop", &[(300, 250)], &[]); + // gpt_slot with empty sizes models an out-of-page slot (no numeric sizes). + let evidence = evidence(vec![dom("ad-oop-0")], vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "a sizeless slot against banner formats is partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "out_of_page_slot")); + } + + // §5.5: matching APS fetchBids -> no provider warning. + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!(!result.slots[0].warnings.iter().any(|w| w.code.starts_with("aps_")), "matching APS should not warn"); + } + + // §5.5: configured aps provider but no APS evidence -> provider warning, still confirmed, strict not failed. + #[test] + fn aps_missing_warns_but_keeps_confirmed() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "missing APS does not flip status"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "aps_evidence_missing")); + assert!(!result.strict_failed(), "provider warning alone must not fail strict"); + } + ``` + + Add a `#[cfg(test)]` constructor in `compare.rs` tests that builds a real + `ExpectedSlot` (the Task 3 type) so comparison tests stay readable: + + ```rust + fn expected_slot(id: &str, div_id: &str, gam_unit_path: &str, sizes: &[(u32, u32)], providers: &[&str]) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: sizes.iter().map(|&(width, height)| ExpectedFormat { width, height, media_type: "banner".to_string() }).collect(), + providers: providers.iter().map(|p| p.to_string()).collect(), + page_patterns: Vec::new(), + } + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because comparison module does not exist. + +- [ ] **Step 3: Implement browser evidence structs** + + Define the minimum collector-independent input shape plus the comparison result + shape the tests assert against: + + All browser-evidence input structs derive `Debug, Clone` and `serde::Deserialize` + (Task 8 decodes them from the collector's `window.__tsAdTemplateEvidence` JSON); + `EvidencePhase` deserializes from `"initial_load"` / `"scroll"`. The comparison- + result structs derive `Debug` (so the Step 1 `assert_eq!`/`matches!` assertions + compile) and `Clone`. Sizes are `(u32, u32)` tuples internally; deserialize them + from JSON `[w, h]` arrays. + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhase { + InitialLoad, + Scroll, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct DomEvidence { + pub dom_id: String, + pub phase: EvidencePhase, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct GptSlotEvidence { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // APS `apstag.fetchBids` evidence (spec §5.5): configured slot ID + observed sizes. + #[derive(Debug, Clone, serde::Deserialize)] + pub struct ApsFetchBidsEvidence { + pub slot_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // DEFERRED in this implementation: `/__ts/page-bids` SPA observation (spec §5.2 + // "when available"). The struct/field are forward scaffolding so the collector and + // JSON can grow it later; Task 8 does NOT populate it and Task 4 JSON does NOT + // surface it in Phase 1. Tracked as a deferred item in Risks. Keep the field so + // `BrowserAdEvidence` deserialization stays forward-compatible (default empty). + #[derive(Debug, Clone, serde::Deserialize)] + pub struct PageBidsEvidence { + pub slot_id: String, + pub phase: EvidencePhase, + } + + // `Warning` is the shared CLI warning type defined in Task 4 (`output.rs`) and + // re-exported from `ad_templates::mod`. It is plain data reused here (not JSON + // logic). Because the collector payload carries warnings, give `Warning` BOTH + // `Serialize` (Task 4 output) and `Deserialize` (Task 8 decode) derives. + use crate::ad_templates::output::Warning; + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct BrowserAdEvidence { + pub dom_ids: Vec, + pub gpt_slots: Vec, + pub aps_calls: Vec, + #[serde(default)] + pub page_bids: Vec, + #[serde(default)] + pub warnings: Vec, + } + + // Comparison output. Uses the core `RuntimeAdStackExpected` enum from Task 1 so + // pure comparison logic does not depend on the output/JSON module. Task 4's + // `RuntimeAdStackExpectedJson` is produced only at serialization time. + #[derive(Debug, Clone)] + pub struct PageVerificationResult { + pub runtime_ad_stack_expected: trusted_server_core::creative_opportunities::RuntimeAdStackExpected, + pub slots: Vec, + pub extra_evidence: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotResult { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhase, + pub evidence: SlotEvidence, + pub warnings: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotEvidence { + pub dom_id: Option, + pub gpt: Option, + } + + #[derive(Debug, Clone)] + pub struct ExtraEvidence { + pub kind: String, + pub phase: EvidencePhase, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<(u32, u32)>, + pub reason: String, + } + ``` + + `RuntimeGateSummary` is the third argument to `compare_page_evidence`; it wraps + the core gate result. Provide `RuntimeGateSummary::unknown_allowed()` (expected + `Unknown`) and `RuntimeGateSummary::auction_disabled()` (expected `No`) test + constructors so comparison tests do not rebuild gate inputs by hand. + +- [ ] **Step 4: Implement DOM/GPT/APS rules** + + Status rules: + - DOM exact ID first, then first prefix match, **excluding `-container`** wrappers + (slot-root resolution, spec §5.3). + - GPT confirms when: GAM unit path matches, the GPT slot element ID equals the + resolved DOM ID **or** an existing `${resolved_dom_id}-container` element + (spec §5.4 — note this is the GPT element-ID match, distinct from the §5.3 DOM + root resolution that skips `-container`), and at least one numeric banner size + overlaps. + - GPT path/div match with no numeric size overlap → `partial` (warn `incompatible_sizes`). + - Matched slot whose configured formats are **all non-banner** (video/native) → + `partial` (warn `unsupported_format`); banner is the only Phase-1 confirmable type. + - DOM-only (no GPT) → `partial` (warn `dom_without_gpt`). + - No DOM and no GPT → `missing`. + + Size-compatibility warnings (spec §5.4 — all are warnings, none flip a confirmed + slot to fail): emit a `Warning` for each of: + - `fluid_size_ignored` — non-numeric observed sizes like `"fluid"` ignored for matching; + - `extra_observed_size` — observed GPT sizes not in the configured set; + - `configured_size_not_observed` — configured sizes never observed (when ≥1 was); + - `out_of_page_slot` — out-of-page GPT slot with no sizes observed; the slot is + reported `partial`, which fails `--strict`. + + Provider + extra evidence: + - APS: configured `providers.aps.slot_id` with matching `fetchBids` → no warning; + missing/ambiguous APS evidence → provider warning only (`aps_evidence_missing` / + `aps_evidence_ambiguous`), never flips status or fails `--strict` in Phase 1. + - Unmatched live DOM/GPT/APS evidence → structured `extra_evidence` (never fails strict). + + Define each warning `code` as a stable string constant so output and tests share them. + +- [ ] **Step 5: Implement strict decision method** + + Add an inherent method on the result so tests can call `result.strict_failed()`: + + ```rust + impl PageVerificationResult { + pub fn strict_failed(&self) -> bool { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } + } + ``` + + - false when `runtime_ad_stack_expected == No`; + - true for any `missing` or `partial` slot when expected is `Yes` or `Unknown`; + - false for provider warnings and extra evidence alone (they are not slot statuses). + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Add pure ad-template evidence comparison" + ``` + +## Task 6: Refactor Static Commands Onto Shared Modules + +**Files:** + +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add characterization tests before refactor** + + These guard behavior across the Step 3 move, so each must assert **exact output + substrings** (capture the command's `Vec`/`String` output and + `assert!(out.contains("..."))`), not just run without panicking — a bare + smoke test cannot catch a wording regression. Mirror the existing assertion style + at `config_ad_templates.rs:570-657`. Pin, with concrete expected strings: + - `lint` not configured → e.g. `"creative_opportunities: not configured"`; + - `lint` with slots + auction disabled → slot count line + `"auction: disabled"`; + - `match --details` → slot div ID, GAM unit path, formats, providers lines; + - `check --expect-no-slots` → success message; + - `check` failure with missing and unexpected slots → the exact failure lines; + - `explain` → each gate line (including `"auction providers configured"`) and the + EdgeZero legacy-fallback warning text. + + Run the existing tests first and copy the real emitted strings so the + characterization assertions match current behavior exactly before refactoring. + +- [ ] **Step 2: Run tests before refactor** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: characterization tests pass against the current implementation. + +- [ ] **Step 3: Move formatting into `ad_templates::output`** + + Move these helpers out of `config_ad_templates.rs`: + - `write_match_result` + - `write_gate` + - `format_slot` + - `format_format` + - `format_providers` + - `join_set` + - `plural` + + Keep command functions small: parse args, load config, call expected/gate logic, render. + +- [ ] **Step 4: Reuse shared gate helper in `explain`** + + Build `AdStackGateInput` from explain flags and config: + + ```rust + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get, + navigation: !args.non_navigation, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: !expected.slots.is_empty(), + consent_allows_auction: Some(!args.consent_denied), + auction_enabled: loaded.settings.auction.enabled, + }); + ``` + + Render the seven shared gate names from `gate` rather than hand-rolled boolean chains. + + **Preserve the explain-only provider gate.** The current `run_explain` + (`config_ad_templates.rs:270`) renders an eighth gate, + `"auction providers configured"` (`!loaded.settings.auction.providers.is_empty()`, + line 299), and ANDs it into its local `runs_ad_stack` decision (line 302). The + shared `evaluate_ad_stack_gate` helper and runtime `should_run_server_side_ad_stack` + intentionally have no provider-configured gate. Do not fold this into + `AdStackGateInput`. Keep `"auction providers configured"` as an explain-only + supplementary `write_gate(...)` line rendered alongside the shared result, and + keep it in `explain`'s own `runs_ad_stack` decision: + + ```rust + let providers_configured = !loaded.settings.auction.providers.is_empty(); + render_shared_gates(out, &gate)?; + write_gate(out, "auction providers configured", providers_configured)?; + let runs_ad_stack = + gate.expected == RuntimeAdStackExpected::Yes && providers_configured; + ``` + + This keeps `explain` output and behavior identical to the current implementation + (verified by the Step 1 characterization test) while still sharing the seven core + runtime gates with `publisher.rs`. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: no output regressions except intentional wording updates covered by tests. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/ad_templates/output.rs crates/trusted-server-cli/src/run.rs + git commit -m "Refactor static ad-template commands" + ``` + +## Task 7: Port Generic Audit Namespace And Browser Collector + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Create: `crates/trusted-server-cli/src/audit/mod.rs` +- Create: `crates/trusted-server-cli/src/audit/page.rs` +- Create: `crates/trusted-server-cli/src/audit/collector.rs` +- Create: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add browser dependencies** + + Add `chromiumoxide` to the root `[workspace.dependencies]` table (inert until a + crate references it via `{ workspace = true }`): + + ```toml + [workspace.dependencies] + # ... existing entries ... + chromiumoxide = "0.9.1" + ``` + + Add the host deps to the CLI crate under its existing + `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` table — NOT a plain + `[dependencies]` table (workspace default target is wasm32; an unconditional dep + compiles for wasm and breaks the build / leaks host-only crates): + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/serde/etc ... + chromiumoxide = { workspace = true } + futures = { workspace = true } + tempfile = { workspace = true } + tokio = { workspace = true } + which = { workspace = true } + ``` + + Verify the root workspace already provides `futures`, `tempfile`, `tokio`, `which` + (it does on this branch); only `chromiumoxide` is a new workspace entry. + +- [ ] **Step 2: Write failing audit parser tests** + + In `run.rs` tests: + + ```rust + #[test] + fn audit_legacy_url_parses_as_page_alias() { + let args = parse(&["ts", "audit", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_page_subcommand_parses() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&["ts", "audit", "ad-templates", "verify", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_is_not_legacy_url() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + ``` + +- [ ] **Step 3: Run parser tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `Audit` command does not exist. + +- [ ] **Step 4: Implement audit Clap namespace in current `run.rs` shape** + + Do not add stale #800 `args.rs`. Add a `Command::Audit(AuditArgs)` variant to the + existing `Command` enum, plus the full audit arg surface. The parser tests in + Step 2 exercise `ad-templates verify`, so the **entire** command surface (including + the verify args) must be defined here for those tests to compile. Task 9 implements + the verify _behavior_ only — it does not redefine these arg types. + + **Visibility:** `audit::run_audit` lives in `audit/mod.rs` and must name these + types in its signature and match their variants, so every audit arg type and its + fields are `pub(crate)` (not private). `PageAuditArgs` (from `audit/page.rs`, Task 7 + Step 5) and `AuditAdTemplatesVerifyArgs` are likewise `pub(crate)`/`pub`. `run.rs` + imports `AuditArgs` for the `Command::Audit(AuditArgs)` variant; everything else is + read by `audit/mod.rs`. (This mirrors `config_ad_templates::AdTemplatesCommand`, + which is `pub` and consumed by `run.rs`.) + + ```rust + // value parser shared by legacy_url and verify urls; rejects non-HTTP(S) schemes. + pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!("unsupported URL scheme `{other}` (expected http or https)")), + } + } + + #[derive(Debug, clap::Args)] + pub(crate) struct AuditArgs { + #[command(subcommand)] + pub(crate) command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + pub(crate) legacy_url: Option, + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditAdTemplatesCommand { + Verify(AuditAdTemplatesVerifyArgs), + } + + // Defined here (not Task 9) so parser tests compile. Task 9 fills in the handler. + #[derive(Debug, clap::Args)] + pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + #[arg(long)] + pub strict: bool, + #[arg(long)] + pub json: bool, + #[arg(long)] + pub scroll: bool, + } + ``` + + Dispatch `Command::Audit(args)` to a single `audit::run_audit(args: AuditArgs)` + entry point (in `audit/mod.rs`) that normalizes the namespace: `legacy_url` (if + present) and `Page` both route to the generic page audit; `AdTemplates(Verify(..))` + routes to the verifier (a stub returning `Ok(())` until Task 9). If Clap cannot make + the optional-subcommand-plus-hidden-positional contract unambiguous, implement a + small `AuditArgs::normalize()` that rejects `legacy_url` values that are not HTTP(S). + Decide arg-type home consistently: keep them in `run.rs` as `pub(crate)` (as shown) + and import into `audit/mod.rs`, or move them next to `run_audit` in `audit/mod.rs` + and import `AuditArgs` into `run.rs` — either works, but do not split them. + +- [ ] **Step 5: Port minimal generic page audit** + + Port useful #800 concepts into `audit/page.rs`, but keep output read-only by default for now: + - parse/validate URL; + - call `AuditCollector::collect_page`; + - print summary with final URL, title, script/resource counts, warnings; + - no draft config generation in this PR unless #800 rebase keeps it explicitly. + +- [ ] **Step 6: Implement collector trait and browser collector base** + + `audit/collector.rs` — define the trait plus its concrete request/response types so + Task 9's `FakeCollector` and the verify orchestration have a contract to assert on: + + ```rust + pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + } + + pub struct BrowserCollectRequest { + pub url: url::Url, + // Pre-navigation init scripts (evaluate-on-new-document). Empty for plain page audit; + // Task 8 passes the ad-template collector script here. + pub init_scripts: Vec, + pub scroll: bool, + } + + pub struct CollectedPage { + pub final_url: url::Url, + pub title: String, + // Generic page-audit signals (counts only; no page HTML/cookies/storage). + pub script_count: usize, + pub resource_count: usize, + pub warnings: Vec, + // Present only when an ad-template init script was injected (Task 8); None for + // plain `ts audit page`. This is how `BrowserAdEvidence` rides on a CollectedPage. + pub ad_evidence: Option, + } + ``` + + `BrowserCollectRequest` carries `init_scripts` + `scroll` so ad-template verification + enables evidence hooks without changing the trait later. + + `audit/browser.rs` should port #800's: + - `which` browser lookup; + - isolated `TempDir` profile; + - current-thread Tokio runtime; + - `Browser::launch`; + - `page.goto`; + - `wait_for_navigation_response`; + - settle loop. + +- [ ] **Step 7: Run compile-focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: parser and non-browser unit tests pass. No test should require installed Chrome yet. + +- [ ] **Step 8: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/audit crates/trusted-server-cli/src/lib.rs crates/trusted-server-cli/src/run.rs + git commit -m "Add audit namespace and browser collector base" + ``` + +## Task 8: Add Browser Ad-Template Evidence Collector + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` + +- [ ] **Step 1: Write JS collector contract fixture tests** + + Add Rust unit tests that inspect generated init-script text and decode a mocked `window.__tsAdTemplateEvidence` JSON payload. These should not launch Chrome. + + Prefer **behavioral** assertions over brittle substring matching: where possible, + assert by decoding a mocked `window.__tsAdTemplateEvidence` payload into + `BrowserAdEvidence` and checking fields. For the few structural checks that must + inspect the script text, pin **exact** marker substrings (no "or equivalent", so + the pass condition is deterministic) — choose the markers to match the strings the + implementation will actually emit: + - `build_ad_template_init_script` output contains the literal `__TS_CONFIG` injection; + - contains the chosen googletag-hook marker (pick ONE and pin it, e.g. + `Object.defineProperty(window, "googletag"`); + - contains the `cmd.push` wrap marker; + - contains the `defineSlot` record marker; + - contains the `apstag.fetchBids` wrap marker; + - embeds only the configured div prefixes / provider IDs passed via `__TS_CONFIG` + (assert a non-configured prefix is absent). + +- [ ] **Step 2: Run tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: failure because collector script/builder does not exist. + +- [ ] **Step 3: Implement init script builder** + + In Rust, build script as: + + ```rust + pub fn build_ad_template_init_script(config: &AdTemplateCollectorConfig) -> Result { + let config_json = serde_json::to_string(config) + .map_err(|error| format!("failed to serialize ad-template collector config: {error}"))?; + Ok(format!(";(() => {{ const __TS_CONFIG = {config_json};\n{}\n}})();", include_str!("ad_template_collector.js"))) + } + ``` + + Keep the JS file generic; pass configured prefixes and APS slot IDs through `__TS_CONFIG`. + +- [ ] **Step 4: Implement read-only JS evidence collection** + + In `ad_template_collector.js`, write to `window.__tsAdTemplateEvidence`: + - `dom_ids`: matched IDs from configured prefixes, excluding `-container`; + - `gpt_slots`: record `defineSlot` calls observed **both** directly **and** when + dispatched from the `googletag.cmd` queue (wrap `cmd.push` so queued callbacks are + instrumented without changing their order — spec §7), **plus** a post-settle + `googletag.pubads().getSlots()` scrape. For each scraped slot capture + `getAdUnitPath()`, `getSlotElementId()`, and `getSizes()` so `getSlots()`-only + slots still carry numeric `sizes` for the §5.4 overlap rule. Normalize sizes from + both `defineSlot` input and `getSizes()` output: `[300,250]` → one `(300,250)`; + `[[300,250],[728,90]]` → two pairs; non-numeric (`"fluid"`) dropped from numeric + sizes and surfaced as a `fluid_size_ignored` warning; + - `aps_calls`: `fetchBids` payloads (configured slot IDs + sizes); + - `warnings`: collector warnings only ({code, message}), no page HTML/cookies/storage. + + Always call original page functions with unchanged arguments, and never override + `navigator.webdriver` (spec §7). + +- [ ] **Step 5: Add browser collector extraction** + + After settle and after optional scroll, evaluate: + + ```javascript + ;() => window.__tsAdTemplateEvidence || null + ``` + + Decode into `BrowserAdEvidence`. If decode fails, return a page warning rather than failing navigation. + +- [ ] **Step 6: Add deterministic scroll** + + In `audit/browser.rs`, implement `scroll` by evaluating: + + ```javascript + ;async () => { + const height = Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight + ) + for (const y of [ + Math.floor(height * 0.33), + Math.floor(height * 0.66), + height, + ]) { + window.scrollTo(0, y) + await new Promise((resolve) => setTimeout(resolve, 250)) + } + window.scrollTo(0, 0) + } + ``` + + Then wait for the same settle quiet period and collect evidence with `phase = "scroll"` where the JS script marks new observations. + +- [ ] **Step 7: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli audit::browser --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: unit tests pass without launching Chrome. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_template_collector.js crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/collector.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Collect browser ad-template evidence" + ``` + +## Task 9: Implement `ts audit ad-templates verify` + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` + +- [ ] **Step 1: Write failing orchestration tests with a fake collector** + + Build a fake collector implementing `AuditCollector` and test: + - one confirmed page exits success in default mode; + - strict missing slot returns error; + - `[auction].enabled = false` returns runtime skipped and does not strict-fail missing evidence; + - one page navigation error plus one success sets JSON `ok = false`; + - invalid `ftp://` URL fails before fake collector is called; + - redirect uses final path for expected slots and emits redirect warning. + + Define the test scaffolding explicitly (no dangling helpers): + + ```rust + // Maps each requested URL to a canned outcome so orchestration is tested without Chrome. + struct FakeCollector { + pages: std::collections::HashMap>, + } + + impl FakeCollector { + // Success page: requested -> final_url, carrying the given ad evidence. + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = std::collections::HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("valid final url"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { pages } + } + // Helper to add a failing page for multi-URL tests. + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages.insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + } + + impl BrowserAdEvidence { + // #[cfg(test)] fixture: one confirmed news slot (atf / ad-atf-0 / /123/news/atf, 300x250). + fn confirmed_news_slot() -> Self { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { dom_id: "ad-atf-0".into(), phase: EvidencePhase::InitialLoad }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".into(), + div_id: "ad-atf-0".into(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + } + + // Runs the verify orchestration with `--json` over `urls` and returns parsed JSON. + // Loads a #[cfg(test)] effective config whose `/news/*` slot is the atf slot above. + fn run_verify_json(collector: &dyn AuditCollector, urls: impl IntoIterator) -> serde_json::Value { /* impl in test module */ } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + BrowserAdEvidence::confirmed_news_slot(), + ); + let json = run_verify_json(&collector, ["https://www.example.com/"]); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + // Warning order is unspecified; assert presence, not index 0. + let warnings = json["pages"][0]["warnings"].as_array().expect("warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + ``` + + `run_verify_json` calls the same `run_verify` entry point used in production but + with the fake collector injected and output captured; define it in the test module + so all six listed cases share it. + +- [ ] **Step 2: Run focused tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because verifier module does not exist. + +- [ ] **Step 3: Wire the verifier handler** + + `AuditAdTemplatesVerifyArgs` already exists from Task 7, Step 4. Replace the Task 7 + stub so `audit::run_audit` routes `AdTemplates(Verify(args))` into a new + `audit::ad_templates::run_verify(args)`. Do not redefine the arg struct. + +- [ ] **Step 4: Implement verification orchestration** + + For each URL: + 1. Collect browser page with ad-template init script and optional scroll. + 2. Parse final URL and normalize final path. + 3. Build expected slots for final path. + 4. Build gate summary using shared core gate helper with `consent_allows_auction = None`. + 5. Add redirect warning (`code = "redirected"`) if requested path differs from final path. + 6. Compare evidence (`compare_page_evidence`) to get a `PageVerificationResult`. + 7. **Assemble the wire `PageJson`** (Task 4 type) from the pieces the comparison + result does not carry: `url` / `final_url` / `requested_path` / `path`, + `gates` (map the gate summary's per-gate states to `GateState`), + `matched_slot_count`, `runtime_ad_stack_expected` (via the `From` impl on + `RuntimeAdStackExpectedJson`), then the `slots` / `extra_evidence` / `warnings` + from the comparison result. `PageVerificationResult` is intentionally URL- and + gate-agnostic; this step is where per-page request context is joined in. + 8. Preserve page-level errors as a `PageJson` with `error: Some(..)` and continue + remaining URLs. + +- [ ] **Step 5: Implement exit behavior** + - Default auditor-assist mode: return `Ok(())` for missing/partial evidence when no page-level collection errors occur. + - `--strict`: return `Err(String)` when any non-skipped page has missing/partial slot. + - Multi-URL page errors: JSON `ok=false`; command returns `Err(String)` after writing JSON/human output. + - Invalid schemes: fail before browser launch and before any output. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: verifier and pure comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_templates.rs crates/trusted-server-cli/src/audit/mod.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Verify ad-template slots from browser evidence" + ``` + +## Task 10: Add Local Browser Fixture Tests + +**Files:** + +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/ad_templates.rs` + +- [ ] **Step 1: Add test-only local HTTP fixture helper** + + In `audit::browser` tests, create a `TcpListener` serving static HTML from strings. Keep it test-only and host-target only. + + Fixture pages: + - direct `googletag.defineSlot`; + - `googletag.cmd.push`; + - late `window.googletag = { cmd: [] }`; + - late `window.apstag`; + - lazy slot created after scroll; + - redirect from `/` to `/news/story`; + - navigation returning 500. + +- [ ] **Step 2: Gate tests when Chrome is unavailable** + + Add helper: + + ```rust + fn chrome_available() -> bool { + ["chrome", "chromium", "google-chrome", "google-chrome-stable"] + .iter() + .any(|name| which::which(name).is_ok()) + } + ``` + + Each browser fixture test should early-return when unavailable. Do not use + `println!` / `eprintln!`; keep the skip reason in the helper name or a skipped + assertion message so clippy stays clean. This keeps CI portable unless Chrome is + installed. + +- [ ] **Step 3: Write fixture tests** + + Tests should assert the collector sees evidence, not real ad network behavior: + - direct GPT evidence confirms; + - command-queue GPT evidence confirms; + - APS `fetchBids` evidence removes APS provider warning; + - lazy slot appears only when `--scroll` is set; + - redirect result uses final path; + - failed page produces page-level error while other pages continue. + +- [ ] **Step 4: Run fixture tests locally** + + Run: + + ```bash + cargo test -p trusted-server-cli browser_fixture --target $(rustc -vV | sed -n 's/^host: //p') -- --nocapture + ``` + + Expected: pass when Chrome/Chromium exists; otherwise tests skip with explicit message. + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/ad_templates.rs + git commit -m "Add browser fixtures for ad-template verification" + ``` + +## Task 11: Update Documentation And Help Snapshots + +**Files:** + +- Modify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` if implementation decisions differ. +- Modify: `trusted-server.example.toml` only if command examples need harmless fictional config comments. +- Modify: `CLAUDE.md` only if verification commands or CLI command surface need to be documented. + +- [ ] **Step 1: Run CLI help manually** + + Run: + + ```bash + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit ad-templates verify --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- config ad-templates --help + ``` + + Expected: nested audit commands are discoverable; hidden legacy `ts audit ` does not dominate help text. + +- [ ] **Step 2: Update docs if help text or behavior differs from spec** + + Keep examples using `https://www.example.com/` only. Do not mention real publisher sites. + +- [ ] **Step 3: Run docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: Prettier passes. + +- [ ] **Step 4: Commit** + + ```bash + git add docs trusted-server.example.toml CLAUDE.md + git commit -m "Document ad-template CLI verification" + ``` + + If no docs changed, skip the commit. + +## Task 12: Final Verification + +**Files:** + +- Verify all touched files. + +- [ ] **Step 1: Rust format** + + Run: + + ```bash + cargo fmt --all -- --check + ``` + + Expected: pass. + +- [ ] **Step 2: Host CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: pass. Browser fixture tests either pass or explicitly skip when Chrome/Chromium is unavailable. + +- [ ] **Step 3: Workspace tests** + + Run: + + ```bash + cargo test --workspace + ``` + + Expected: pass. + +- [ ] **Step 4: Clippy** + + Run: + + ```bash + cargo clippy --workspace --all-targets --all-features -- -D warnings + ``` + + Expected: pass. + +- [ ] **Step 5: Wasm isolation proof** + + `trusted-server-adapter-fastly` does **not** depend on `trusted-server-cli`, so the + adapter build never compiles the CLI crate and cannot detect a CLI-crate dep leak. + The real proof is building the **CLI crate itself** for the wasm target (its modules + are `#[cfg(not(target_arch = "wasm32"))]`, so a wasm build must succeed with the + host-only deps compiled out). Note the workspace default target is already + `wasm32-wasip1`, so Steps 3–4 (`cargo test/clippy --workspace`) also build the CLI + crate for wasm — but make the isolation check explicit: + + ```bash + # Real CLI isolation proof: CLI crate must build for wasm with host deps excluded. + cargo build --package trusted-server-cli --target wasm32-wasip1 + # Adapter still built to confirm the production artifact is unaffected. + cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 + ``` + + Expected: both pass. If `chromiumoxide`/`tokio`/etc. leaked into a non-target-cfg + dependency table, the first command fails — that is the guard. + +- [ ] **Step 6: Docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: pass. + +- [ ] **Step 7: Inspect final diff** + + Run: + + ```bash + git status --short + git diff --stat origin/server-side-ad-templates-impl...HEAD + git log --oneline origin/server-side-ad-templates-impl..HEAD + ``` + + Expected: only intended CLI/core/doc files changed; no `.env`, operator `trusted-server.toml`, or generated browser artifacts included. + +## Risks And Watch Points + +- `chromiumoxide` must remain a host-only `trusted-server-cli` dependency. Any wasm build failure here means the dependency leaked. +- `ts audit ` compatibility must not swallow `ts audit ad-templates` as a URL. +- Runtime gate extraction (Task 1) only touches `should_run_server_side_ad_stack` + (the navigation gate). `/__ts/page-bids` is **intentionally NOT routed** through + `evaluate_ad_stack_gate` — its gate semantics differ (bot/prefetch skip the auction + but keep slots; no `is_navigation`/`is_get` gate). Its parity is preserved by + leaving it untouched, not by sharing the helper. Do not reroute page-bids. Keep + existing publisher and page-bids tests passing. +- The browser collector must not capture page HTML, cookies, storage, request bodies, or arbitrary DOM. Only collect configured-prefix DOM IDs and ad-related evidence. Never override `navigator.webdriver`. +- `runtime_ad_stack_expected = "unknown"` is normal for live consent state; do not over-model consent unless the collector can prove it. +- Browser fixture tests must not depend on real GPT/APS network calls. +- **Deferred in this implementation:** `/__ts/page-bids` SPA observation (spec §5.2 + "when available"). `PageBidsEvidence` exists as forward scaffolding but is not + collected (Task 8), surfaced in JSON (Task 4), or tested. Revisit if SPA route + verification is prioritized. +- Keep generation (`ts audit ad-templates generate`) out of this PR. diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md new file mode 100644 index 000000000..f3542580b --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -0,0 +1,252 @@ +# Dedicated Server-Side Ad Templates Switch and Cache Policy Plan + +> **For agentic workers:** Implement this plan task-by-task, keeping the dedicated +> template switch separate from the global auction configuration. + +**Goal:** Add an explicit on/off switch for server-side ad templates, while +retaining the browser-facing cache policy from issue #1007: + +- Server-side ad templates active: `Cache-Control: private, no-store`. +- Structurally inactive server-side ad templates: successful request-eligible GET + document HTML and its `304 Not Modified` revalidations use exactly + `Cache-Control: private, max-age=60`, except origin `private`/`no-store` + policies remain unchanged. +- Responses other than `200 OK` and `304 Not Modified`, non-GET, non-document, + bot, prefetch, and consent-denied responses keep the origin browser cache policy. +- CDN-specific cache headers must not change when templates are inactive. + +**Issue context:** The current cache-policy change uses the runtime +`should_run_ad_stack` gate. That gate is also affected by `[auction].enabled`, +which is not the right configuration boundary for publisher templates. A +browser can call `POST /auction`, and that endpoint is a separate server-run +auction API. The new switch must disable publisher HTML/page-bids template +delivery without disabling that API. + +## Configuration decision + +Add this field to the existing `[creative_opportunities]` section: + +```toml +[creative_opportunities] +enabled = true +``` + +Use `enabled = false` to turn off server-side ad templates while retaining the +slot definitions and keeping direct `POST /auction` behavior available. + +### Compatibility rules + +- The field defaults to `true` when omitted, preserving existing behavior for + deployments that already have `[creative_opportunities]` configured. +- The section remains optional. An absent section continues to mean that the + feature is unavailable. +- Serialize the default `true` value as omitted, matching the existing + rollback-compatibility pattern for newer creative-opportunity fields. An + explicit `false` must remain serialized so the setting is not silently lost. +- `auction.enabled` remains a separate auction/orchestrator setting. Do not use + it as the dedicated template switch and do not thread the new template flag + into `POST /auction`. + +## Current cache behavior to retain + +The existing HTML policy block in `publisher.rs` must remain structurally +consistent with the current issue #952 behavior: + +1. For an eligible request that runs the server-side ad stack and receives HTML: + - Set `Cache-Control: private, no-store`. + - Remove `ETag` and `Last-Modified`. + - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, + and `Cloudflare-CDN-Cache-Control`. +2. For a request-eligible GET document response where the server-side ad stack + is structurally inactive, including an explicit template disable: + - On `200 OK` HTML and `304 Not Modified`, set exactly + `Cache-Control: private, max-age=60`, replacing origin browser policies as + specified by issue #1007 unless the origin sends `private` or `no-store`. + The `private` directive prohibits storage by shared caches that use + `Cache-Control`. Applying the policy to `304` prevents revalidation from + restoring the origin freshness policy on the stored document. + - Leave validators and all CDN-specific cache headers untouched. +3. For other statuses, non-GET, non-document, bot, prefetch, and consent-denied + responses, preserve the origin browser cache policy. +4. Apply request-scoped privacy finalization after this policy so GPT diagnostics + and cookie-bearing responses can still require `private, no-store`; a + cookie-bearing response therefore ends as `private, max-age=0` when it did + not already carry a stricter policy. + +## File map + +### Configuration and compatibility + +- `crates/trusted-server-core/src/creative_opportunities.rs` + - Add `CreativeOpportunitiesConfig::enabled` with a default-true serde + implementation and documentation. + - Add a small accessor if it improves readability, but keep the source of + truth in this config type. + - Update config constructors and serialization tests. +- `crates/trusted-server-core/src/settings.rs` + - Keep `creative_opportunities` parsing and runtime preparation compatible with + the new field. + - Make `creative_opportunity_slots()` return an empty slice when the section + is absent or explicitly disabled, so all adapters receive one consistent + runtime view. + - Add TOML and environment-override coverage for `enabled = false`. +- `crates/trusted-server-core/src/config.rs` + - Extend legacy-schema tests to prove default `enabled = true` is omitted from + serialized blobs and remains readable by older binaries. + - Prove an explicit `enabled = false` is serialized, making rollback failure + loud rather than silently re-enabling templates. +- `trusted-server.example.toml` + - Document `creative_opportunities.enabled` and show how to turn templates off + without deleting slot definitions. +- `docs/guide/configuration.md` + - Add the field to the creative-opportunities reference and document the + environment override: + `TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false`. + - Clarify that this switch controls publisher HTML/page-bids template + delivery, not direct `POST /auction` callers. +- `CHANGELOG.md` + - Add an entry describing the dedicated template switch and cache behavior. + +### Publisher execution and cache policy + +- `crates/trusted-server-core/src/publisher.rs` + - Include the dedicated flag in the initial publisher eligibility decision. + - Do not match, dispatch, or inject server-side ad templates when the flag is + false, even if slots are configured and `[auction].enabled` is true. + - Apply the issue #1007 inactive-HTML cache policy in this state. + - Update skip-reason diagnostics/telemetry so `ad_templates_disabled` is + distinguishable from `auction_disabled`, consent denial, bots, prefetch, and + no matching slots. + - Update `handle_page_bids` so an explicit template disable returns the normal + empty JSON shape (`slots: []`, `bids: {}`) rather than slot definitions. Keep + the current `404` behavior for an absent `[creative_opportunities]` section. + - Extend the existing SSAT cache-policy and eligibility tests. +- `crates/trusted-server-core/src/auction/endpoints.rs` + - Do not gate `POST /auction` on the new template flag. + - Add a regression test or test fixture proving that disabling + `creative_opportunities.enabled` does not suppress a direct auction request + when providers are configured. + - Separately document/verify the existing behavior of `[auction].enabled` for + this endpoint; do not conflate that global setting with the new template + switch. + +### Adapter propagation and browser behavior + +The adapters already pass `Settings::creative_opportunity_slots()` into the +publisher/page-bids handlers. Update and verify these call sites so the central +empty-slice behavior is honored; avoid adding four divergent config checks: + +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/src/app.rs` +- `crates/trusted-server-adapter-cloudflare/src/app.rs` +- `crates/trusted-server-adapter-spin/src/app.rs` + +No route-level flag is needed if the core `Settings` accessor and handlers are +correct. Add adapter route assertions only where existing fixtures make them +useful. + +The browser runtime already defaults `window.tsjs.adSlots` and +`window.tsjs.bids` to empty values when the edge does not inject templates. If +terminology is updated, adjust these comments/tests without changing runtime +semantics: + +- `crates/trusted-server-js/lib/src/core/index.ts` +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Relevant page-bids tests under `crates/trusted-server-js/lib/test/integrations/gpt/` + +## Implementation tasks + +### Task 1: Add and serialize the dedicated setting + +- [ ] Add `enabled: bool` to `CreativeOpportunitiesConfig` with default `true`. +- [ ] Use `skip_serializing_if` so the default value does not appear in stored + config blobs; explicit `false` must serialize. +- [ ] Update all Rust struct literals in `creative_opportunities.rs` and + `publisher.rs` tests. +- [ ] Add parsing, default, false-value, and environment-override tests. +- [ ] Update the legacy compatibility tests in `config.rs`. + +### Task 2: Thread the setting through publisher eligibility + +- [ ] Update `should_run_server_side_ad_stack` to accept the dedicated template + flag as an explicit gate, with a descriptive parameter/doc comment. +- [ ] Ensure initial publisher slot matching and `Settings::creative_opportunity_slots` + do not expose slots when templates are disabled. +- [ ] Preserve the existing `[auction].enabled` and consent gates as separate + conditions. +- [ ] Add an `ad_templates_disabled` diagnostic/telemetry skip reason where the + current branch records a skipped auction. + +### Task 3: Apply the cache policy to the dedicated-off state + +- [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header + removal unchanged. +- [ ] Keep the inactive-HTML `private, max-age=60` behavior from issue #1007. +- [ ] Verify that structurally inactive request-eligible responses replace the + browser-facing `Cache-Control` for `200 OK` GET document HTML and + `304 Not Modified` revalidations, including origin `no-cache` and zero-age + policies, while preserving `private` and `no-store`. +- [ ] Preserve `ETag`, `Last-Modified`, and every CDN-specific header. +- [ ] Verify that other statuses, non-GET, non-document, bot, prefetch, and + consent-denied responses retain the origin browser cache policy. +- [ ] Verify that request-scoped GPT diagnostics and cookie privacy override this + policy. + +### Task 4: Gate SPA page-bids/template delivery + +- [ ] Include `co_config.enabled` in the `ad_stack_enabled` decision in + `handle_page_bids`. +- [ ] Return empty slots and bids for an explicit disable while retaining the + endpoint and its existing response privacy headers. +- [ ] Keep the absent-section `404` behavior unchanged. +- [ ] Add tests for enabled, disabled, absent, consent-denied, bot, and prefetch + cases as appropriate; preserve existing tests for `[auction].enabled=false`. + +### Task 5: Protect direct `POST /auction` from accidental coupling + +- [ ] Add a focused endpoint test with `creative_opportunities.enabled=false` + and a recording provider. +- [ ] Assert that the provider still sees the direct auction request and that + the response remains a normal OpenRTB response. +- [ ] If the test reveals that `[auction].enabled=false` also needs a separate + product decision for `/auction`, record that as a follow-up rather than + changing it as part of the template-switch work. + +### Task 6: Update docs, examples, comments, and adapter coverage + +- [ ] Update the example config, configuration guide, and changelog. +- [ ] Update stale comments that call `[auction].enabled` the universal template + kill switch. +- [ ] Verify all four adapter call sites use the centralized disabled-slot view. +- [ ] Run JS tests if comments or tests are touched; no JS behavior change is + expected. + +## Test plan + +Use target-matched commands; do not run bare workspace tests because the +workspace contains multiple runtime targets. + +- [ ] `cargo test-axum -p trusted-server-core publisher` +- [ ] `cargo test-fastly` +- [ ] `cargo test-axum` +- [ ] `cargo test-cloudflare` +- [ ] `cargo test-spin` +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy-fastly` +- [ ] `cargo clippy-axum` +- [ ] `cargo clippy-cloudflare` +- [ ] `cargo clippy-cloudflare-wasm` +- [ ] `cargo clippy-spin-native` +- [ ] `cargo clippy-spin-wasm` +- [ ] `cd crates/trusted-server-js/lib && npx vitest run` if JS tests/comments change +- [ ] `cd docs && npm run format` if documentation formatting is required + +## Non-goals + +- Do not change CDN-specific cache policy for inactive templates. +- Do not change adapter response privacy or cookie handling. +- Do not use `auction.rewrite_creatives` as the template switch; it controls + creative URL rewriting, not whether the server-side template stack runs. +- Do not gate or disable direct `POST /auction` as part of this feature. +- Do not remove slot definitions when the switch is off; the point of the switch + is to provide a reversible runtime control. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..5b9a455e7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,967 @@ +# #1009 ESI Validation Spike + +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See +> [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) +``` + +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check no shared dependency was forced to move** + +```bash +git diff --stat Cargo.lock +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" +``` + +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. + +- [ ] **Step 4: Record and commit, or stop** + +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. + +Propagate a **lineage ID plus the experiment arm** through the whole chain: + +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture C1 and C2 status separately** + +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. + +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** + +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. + +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. + +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. + +- [ ] **Step 4: Write and read C2 — with the real API** + +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: + +```rust +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; +``` + +Correct shape, using a transaction so a cold cache under load transforms once: + +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + found.to_stream()? // C2 HIT — skip origin fetch and transform +} else { + unreachable!("a transaction is either obliged to insert or has found an item") +}; +``` + +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` — measured, see the Stage 0 findings): + +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. + +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. + +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own ESI include tag comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own nested ESI include is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); +``` + +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** + +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal +rather than an address. + +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** + +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. + +- [ ] **Step 6: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 7: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own nested ESI include is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md b/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md new file mode 100644 index 000000000..0fb8977a0 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md @@ -0,0 +1,46 @@ +# Contiguous Generated Slot Tables Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Keep generated creative-opportunity slot and provider tables contiguous with their parent section. + +**Architecture:** Normalize the document positions carried by generated `toml_edit` tables before inserting them into the target document. Anchor the whole generated subtree at the target creative section and rely on stable serialization order. + +**Tech Stack:** Rust, `toml_edit`, Cargo tests + +--- + +### Task 1: Reproduce the position collision + +**Files:** + +- Modify/Test: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Add `splice_keeps_generated_slots_and_providers_contiguous` with a late creative section and unrelated tables at colliding positions. +- [ ] Assert no unrelated table header occurs between `[creative_opportunities]`, all generated slots, and their provider subtables. +- [ ] Add `splice_groups_a_new_creative_section_with_its_slots` for an input that has no creative section, proving the newly created parent and generated subtree share the final anchor. +- [ ] Run each focused test with `cargo test_cli_macos -- --exact` and confirm both ordering assertions fail. + +### Task 2: Normalize imported table positions + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Add a small recursive helper using `Table::set_position`, `Table::iter_mut`, and `ArrayOfTables::iter_mut` to assign one anchor position to every table in the generated slot subtree. +- [ ] Use the existing creative table's position; for a newly created section, allocate one greater than the greatest parsed position and explicitly assign that anchor to both the new parent and its generated subtree. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests::splice_keeps_generated_slots_and_providers_contiguous -- --exact` and confirm it passes. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests::splice_groups_a_new_creative_section_with_its_slots -- --exact` and confirm it passes. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests` and confirm the complete module suite passes. + +### Task 3: Verify and deliver + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Run `./scripts/test-cli.sh`. +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets -- -D warnings`. +- [ ] Confirm `trusted-server.toml` and the user's existing `fastly.toml` change remain untouched. +- [ ] Commit the verified generator fix on the current feature branch. diff --git a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md new file mode 100644 index 000000000..873168438 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md @@ -0,0 +1,723 @@ +# PR 823 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4958563121`, verify the branch, publish it, and answer all 28 inline threads. + +**Architecture:** Correct the review findings at four existing seams: core runtime gate APIs, pure CLI projection/comparison, crawl generation and TOML persistence, and the shared browser session. Keep page-controlled work bounded, use one source of truth for runtime/browser behavior, and preserve operator-authored configuration outside the managed creative-opportunities fields (`slot`, `gam_network_id`, `section_root`, and `section_segment`). + +**Tech Stack:** Rust 2024, clap 4, toml_edit 0.23, chromiumoxide 0.9, Tokio current-thread runtime, serde/serde_json, embedded JavaScript collector, mdBook documentation, GitHub CLI. + +--- + +## File Map + +- `crates/trusted-server-core/src/creative_opportunities.rs`: allocation-free gate evaluation, gate diagnostics, pattern validation, consent semantics. +- `crates/trusted-server-core/src/publisher.rs`: named gate input at the runtime call site. +- `crates/trusted-server-cli/src/ad_templates/{expected,compare,output}.rs`: runtime-equivalent projection, typed formats, confirmability, safe output. +- `crates/trusted-server-cli/src/commands/config/ad_templates.rs`: static command validation, gate parity, lint, escaping. +- `crates/trusted-server-cli/src/commands/audit/{collector,browser,ad_templates,ad_template_collector.js}.rs`: shared browser options/session and verifier behavior. +- `crates/trusted-server-cli/src/commands/audit/generate/{browser_collector,evidence,gpt_slots,crawl_plan,page_patterns,unit_template,slot_toml,mod,validate}.rs`: crawl evidence, inference, persistence, and dry-run safety. +- `crates/trusted-server-cli/src/commands/audit/{mod,page}.rs`, `crates/trusted-server-cli/src/run.rs`, `crates/trusted-server-cli/src/main.rs`: clap contracts and exit outcomes. +- `docs/guide/cli.md`, `scripts/test-cli.sh`, `.github/workflows/test.yml`: operator contract and enforced browser CI. + +## Task 1: Make the runtime gate API allocation-free and reusable + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add failing core tests** + +Add tests that sweep all 64 boolean combinations with `consent_allows_auction: None`, assert the expected `No`/`Unknown` result, assert `blocking_gates()` derives diagnostics without an owned `Vec`, and exercise the specific page-pattern validation error. + +Use a borrowed/static iterator contract: + +```rust +pub fn blocking_gates(self) -> impl Iterator { + AdStackGateName::ALL + .into_iter() + .filter(move |gate| gate.blocks(self.input)) +} + +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-core --target "$(rustc -vV | awk '/host:/ {print $2}')" ad_stack_gate -- --nocapture +``` + +Expected: failure because the unknown-consent sweep and allocation-free diagnostic API are not implemented. + +- [ ] **Step 3: Implement the minimal core change** + +Store the original `AdStackGateInput` in `AdStackGateResult`, compute `expected` with boolean expressions rather than `Vec::push`, expose a zero-allocation iterator over a `const ALL`, make `compile_page_pattern` crate-private, and add `validate_page_pattern`. Document that `None` means unknown and differs from denied (`Some(false)`). Preserve the detailed glob error in `compile_patterns`. + +Delete `should_run_server_side_ad_stack`; construct `AdStackGateInput` with named fields in `publisher.rs`. Import the gate types at module scope. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow command again, then: + +```bash +cargo test-fastly creative_opportunities +cargo test-axum creative_opportunities +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Align ad stack gate diagnostics with runtime" +``` + +## Task 2: Align expected-slot projection and comparison with runtime behavior + +**Files:** + +- Modify: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` + +- [ ] **Step 1: Add failing projection and comparison tests** + +Cover: + +- an unrenderable dynamic slot is omitted from expected slots and does not make `matched_slots` pass; +- the diagnostic says the runtime omits the slot for that path; +- `MediaType` remains typed through comparison; +- video/native-only slots produce `Unconfirmable` and do not fail strict; +- a sizeless out-of-page slot against banner-configured formats is `Partial` and fails strict; +- an incompatible banner is still `Partial` and fails strict; +- a missing slot has `phase: None` and JSON omits `phase`; +- server-side APS configuration alone does not emit `aps_evidence_missing`; +- collector warnings are appended to page warnings; +- human output contains expectation, gates, matched count, extra evidence, and warnings; +- bidi override/isolate characters are escaped. + +The central type changes are: + +```rust +pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + pub media_type: MediaType, +} + +pub enum SlotStatus { + Confirmed, + Partial, + Missing, + Unconfirmable, +} + +pub struct SlotResult { + pub phase: Option, + // existing fields +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::expected +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::compare +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +``` + +Expected: new assertions fail on current projection/status/warning behavior. + +- [ ] **Step 3: Implement projection, comparison, and output changes** + +Filter `match_slots` with `render_gam_unit_path(...).map(...)` while building `ExpectedSlot`. Remove the unconditional client-side APS check. Compute confirmability before assigning status. Map typed media values to strings only in `to_slot_json`. Make JSON phase `Option` with `skip_serializing_if = "Option::is_none"`. Extend warnings with `evidence.warnings` after decode. + +Extend `is_terminal_control` with `0x202A..=0x202E` and `0x2066..=0x2069`. Apply `escape_terminal_text` to every human-facing page/config-derived field. + +- [ ] **Step 4: Verify GREEN** + +Run all three narrow commands again. + +Expected: all selected tests pass with no warnings. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/ad_templates crates/trusted-server-cli/src/commands/audit/ad_templates.rs +git commit -m "Match ad template verification to runtime behavior" +``` + +## Task 3: Correct static CLI contracts and process exit semantics + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/config/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/main.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `Cargo.lock` + +- [ ] **Step 1: Add failing parser, normalization, lint, and outcome tests** + +Add tests proving: + +- bare and full-URL forms normalize spaces, dot segments, tabs, queries, and fragments identically; +- `/r?to=https://example.com` remains a bare path; +- `check` requires exactly one expectation mode and rejects `--allow-extra-slots --expect-no-slots` through clap; +- `--method` accepts a valid `http::Method` and uses exact GET semantics; +- `lint` reports each invalid configured pattern; +- `explain` uses `gate.expected` even when providers are empty and prints provider state separately; +- `--edgezero-enabled` is rejected because the unsupported model is removed; +- bare `ts audit` displays help rather than a drifting manual error; +- parser coverage includes lint, explain, generate, verify profiles/options, and the no-`--adapter` contract; +- an assertion outcome maps to exit 1 and a tool error maps to exit 2. + +Use an explicit process outcome: + +```rust +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + Success, + AssertionFailed, +} + +impl RunOutcome { + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} +``` + +Tool failures remain `Err(String)` and therefore exit 2. Assertion commands write their failure to stderr before returning `AssertionFailed`, avoiding `log::error!` filtering. + +- [ ] **Step 2: Run parser/static tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::config::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" run::tests +``` + +Expected: current hand-rolled validation, normalization, and exit behavior fail the new tests. + +- [ ] **Step 3: Implement the CLI contract** + +Use a dummy HTTPS base with `Url::options().base_url(...)` for bare paths after anchored scheme detection on the pre-query slice. Add clap `ArgGroup`, `conflicts_with`, `arg_required_else_help`, typed `http::Method`, and browser settle validation. Add `http = { workspace = true }` to the CLI host dependencies. + +Return `RunOutcome` from dispatchable CI commands. Keep edgezero delegated errors as tool errors. Remove the unsupported EdgeZero flag/text and route gate output through `blocking_gates()`. + +- [ ] **Step 4: Verify GREEN** + +Run the two narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/main.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/config/ad_templates.rs +git commit -m "Define ad template CLI assertion contracts" +``` + +## Task 4: Make the injected collector bounded and behavior-preserving + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] **Step 1: Add failing JavaScript-contract and decoder tests** + +Add tests/fixtures for an out-of-`u32` size beside a valid slot, a truthy `googletag.cmd` without `push`, multiple `cmd.push` arguments, 512-character capture limits, and non-enumerable/closure-local wrapping. Replace the existing `contains("cmd.push")` assertion with assertions that the no-op wrapper is absent. + +The JavaScript bounds are: + +```javascript +const __TS_MAX_STRING = 512 +function __ts_text(value) { + return String(value).slice(0, __TS_MAX_STRING) +} + +if (width > 4294967295 || height > 4294967295) return null +``` + +The setter must always retain the publisher value: + +```javascript +set(value) { + try { + internal = wrap(value) + } catch (error) { + internal = value + __ts_push(__ts_ev.warnings, { + code: "wrap_failed", + message: __ts_text(error), + }) + } +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::collector +cargo test --package trusted-server-cli --target "$HOST_TARGET" collector_payload +``` + +Expected: current script permits oversized integers and retains the behavior-changing wrapper. + +- [ ] **Step 3: Implement minimal collector changes** + +Guard all page-derived strings through `__ts_text`, enforce numeric upper bounds, delete the `cmd.push` wrapper, use a closure-local `WeakSet` for wrapped objects, and install wrapped functions with non-enumerable `Object.defineProperty`. Soften the header claim to “observes without capturing page data.” + +Before serde decode, stringify the evidence inside the page and return a small +sentinel instead of the payload when the serialized string exceeds 1 MiB +(`MAX_EVIDENCE_PAYLOAD_BYTES = 1_048_576`). On the Rust side, the sentinel +produces an `ad_evidence_too_large` warning and `ad_evidence: None`; it does not +fail navigation or the whole collection. This bounds CDP transfer and Rust +decode/allocation while preserving a precise operator diagnostic. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/ad_template_collector.js crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/browser.rs +git commit -m "Bound browser ad template evidence collection" +``` + +## Task 5: Unify browser launch, session reuse, and settling + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing fake-collector and browser configuration tests** + +Cover one launch/session for multiple URLs, root included in the profile batch, page close on success/error, host-only `Path=/` cookies, explicit final-URL failure, same-host HTTP-to-HTTPS acceptance, host/downgrade/port refusal, new-headless 1280x800 defaults, headful/profile/proxy/consent parity, `$CHROME` parity, and generic/legacy default-on consent. + +Extend the trait with a default batch method so fakes remain simple: + +```rust +pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + requests.iter().cloned().map(|request| self.collect_page(request)).collect() + } +} +``` + +The real browser implementation overrides `collect_pages` to create one runtime, +temporary profile, browser, handler, and sequentially closed pages. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::browser_collector +``` + +Expected: verifier launches per URL, browser defaults diverge, and tabs/cookies/final URL handling fail new assertions. + +- [ ] **Step 3: Implement shared browser configuration and batching** + +Move executable resolution and launch-option construction into `browser.rs` as crate-visible helpers used by both collectors. Flatten shared browser options into generate and verify, while keeping generation-only pacing/crawl flags local. Build cookies with explicit domain from `url.host_str()` and `path = Some("/".to_string())`; do not set `url` simultaneously. + +In each page collector, capture the inner result, always call bounded `page.close().await`, then return the captured result. Batch verify requests via `collect_pages`. Include the root in each profile's batch rather than collecting it in a throwaway session. Use `spawn_blocking` for scraper analysis before folding results. + +- [ ] **Step 4: Bound post-navigation work and correct settle semantics** + +Install `performance.setResourceTimingBufferSize(100000)` before navigation. Make `settle` return warnings and wrap every `evaluate`, URL/title read, scroll operation, and evidence read in a per-operation timeout. Accrue quiet only after `document.readyState` is `interactive` or `complete`; sleep `min(remaining_quiet, 250ms)` so short quiet values are honored. Treat `wait_for_navigation` timeout as a warning after successful `goto`. + +Propagate GPT/link/sitemap evaluation errors as notes, set `await_promise` for sitemap discovery, and warn when only the main frame is inspected while child frames exist. + +- [ ] **Step 5: Verify GREEN** + +Run all three narrow commands again. If Chrome is available, also run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: -- --ignored --test-threads=1 +``` + +Expected: unit/fake tests pass; browser fixtures execute and pass when Chrome exists. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit +git commit -m "Share browser sessions across ad template audits" +``` + +## Task 6: Preserve crawl evidence and make inference conservative + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference tests** + +Add focused tests for: + +- `annonsü1`/`annonsü2` and `ünicode-ad-a`/`ünicode-ad-b` prefixes; +- desktop-empty/mobile-present and the inverse; +- two disjoint unrelated placements retained, two with a useful prefix or three fragments refused; +- same-page normalized UUID collisions retained with raw div IDs and all formats; +- 16+ digit numeric stable segments retained; +- comma-separated SRA `dids` ignored; +- locale `/en` pattern emitted as `/en` and every emitted glob matches its source path; +- glob metacharacters escaped with `glob::Pattern::escape`; +- percent-encoded noise/extension paths and `.html`/`.htm`/`.php` treatment; +- dropped-section notes capped at ten plus “and N more”; +- both ambiguous template rows result in explicit `Refuse`; +- real crawl evidence can infer `section_segment = 1`; +- refused slots do not appear in rendered output and their reasons appear in notes. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::evidence +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::gpt_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::page_patterns +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::crawl_plan +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::unit_template +``` + +Expected: each new regression reproduces its review finding. + +- [ ] **Step 3: Implement evidence-preserving discovery** + +Use the last matching `char_indices` byte boundary for shared prefixes. Remove an empty-page marker whenever a later profile yields slots. Require `(useful shared prefix || group size >= 3)` before classifying disjoint same-shape slots as fragments; emit an ambiguity diagnostic otherwise. + +Group normalized collisions within a page before deduplication. When a group has multiple raw div IDs, keep raw entries, make their generated IDs unique, and attach a collision note. Restrict ephemeral hex matching to tokens containing at least one `a..f`, or an explicit UUID shape; never treat all-digit identifiers as hashes. Reject gampad fallback when parsed `dids` contains a comma. + +- [ ] **Step 4: Implement conservative patterns/templates** + +Emit the observed short path for locale landing pages, escape literal prefixes, decode only for filtering while retaining encoded request paths for matching, and cap notes. Teach crawl planning to carry/infer the section depth used by page-pattern generation. + +Delete the tautological witness check and move its explanatory invariant into `analyse_slot` docs. Keep the existing conservative `Refuse` result for non-derivable slugs and unwitnessed roots. Filter all `Refuse` decisions before `RenderSlot` creation and push each reason into notes. + +- [ ] **Step 5: Verify GREEN** + +Run all five narrow commands again, then: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate +``` + +Expected: the generate module suite passes. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve ad template crawl evidence" +``` + +## Task 7: Make slot persistence and dry-run output safe + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `Cargo.lock` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/validate.rs` + +- [ ] **Step 1: Add failing persistence tests** + +Cover: + +- trailing comments after the final slot; +- a multiline string line beginning `[foo]`; +- an array continuation beginning `[300, 250]`; +- non-contiguous slot tables; +- byte-identical unrelated sections/comments and CRLF preservation; +- end-to-end `--replace` through `run_update_slots`; +- dry-run source file byte identity; +- stdout contains only a zero-context unified diff of managed + creative-opportunities changes and does not contain `admin_password` or + unrelated config; +- notes/rollback warning go to stderr; +- a concurrent source edit between initial read and write is refused; +- rerun unions formats and reports broad-prefix collapse. + +Change `run_update_slots` to accept separate writers: + +```rust +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()>; +``` + +- [ ] **Step 2: Run persistence tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots +``` + +Expected: current line scanner corrupts/preserves incorrectly and dry-run leaks the complete config. + +- [ ] **Step 3: Implement a TOML-aware managed edit** + +Parse the source as `DocumentMut` and update the complete managed field set: +`creative_opportunities.slot`, `gam_network_id`, `section_root`, and +`section_segment`. Insert the generated array-of-tables and upsert only scalar +values that generation actually inferred. A generated `None` preserves the +existing scalar on both merge and `--replace`; absence of fresh evidence is +never an instruction to delete operator configuration. Retain decorations on +all other items. Before returning, parse both documents and compare canonical +clones with all four managed fields removed; return an error if any other item +differs. Preserve CRLF after serialization. Add regression cases in Step 1 for +an unresolved network ID and literal-only rerun retaining existing +`gam_network_id`/section policy. + +Document `splice_creative_slots` at its definition and remove the orphaned comments. Replace the `let _ = network_id` presence check with `keys.network_id.is_none()` logic. + +- [ ] **Step 4: Implement secret-safe dry-run and stale-read protection** + +Add `similar` as a workspace/CLI dependency and render a zero-context unified +diff between the old and new managed creative-opportunities projection. The +projection contains only `gam_network_id`, `section_root`, `section_segment`, +and the slot array, so every generated scalar change is visible without +including unrelated operator keys: + +```rust +let diff = similar::TextDiff::from_lines(old_managed, new_managed); +writeln!(out, "{}", diff.unified_diff().context_radius(0).header("configured creative opportunities", "generated creative opportunities"))?; +``` + +Send all notes to `err`. Immediately before atomic rename, re-read the config and compare it with the original bytes; refuse on mismatch. Do not perform this check on dry-run because no write occurs. + +In `merge_render_slots`, union discovered formats into a matching existing slot and count how many discovered slots map to each existing prefix; report counts greater than one. + +- [ ] **Step 5: Verify GREEN** + +Run both narrow commands again and confirm all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve operator config during slot generation" +``` + +## Task 8: Complete documentation, test hygiene, and CI enforcement + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `scripts/test-cli.sh` +- Modify: `.github/workflows/test.yml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: touched Rust tests and comments under `crates/trusted-server-cli/src/` + +- [ ] **Step 1: Add/restore parser and CI guard tests** + +Restore the `audit` no-`--adapter` parser test. Add a script contract that sets `TS_AUDIT_BROWSER_TESTS=1`; browser fixture tests panic when that variable is set and Chrome cannot be resolved. Configure the workflow with a browser setup action or the runner's installed Chrome path and export `CHROME` before `scripts/test-cli.sh`. + +- [ ] **Step 2: Replace sensitive-looking fixtures and stale assertions** + +Replace sensitive or customer-shaped fixtures introduced by this PR with fictional network IDs, publisher names, URL shapes, and neutral div tokens. Update comments to describe shapes rather than customers. + +Correct all touched `expect` messages to start with `should`, remove redundant crate/file `dead_code` allowances and annotate only genuinely deferred fields, reorder `Audit`, simplify the Prebid query parser so keys—not substrings—are matched, and bind legacy URLs directly without an impossible `expect`. + +- [ ] **Step 3: Document the complete operator contract** + +In `docs/guide/cli.md`, document: + +- `config ad-templates lint|match|check|explain` and every flag; +- shared `--app-config`, `--manifest`, and `--no-env` behavior; +- `audit ad-templates generate|verify` browser/profile/proxy/consent/settle flags; +- dry-run stdout diff versus stderr notes; +- exit 0 success, exit 1 assertion drift, exit 2 tool/configuration error; +- refused slots are omitted with reasons; +- locale-prefixed inference and section depth; +- `Unconfirmable` strict behavior and optional evidence phase. + +Update the existing design/output examples where the wire contract changed. + +- [ ] **Step 4: Run format and focused checks** + +Run: + +```bash +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: both commands exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/test.yml scripts/test-cli.sh docs crates/trusted-server-cli/src +git commit -m "Document and enforce ad template audit contracts" +``` + +## Task 9: Run full verification and repair regressions + +**Files:** + +- Modify only files implicated by a failing check. + +- [ ] **Step 1: Run format and CLI/browser tests** + +```bash +cargo fmt --all -- --check +./scripts/test-cli.sh +``` + +Expected: exit 0; browser fixture output shows tests executed rather than skipped. + +- [ ] **Step 2: Run repository target suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all suites exit 0. + +- [ ] **Step 3: Run all target-matched clippy gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings +``` + +Expected: all commands exit 0 with no warnings. + +- [ ] **Step 4: Run cross-adapter parity gates** + +```bash +cargo fmt --manifest-path crates/trusted-server-integration-tests/Cargo.toml -- --check +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --all-targets -- -D warnings +``` + +Expected: formatting, parity tests, and integration-test clippy exit 0. + +- [ ] **Step 5: Run JavaScript and documentation checks** + +```bash +cd crates/trusted-server-js/lib && npx vitest run && npm run format && node build-all.mjs +cd ../../.. && cd docs && npm run format +``` + +Expected: tests/build/format exit 0. + +- [ ] **Step 6: Inspect the final diff against the review** + +Run: + +```bash +git diff --check origin/main...HEAD +git status --short +``` + +Walk the 28-thread traceability table and every summary category in the design spec. Confirm each has a code/doc/test resolution or an evidence-backed response. + +- [ ] **Step 7: Commit any verification-only corrections** + +If verification required changes, inspect `git diff --name-only`, stage each +listed path explicitly (never `git add .`), and commit them as `Resolve ad +template review regressions`. Record those exact paths in the execution log. +Skip this commit when verification required no changes. + +## Task 10: Publish and answer GitHub review threads + +**Files:** + +- No repository files unless publication reveals a conflict. + +- [ ] **Step 1: Push the verified branch** + +```bash +git push origin feature/ts-cli-ad-templates +``` + +Expected: push succeeds and PR 823 shows the verified head commit. + +- [ ] **Step 2: Correct the PR description** + +Change the legacy alias statement to say bare `ts audit ` aliases to `ts audit generate `. Preserve all unrelated PR-body content. + +- [ ] **Step 3: Reply to every inline thread** + +For each ID in the spec traceability table, post through: + +```bash +gh api repos/IABTechLab/trusted-server/pulls/823/comments//replies -f body='' +``` + +Each reply must name the concrete behavior changed and, where useful, the focused test. For question threads, state the chosen behavior: union formats and diagnose broad prefixes; default consent assumption on; keep conservative refusal and align docs; allow only same-host HTTP-to-HTTPS upgrades; remove the unsupported EdgeZero model. + +- [ ] **Step 4: Verify publication** + +Query PR 823's head SHA, review comments, checks, and unresolved threads. Confirm all 28 inline comments have one reply and no reply claims a fix absent from the pushed diff. + +- [ ] **Step 5: Report the result** + +Summarize commits, verification commands, any environment limitation, PR link, and thread reply count. Do not claim checks pass without fresh output from Task 9. diff --git a/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md b/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md new file mode 100644 index 000000000..c574c4104 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md @@ -0,0 +1,49 @@ +# Pre-navigation Cookie Installation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow domain/path-scoped operator cookies to be installed before the audit's first navigation. + +**Architecture:** Add one browser-level cookie installation helper beside `host_cookie`, and call it before creating each audit page. Preserve explicit host-only and root-path scope while avoiding `Page::set_cookie`'s `about:blank` validation. + +**Tech Stack:** Rust, chromiumoxide/CDP, Tokio, Cargo tests + +--- + +### Task 1: Reproduce the pre-navigation failure + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] Add a Chrome-backed test that installs a host-only cookie before navigating away from `about:blank` and asserts it reaches the first document. +- [ ] Exercise the existing `BrowserCollector` end-to-end against a local HTTP fixture, supplying the cookie through `BrowserCollectRequest`, so the RED test compiles before the fix exists. +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests::supplied_cookie_reaches_first_navigation -- --ignored --exact --nocapture` and confirm it fails with `Blank page can not have cookie`. +- [ ] Add a Chrome-backed error test against the wished-for `set_browser_cookies` API, using an invalid cookie name, and assert the error contains the name but not the secret value. +- [ ] Run that error test and confirm RED because `set_browser_cookies` does not exist yet. + +### Task 2: Install cookies at browser scope + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] Add `set_browser_cookies(&Browser, &[(String, String)], &Url) -> Result<(), String>` beside `host_cookie`; install one cookie per browser call so failures retain name-only context without exposing values. +- [ ] Invoke it before page creation in both collectors and remove page-level cookie installation. +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests::supplied_cookie_reaches_first_navigation -- --ignored --exact --nocapture` and confirm it passes. +- [ ] Run the focused error test and confirm it passes. + +### Task 3: Verify the change + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests`. +- [ ] Run `cargo test_cli_macos commands::audit::generate::browser_collector::tests`. +- [ ] Run `./scripts/test-cli.sh` to exercise the portable host-target suite and ignored browser fixtures. +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets -- -D warnings`. +- [ ] Inspect the diff to confirm no cookie values are logged and `fastly.toml` remains untouched. diff --git a/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md new file mode 100644 index 000000000..ca4245314 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md @@ -0,0 +1,167 @@ +# Ad-template Generation Progress Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Show immediate, safe, profile-aware progress while `ts audit ad-templates generate` performs a long browser crawl. + +**Architecture:** Add typed progress events to the `AuditCollector` boundary so the browser can report work before buffered page results are returned. Render and flush those events from `run_update_slots` on stderr, using only URL paths. Preserve crawl/progress errors over teardown errors while always closing and waiting for Chrome. + +**Tech Stack:** Rust 2024, `std::io::Write`, existing `url`, `tokio`, `chromiumoxide`, and CLI test helpers; no new dependency. + +--- + +### Task 1: Define and render safe progress events + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing renderer and writer tests** + +Add tests in `generate/mod.rs` for events covering launch, `1/?`, `2/17`, and finalization. Assert that `https://user:pass@publisher.example/news?token=secret#fragment` renders only `/news`, terminal control bytes are escaped, stdout remains untouched, and a counting writer records an explicit `flush()`. Add writers that fail independently on `write()` and `flush()` and assert a CLI output error. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin progress -- --nocapture +``` + +Expected: FAIL because the progress event and renderer do not exist. + +- [ ] **Step 3: Add the progress model and renderer** + +In `collector.rs`, define a small event enum and callback type: + +```rust +pub(crate) enum CollectionProgress<'a> { + Launching, + Loading { + current: usize, + total: Option, + url: &'a Url, + }, + Planning, + Finalizing, +} + +pub(crate) type ProgressSink<'a> = + &'a mut dyn FnMut(CollectionProgress<'_>) -> CliResult<()>; +``` + +Add concise doc comments to the enum, every variant, and the callback alias. The +callback documentation must state that returning an error stops new collection +work but does not bypass an already-launched browser's finalization/close/wait. + +In `generate/mod.rs`, add a `write_collection_progress` helper that accepts a profile label, formats only `url.path()` (or `/` when empty), sanitizes it with `escape_terminal_text`, writes one line to stderr, and immediately calls `flush()`. Render and test `Planning` between the root load and subsequent page loads. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all progress renderer/writer tests pass. + +### Task 2: Propagate progress through collectors with teardown-safe failures + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Write failing collector tests** + +Test default `collect_pages` and `collect_site` count semantics, including an attempted page whose collection fails. The exact dynamic-site sequence is root `1/?`, planning, then follow-ups `2/total` through `total/total`; totals include the root and failed attempts advance the count. Add a Chrome-backed test whose progress callback fails during collection. It must return the progress error only after the browser teardown path completes. Extend the existing result-combination unit tests to cover first-error preservation across a collection/planning error, a later finalization-progress error, close error, and wait error, while proving finalization, close, and wait were all attempted. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::browser_collector::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::collector::tests -- --nocapture +``` + +Expected: FAIL because collectors do not accept or emit progress callbacks. + +- [ ] **Step 3: Add callbacks to the collector boundary** + +Extend `collect_pages` and `collect_site` with `ProgressSink`. Default collectors emit `Loading` before each page. The root of a dynamically planned site emits `current: 1, total: None`, followed by `Planning`; after planning, default `collect_site` iterates follow-ups itself with an explicit offset so they report `2/total` onward. Fixed batches emit totals including the root, and failed attempts still consume their position. + +Pass the callback into `with_browser`. Adapt `BrowserAuditCollector::collect_page` with an explicit no-op progress sink because single-page artifact generation has no command progress writer. Emit `Launching` before browser launch, `Loading` immediately before each navigation, `Planning` immediately before invoking the root planner, and `Finalizing` before close/wait. Track only the first crawl/progress error: on callback failure, stop scheduling pages, still attempt finalization, `browser.close()`, and `browser.wait()`, then return that first error ahead of teardown errors. + +Extend `scripts/test-cli.sh` with a second ignored-test filter for +`commands::audit::generate::browser_collector::tests::` so the new Chrome-backed +progress-failure test is actually executed under `TS_AUDIT_BROWSER_TESTS=1` and +single-threaded, alongside the existing three browser audit fixtures. + +- [ ] **Step 4: Run unit and Chrome-backed tests and confirm GREEN** + +Run the focused command, then: + +```bash +./scripts/test-cli.sh +``` + +Expected: collector unit tests and all four Chrome-backed tests pass. + +### Task 3: Wire profile-aware progress into generation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing generation tests** + +Update `run_update_slots` tests to assert the stderr buffer contains progress for the first profile's `1/?` root, planning, later known totals, the second profile's `1/total` root, and finalization. Assert dry-run diff/success output on stdout contains no progress lines. Add an ordering test with a shared observable writer and fake collector: from inside `collect_site`, after invoking and flushing the progress callback but before returning, assert the progress bytes are already visible. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots -- --nocapture +``` + +Expected: FAIL because `run_update_slots` does not provide progress callbacks. + +- [ ] **Step 3: Connect callbacks and profiles** + +Create a progress closure for the first profile and pass it to `collect_site`. Pass `err` through `crawl_sections`, create a closure for each later profile, and pass it to `collect_pages`. Keep notes and final summary behavior unchanged. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all generation tests pass and progress appears only in stderr. + +### Task 4: Verify and ship + +**Files:** + +- Verify all modified files plus the two design documents. + +- [ ] **Step 1: Format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +git diff --check +cd docs && npm run format +``` + +Expected: all commands exit 0. + +- [ ] **Step 2: Run the complete local CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: unit, config, proxy, documentation, and Chrome-backed tests pass. + +- [ ] **Step 3: Review the scoped diff** + +Confirm no cookie values, real publisher data, or changes to the pre-existing `fastly.toml` modification are included. Request an independent code review and address concrete findings. + +- [ ] **Step 4: Commit and push** + +Stage only the progress implementation and its design/plan documents. Commit with `Show ad-template generation progress`, push `feature/ts-cli-ad-templates`, and confirm local HEAD matches the remote branch. diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md new file mode 100644 index 000000000..1dca98982 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -0,0 +1,78 @@ +# Refuse Volatile Div-ID Collisions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prevent `ts audit ad-templates generate --replace` from writing exact per-render div IDs when several live elements normalize to one runtime prefix. + +**Architecture:** Keep collision detection in GPT discovery, where normalized and raw IDs are both available. On the first distinct collision, remove the tentatively accepted normalized slot and mark the group ambiguous; suppress all later members and emit one actionable diagnostic. Carry a separate evidence-present bit into `EvidenceTable` so collision-only pages are not classified as bot challenges. + +**Tech Stack:** Rust, Chromium GPT evidence model, built-in Rust test framework. + +--- + +### Task 1: Specify refusal behavior + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Add assertions for documented `DiscoveredSlots::had_slot_evidence` and run one focused test to observe the expected missing-field compile failure. +- [ ] Add only the documented field with its derived/default false value so behavioral tests can compile; do not wire discovery or classification yet. +- [ ] Change the same-page collision test to require zero emitted slots and one refusal diagnostic. +- [ ] Assert the diagnostic names `ad-in_content`, explains that a broad prefix resolves only one element and raw IDs are volatile, and tells the operator to expose distinct stable IDs. +- [ ] Rename the test to `same_page_hex_normalization_collision_is_refused`. +- [ ] Extend `repeated_raw_div_after_a_normalization_collision_is_deduplicated` with repeats of both initial raw IDs and a third distinct ID; require zero slots and one diagnostic. +- [ ] Add `request_normalization_collision_is_refused`; require zero slots, one diagnostic with the same prefix/safety/action content, true evidence, and a surviving request-derived network ID. +- [ ] Add `ambiguous_registry_stem_still_suppresses_request_fallback` and require no slot resurrection. +- [ ] Require every registry/request collision test to assert `had_slot_evidence` is true. +- [ ] Add `collision_only_page_is_not_classified_as_empty` using a discovered collision result. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin same_page_hex_normalization_collision_is_refused` and confirm RED because two raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin repeated_raw_div_after_a_normalization_collision_is_deduplicated` and confirm RED because raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin request_normalization_collision_is_refused` and confirm RED because request-derived raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm RED because the ambiguous registry group remains deployable. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm RED because collision-only evidence is classified as empty. + +### Task 2: Refuse ambiguous collision groups + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Replace raw-ID preservation with a collision result that distinguishes first ambiguity from later members. +- [ ] Remove the initially accepted normalized slot when ambiguity is first proven. +- [ ] Suppress the colliding and subsequent raw members. +- [ ] Emit one message naming the prefix, both unsafe representations, and the publisher-markup action. +- [ ] Set `had_slot_evidence` for any otherwise usable registry/request candidate and use it in empty-page classification. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin normalization_collision` and confirm the registry and request collision tests GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm registry precedence GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm GREEN. + +### Task 3: Verify and deliver + +**Files:** + +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Verify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Verify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` + +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `./scripts/test-cli.sh aarch64-apple-darwin`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings`. +- [ ] Run `cd docs && npm run format`. +- [ ] Run `git diff --check` and inspect the scoped diff. +- [ ] Commit and push the fix to `feature/ts-cli-ad-templates`. + +### Task 4: Refuse a known single-observation volatile family + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] Add failing registry and request tests for a single `__` observation. +- [ ] Add a recognizer keyed on the token shape — ten or more leading digits followed by more alphanumerics — in any position that has a non-empty family prefix before it and placement content after it. +- [ ] Omit matching slots while preserving evidence/network discovery and emit one deduplicated actionable diagnostic naming the family prefix. +- [ ] Add negative tests proving IDs with no token, a bare digit run, or a trailing token remain eligible. +- [ ] Run the focused tests, then repeat Task 3 verification and delivery. diff --git a/docs/superpowers/plans/2026-08-19-trusted-client-ip-header.md b/docs/superpowers/plans/2026-08-19-trusted-client-ip-header.md new file mode 100644 index 000000000..5bb7ffe62 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-trusted-client-ip-header.md @@ -0,0 +1,719 @@ +# Trusted Client IP Header Implementation Plan + +## Review remediation + +- [x] Add a shared helper that removes the configured IP and authentication + headers from a core request, with a failing unit test proving both names + are removed while unrelated headers remain. +- [x] Invoke that helper from the outer request middleware in the Axum, + Cloudflare, and Spin adapters so a shared multi-adapter configuration + cannot expose either trust header to routing or integrations. +- [x] Add a regression assertion proving the Fastly entry-point `ClientInfo` + address reaches request-scoped services, which are the EC input. +- [x] Remove the partial `X-Forwarded-For` hardening from this PR and its + documentation. Handle trusted XFF reconstruction consistently across all + adapters in a separate change. +- [x] Document that redaction protects logs and debug output but the secret is + serialized into the Trusted Server application-config blob. +- [x] Retain `/.worktrees/` because this checkout has an active unrelated + worktree under that path; removing the ignore would expose its contents. +- [x] Run formatting plus Fastly, Axum, Cloudflare, and Spin tests and + target-matched Clippy checks. + +## Review follow-up: header-safe shared secrets + +PR review found that the request path reads the authentication field with +`HeaderValue::to_str`, while configuration originally accepted any string of at +least 32 UTF-8 bytes. A non-ASCII secret could therefore pass startup validation +but never authenticate a request. Whitespace accepted by `HeaderValue::to_str` +is also unsuitable because an intermediary may normalize it. + +- [x] Add focused settings tests for the 31/32-byte boundary and rejection of + non-ASCII, horizontal-tab, space, DEL, and other control bytes. Assert + rejected values remain redacted, and run the tests first to demonstrate + the current failure. +- [x] Accept only `shared_secret` bytes in the ASCII graphic range + `0x21..=0x7e`, retaining the 32-byte minimum and redacted errors. +- [x] Update the configuration and Fastly guides to specify 32 or more ASCII + graphic bytes and recommend hexadecimal or base64url generation. +- [x] Run focused settings tests, formatting, target-matched tests, Clippy, and + the Fastly release build before updating the PR. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve the reader IP behind an authenticated fronting CDN while preserving peer-IP fallback and stripping all trust headers before routing. + +**Architecture:** Add an optional validated `TrustedClientIpConfig` to core settings, including fixed-size constant-time shared-secret verification. The Fastly adapter resolves the address exactly once from the original request, removes both static and configured spoofable headers, and shares the selected address with `ClientInfo` and response geo finalization. + +**Tech Stack:** Rust 2024, Serde/TOML, validator, `sha2`, `subtle`, Fastly Compute SDK, Viceroy, Markdown/VitePress documentation. + +--- + +## File Map + +| File | Responsibility | +| ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/settings.rs` | Define, deserialize, validate, redact, and authenticate the optional trusted-client-IP configuration. | +| `crates/trusted-server-core/src/http_util.rs` | Treat `Fastly-Client-IP` as spoofable unless it was consumed before sanitization. | +| `crates/trusted-server-adapter-fastly/src/platform.rs` | Resolve one authenticated header value or fall back to the SDK peer IP; construct `ClientInfo` with the resolved value. | +| `crates/trusted-server-adapter-fastly/src/compat.rs` | Remove configured trust headers and the static spoofable-header set from the Fastly request. | +| `crates/trusted-server-adapter-fastly/src/main.rs` | Resolve once before sanitization and feed the same IP into request services and geo finalization. | +| `trusted-server.example.toml` | Show the optional configuration with fictional values. | +| `docs/guide/configuration.md` | Document fields, validation, fallback, and environment overrides. | +| `docs/guide/fastly.md` | Document the front-door overwrite requirement and no-code routing limitation. | + +### Task 1: Add the validated trusted-client-IP configuration + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs:1-20` +- Modify: `crates/trusted-server-core/src/settings.rs:1871-1950` +- Test: `crates/trusted-server-core/src/settings.rs:2590-end` + +- [ ] **Step 1: Write parsing and default-behavior tests** + +Add tests beside the existing settings tests. Build TOML from +`crate_test_settings_str()` and append: + +```rust +#[test] +fn trusted_client_ip_is_absent_by_default() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without trusted client IP"); + + assert!( + settings.trusted_client_ip.is_none(), + "should leave trusted client IP disabled by default" + ); +} + +#[test] +fn trusted_client_ip_parses_and_redacts_secret() { + let toml = format!( + "{}\n[trusted_client_ip]\nip_header = \"fastly-client-ip\"\nauth_header = \"x-ts-client-ip-auth\"\nshared_secret = \"unit-test-shared-secret-0123456789\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml) + .expect("should parse trusted client IP settings"); + let config = settings + .trusted_client_ip + .as_ref() + .expect("should contain trusted client IP settings"); + + assert_eq!(config.ip_header, "fastly-client-ip"); + assert_eq!(config.auth_header, "x-ts-client-ip-auth"); + assert_eq!( + format!("{:?}", config.shared_secret), + "[REDACTED]", + "should redact shared secret" + ); +} +``` + +- [ ] **Step 2: Run the new tests and verify RED** + +Run: + +```bash +cargo test --package trusted-server-core --target aarch64-apple-darwin trusted_client_ip -- --nocapture +``` + +Expected: compilation fails because `Settings::trusted_client_ip` and +`TrustedClientIpConfig` do not exist. + +- [ ] **Step 3: Add the minimum serializable settings shape** + +Add the imports needed for fixed-size digest comparison: + +```rust +use sha2::{Digest as _, Sha256}; +use subtle::ConstantTimeEq as _; +``` + +Define the configuration immediately before `Settings`. Do not add the schema +validator attribute until Step 6, so the intermediate RED build remains valid: + +```rust +/// Authenticated client-IP forwarding configuration for a trusted front door. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct TrustedClientIpConfig { + /// Request header containing exactly one forwarded IPv4 or IPv6 address. + pub ip_header: String, + /// Request header containing the shared authentication secret. + pub auth_header: String, + /// Shared secret overwritten by the trusted front door on every request. + #[validate(custom(function = validate_redacted_not_empty))] + pub shared_secret: Redacted, +} + +impl TrustedClientIpConfig { + /// Minimum accepted authentication secret length in ASCII graphic bytes. + const MIN_SHARED_SECRET_LENGTH: usize = 32; + + /// Compares a request authentication value with the configured secret. + #[must_use] + pub fn authenticates(&self, candidate: &str) -> bool { + let expected = Sha256::digest(self.shared_secret.expose().as_bytes()); + let actual = Sha256::digest(candidate.as_bytes()); + bool::from(expected.ct_eq(&actual)) + } +} +``` + +Add the optional nested field to `Settings`: + +```rust +/// Optional authenticated client-IP forwarding configuration. +#[serde(default)] +#[validate(nested)] +pub trusted_client_ip: Option, +``` + +- [ ] **Step 4: Add fail-closed configuration tests** + +Use a small helper that appends a `[trusted_client_ip]` block to the standard +test TOML. Add individual tests proving that parsing/validation rejects: + +```rust +#[test] +fn trusted_client_ip_rejects_identical_headers() { /* same name */ } + +#[test] +fn trusted_client_ip_rejects_case_insensitive_identical_headers() { + /* x-client-ip and X-Client-IP */ +} + +#[test] +fn trusted_client_ip_rejects_unsafe_ip_header() { /* ip_header = "host" */ } + +#[test] +fn trusted_client_ip_rejects_unsafe_auth_header() { /* auth_header = "authorization" */ } + +#[test] +fn trusted_client_ip_rejects_fastly_tls_bridge_headers() { + // Exercise x-ts-tls-protocol and x-ts-tls-cipher in both positions. +} + +#[test] +fn trusted_client_ip_rejects_empty_secret() { /* shared_secret = "" */ } + +#[test] +fn trusted_client_ip_rejects_a_31_byte_secret() { /* shared_secret = 31 * "a" */ } + +#[test] +fn trusted_client_ip_accepts_a_32_byte_ascii_graphic_secret() { + /* shared_secret = 32 * "a" */ +} + +#[test] +fn trusted_client_ip_rejects_non_header_safe_secrets_without_exposing_them() { + // Exercise a >=32-byte non-ASCII value, HTAB, space, DEL, and another + // control byte. Assert each validation error omits the rejected value. +} + +#[test] +fn trusted_client_ip_rejects_malformed_header_names() { /* spaces/control bytes */ } + +#[test] +fn trusted_client_ip_rejects_incomplete_section() { /* omit each required field */ } + +#[test] +fn trusted_client_ip_authentication_is_exact() { + let config = trusted_client_ip_test_config(); + assert!(config.authenticates("unit-test-shared-secret-0123456789")); + assert!(!config.authenticates("wrong-secret")); + assert!(!config.authenticates(" unit-test-shared-secret-0123456789")); +} +``` + +Each rejection assertion must check that `Settings::from_toml` returns an error, +not merely that `validate()` fails on a manually constructed value. + +- [ ] **Step 5: Run the validation tests and verify RED** + +Run the same filtered core test command. Expected: parsing tests pass, while the +unsafe/identical-header tests fail because cross-field validation is not yet +implemented. + +- [ ] **Step 6: Implement header-name and shared-secret validation** + +Add `#[validate(schema(function = "validate_trusted_client_ip_config"))]` to +`TrustedClientIpConfig`, then add helpers that parse names through +`http::HeaderName`, compare normalized lowercase names, and enforce the spec: + +```rust +fn validate_trusted_client_ip_config( + config: &TrustedClientIpConfig, +) -> Result<(), ValidationError> { + let ip_header = http::HeaderName::from_bytes(config.ip_header.as_bytes()) + .map_err(|_| ValidationError::new("invalid_trusted_client_ip_header"))?; + let auth_header = http::HeaderName::from_bytes(config.auth_header.as_bytes()) + .map_err(|_| ValidationError::new("invalid_trusted_client_ip_auth_header"))?; + + if ip_header == auth_header { + return Err(ValidationError::new("duplicate_trusted_client_ip_headers")); + } + if ip_header.as_str() != "fastly-client-ip" && !ip_header.as_str().starts_with("x-") { + return Err(ValidationError::new("unsafe_trusted_client_ip_header")); + } + if !auth_header.as_str().starts_with("x-") { + return Err(ValidationError::new("unsafe_trusted_client_ip_auth_header")); + } + for reserved in ["x-ts-tls-protocol", "x-ts-tls-cipher"] { + if ip_header.as_str() == reserved || auth_header.as_str() == reserved { + return Err(ValidationError::new("reserved_trusted_client_ip_header")); + } + } + let shared_secret = config.shared_secret.expose().as_bytes(); + if shared_secret.len() < TrustedClientIpConfig::MIN_SHARED_SECRET_LENGTH { + return Err(ValidationError::new("short_trusted_client_ip_shared_secret")); + } + if !shared_secret.iter().all(|byte| matches!(byte, b'!'..=b'~')) { + return Err(ValidationError::new("invalid_trusted_client_ip_shared_secret")); + } + Ok(()) +} +``` + +Use descriptive validation messages if the validator API allows them without +duplicating logic. Do not include `shared_secret` in errors. Test the 31/32-byte +boundary plus non-ASCII, horizontal tab, space, DEL, and another control byte. + +- [ ] **Step 7: Run the filtered core tests and verify GREEN** + +Run the filtered command from Step 2. Expected: all `trusted_client_ip` tests +pass with no warnings. + +- [ ] **Step 8: Run the complete native core suite** + +Run: + +```bash +cargo test --package trusted-server-core --target aarch64-apple-darwin +``` + +Expected: PASS. + +- [ ] **Step 9: Commit the settings contract** + +```bash +git add crates/trusted-server-core/src/settings.rs +git commit -m "Add trusted client IP configuration" +``` + +### Task 2: Resolve and sanitize the authenticated Fastly client IP + +**Files:** + +- Modify: `crates/trusted-server-core/src/http_util.rs:35-65` +- Modify: `crates/trusted-server-adapter-fastly/src/platform.rs:690-730` +- Modify: `crates/trusted-server-adapter-fastly/src/compat.rs:50-105` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs:115-165` +- Test: `crates/trusted-server-adapter-fastly/src/platform.rs:730-end` +- Test: `crates/trusted-server-adapter-fastly/src/compat.rs:65-110` + +- [ ] **Step 1: Write resolver tests for the desired behavior** + +In `platform.rs`, add a test helper returning a `TrustedClientIpConfig` with +fictional headers/secrets, plus focused tests for: + +```rust +#[test] +fn resolve_client_ip_uses_peer_without_config() { /* None config */ } + +#[test] +fn resolve_client_ip_accepts_authenticated_ipv4() { /* 198.51.100.7 */ } + +#[test] +fn resolve_client_ip_accepts_authenticated_ipv6() { /* 2001:db8::7 */ } + +#[test] +fn resolve_client_ip_falls_back_for_missing_authentication() { /* no auth */ } + +#[test] +fn resolve_client_ip_falls_back_for_empty_authentication() { /* auth = "" */ } + +#[test] +fn resolve_client_ip_falls_back_for_wrong_authentication() { /* wrong auth */ } + +#[test] +fn resolve_client_ip_falls_back_for_duplicate_authentication() { + // append_header twice; do not use set_header +} + +#[test] +fn resolve_client_ip_falls_back_for_missing_ip_header() { /* valid auth only */ } + +#[test] +fn resolve_client_ip_falls_back_for_invalid_ip_forms() { + // Separate assertions for whitespace, port, IPv6 zone, comma list, empty. +} + +#[test] +fn resolve_client_ip_falls_back_for_duplicate_ip_headers() { + // append_header twice. +} +``` + +Pass a documentation address such as `203.0.113.9` as the explicit peer value +so unit tests do not depend on SDK connection metadata. Build non-UTF-8 cases +with `fastly::http::HeaderValue::from_bytes` and test both auth and IP fields. + +- [ ] **Step 2: Run the Fastly resolver tests and verify RED** + +Run: + +```bash +cargo test-fastly resolve_client_ip -- --nocapture +``` + +Expected: compilation fails because `resolve_client_ip` does not exist. If +Viceroy again fails before executing tests because the macOS native certificate +keychain is unavailable, record that environmental blocker and still require +the compile phase to succeed after implementation. + +- [ ] **Step 3: Implement a single-value reader and resolver** + +Import `TrustedClientIpConfig` and add: + +```rust +fn single_header_str<'a>(req: &'a Request, name: &str) -> Option<&'a str> { + let mut values = req.get_header_all(name); + let value = values.next()?; + if values.next().is_some() { + return None; + } + value.to_str().ok() +} + +/// Selects an authenticated forwarded address or the immediate peer fallback. +#[must_use] +pub(crate) fn resolve_client_ip( + req: &Request, + peer_ip: Option, + config: Option<&TrustedClientIpConfig>, +) -> Option { + let Some(config) = config else { + return peer_ip; + }; + let Some(auth) = single_header_str(req, &config.auth_header) else { + return peer_ip; + }; + if !config.authenticates(auth) { + return peer_ip; + } + single_header_str(req, &config.ip_header) + .and_then(|value| value.parse::().ok()) + .or(peer_ip) +} +``` + +Keep failures non-fatal and do not log header values. If debug logs are added, +log only a fixed reason category. + +- [ ] **Step 4: Pass the resolved value into `ClientInfo`** + +Change the constructor signature and assignment: + +```rust +pub fn client_info_from_request(req: &Request, client_ip: Option) -> ClientInfo { + ClientInfo { + client_ip, + // existing TLS/JA4/server fields unchanged + } +} +``` + +Add a direct unit test proving a supplied address is preserved even though a +synthetic Fastly request has no client connection metadata. This is the +regression test for shared geo/`ClientInfo` wiring: Step 9 derives the geo input +back from this constructed `ClientInfo`, rather than retaining an independent +parallel value. + +In `main.rs`, update the existing call immediately to pass the already-captured +peer value: + +```rust +let client_info = client_info_from_request(&req, client_ip); +``` + +This is a compile-preserving signature migration only; behavior is still the +old peer-IP behavior until Step 9 wires the resolver. + +- [ ] **Step 5: Write the static sanitization test and verify RED** + +Extend `compat.rs` tests using the function's current one-argument signature: + +```rust +#[test] +fn sanitize_fastly_forwarded_headers_strips_fastly_client_ip_without_config() { + // Set Fastly-Client-IP, sanitize, assert absent. +} +``` + +Run: + +```bash +cargo test-fastly sanitize_fastly_forwarded_headers -- --nocapture +``` + +Expected: the new test fails because `Fastly-Client-IP` is still preserved. + +- [ ] **Step 6: Strip the static header and migrate the helper signature** + +Add `"fastly-client-ip"` to `SPOOFABLE_FORWARDED_HEADERS`. Change the Fastly +compatibility helper to accept an intentionally unused +`Option<&TrustedClientIpConfig>` while preserving its current static loop: + +Import `trusted_server_core::settings::TrustedClientIpConfig` in `compat.rs`. + +```rust +pub(crate) fn sanitize_fastly_forwarded_headers( + req: &mut fastly::Request, + _config: Option<&TrustedClientIpConfig>, +) { + // Existing static loop unchanged. +} +``` + +Update existing compat test calls and the entry-point call to pass `None`. Run +the Step 5 command again. Expected: PASS (or the documented Viceroy environment +failure after successful compilation). This leaves the code green before the +next behavior test. + +- [ ] **Step 7: Write configured sanitization tests and verify RED** + +Now that the two-argument API compiles, add: + +```rust +#[test] +fn sanitize_fastly_forwarded_headers_strips_configured_trust_headers() { + // Set custom IP and auth fields, sanitize with Some(config), assert absent. +} +``` + +Also cover a configuration whose `ip_header` is `fastly-client-ip` to prove +double removal is harmless. Run the focused sanitization command. Expected: the +custom-header test fails because the config argument is not yet consumed. + +- [ ] **Step 8: Implement configured sanitization and verify GREEN** + +Remove the configured IP/auth headers first, then loop over the static list. +Never read or log their values. Run the focused sanitization command again. +Expected: PASS (or the documented post-compilation Viceroy environment failure). + +- [ ] **Step 9: Wire one resolved address through the entry point** + +In `main.rs`, after `settings_snapshot` is created and before sanitization: + +Add `resolve_client_ip` to the existing `crate::platform` import. + +```rust +let trusted_client_ip = settings_snapshot + .as_deref() + .and_then(|settings| settings.trusted_client_ip.as_ref()); +let resolved_client_ip = resolve_client_ip( + &req, + req.get_client_ip_addr(), + trusted_client_ip, +); +compat::sanitize_fastly_forwarded_headers(&mut req, trusted_client_ip); +``` + +Remove the later direct `req.get_client_ip_addr()` capture. Pass +`resolved_client_ip` to `client_info_from_request`, then derive the geo/finalize +input from that object: + +```rust +let client_info = client_info_from_request(&req, resolved_client_ip); +let client_ip = client_info.client_ip; +``` + +Leave both calls to `apply_entry_point_finalize_headers(..., client_ip)` +unchanged. This creates one stored source of truth: the address inserted into +request services and the address passed to response geo are both read from the +same `ClientInfo` construction. The `client_info_from_request` preservation test +from Step 4 proves that selected addresses cross this boundary unchanged. + +The startup-error path has no settings snapshot, so it passes `None`, uses the +peer address, and still strips `Fastly-Client-IP` through the static list. + +- [ ] **Step 10: Run focused tests and verify GREEN** + +Run: + +```bash +cargo test-fastly resolve_client_ip -- --nocapture +cargo test-fastly sanitize_fastly_forwarded_headers -- --nocapture +``` + +Expected: PASS when Viceroy can run. On the known keychain failure, require both +commands to compile the Fastly Wasm test binary successfully before the same +external Viceroy startup error. + +- [ ] **Step 11: Verify Fastly compilation** + +Run: + +```bash +cargo check-fastly +``` + +Expected: PASS with no warnings. + +- [ ] **Step 12: Commit the Fastly behavior** + +```bash +git add \ + crates/trusted-server-core/src/http_util.rs \ + crates/trusted-server-adapter-fastly/src/platform.rs \ + crates/trusted-server-adapter-fastly/src/compat.rs \ + crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Resolve authenticated forwarded client IP" +``` + +### Task 3: Document secure front-door configuration + +**Files:** + +- Modify: `trusted-server.example.toml:12-20` +- Modify: `docs/guide/configuration.md:50-90` +- Modify: `docs/guide/fastly.md:50-135` + +- [ ] **Step 1: Add the commented example configuration** + +Add after `[publisher]` in `trusted-server.example.toml`: + +```toml +# Optional: trust a fronting CDN's reader IP only when it also supplies the +# matching shared secret. The front door must overwrite both headers. +# [trusted_client_ip] +# ip_header = "fastly-client-ip" +# auth_header = "x-ts-client-ip-auth" +# shared_secret = "replace-with-a-random-shared-secret" +``` + +- [ ] **Step 2: Add the configuration reference** + +Add `[trusted_client_ip]` to the key-sections table and document: + +- all three fields and their required status when the section exists; +- absence preserving peer-IP behavior; +- exact-one-value parsing and fail-to-peer behavior; +- safe header-name restrictions; +- `TRUSTED_SERVER__TRUSTED_CLIENT_IP__*` override names; +- redaction and random-secret requirements. + +Use only fictional `example.com` domains and documentation IP ranges. + +- [ ] **Step 3: Add Fastly front-door instructions** + +Add a "CDN-fronted client IP" section explaining: + +1. The public front door must overwrite the configured IP and auth headers on + every backend request; preserving browser-supplied values is unsafe. +2. For VCL/Fastly chaining, overwrite `Fastly-Client-IP` from the initial + `client.ip` and set the authentication header to the shared secret. +3. Configure the identical header names and secret in Trusted Server. +4. Direct or unauthenticated requests fall back to the immediate peer. +5. Fastly no-code request routing has no header-injection point; if it does not + preserve the reader IP, this mechanism cannot recover it. + +Link to Fastly's official `client.ip`, `Fastly-Client-IP`, and service-chaining +documentation. Do not include a deployable real secret. + +- [ ] **Step 4: Format and inspect documentation** + +Run: + +```bash +(cd docs && npm run format) +git diff --check +git diff -- trusted-server.example.toml docs/guide/configuration.md docs/guide/fastly.md +``` + +Expected: formatting succeeds; only the intended example and guide sections +change; no whitespace errors. + +- [ ] **Step 5: Commit documentation** + +```bash +git add trusted-server.example.toml docs/guide/configuration.md docs/guide/fastly.md +git commit -m "Document trusted client IP forwarding" +``` + +### Task 4: Run final verification and review + +**Files:** + +- Verify: all files changed by Tasks 1-3 + +- [ ] **Step 1: Run formatting checks** + +```bash +cargo fmt --all -- --check +(cd docs && npm run format) +``` + +Expected: PASS with no further diffs after formatting. + +- [ ] **Step 2: Run target-matched tests** + +```bash +cargo test --package trusted-server-core --target aarch64-apple-darwin +cargo test-fastly +``` + +Expected: all core and Fastly tests pass. If Viceroy remains blocked by the +native certificate keychain, preserve the complete error output and separately +confirm `cargo check-fastly` succeeds; do not report `cargo test-fastly` as +passing. + +- [ ] **Step 3: Run target-matched compilation and linting** + +```bash +cargo check-fastly +cargo clippy-fastly +``` + +Expected: PASS with warnings denied by the repository alias. + +- [ ] **Step 4: Run unaffected-adapter regression tests required by CI** + +Because `Settings` and the shared spoofable-header list are in core, run: + +```bash +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: PASS. + +- [ ] **Step 5: Review the final diff for security invariants** + +Confirm from the diff that: + +- no code trusts a forwarded IP without successful authentication; +- duplicate or non-UTF-8 fields cannot be accepted through a first-value API; +- no log or error formats `shared_secret` or request header values; +- both configured trust fields are removed before conversion/routing; +- `ClientInfo` and response geo receive the same resolved address; +- configuration absence preserves peer-IP behavior. + +- [ ] **Step 6: Commit any verification-only corrections** + +If verification required code changes, repeat the narrow failing test first, +then commit only the correction with an imperative sentence-case message. If no +changes were needed, do not create an empty commit. + +- [ ] **Step 7: Request final code review** + +Use `superpowers:requesting-code-review` against the branch diff from `main` and +address any correctness or security findings before handoff. diff --git a/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md new file mode 100644 index 000000000..9f3735d91 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md @@ -0,0 +1,307 @@ +# PR 823 Round-5 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4989897698` while preserving generation compatibility and enforcing root-less template safety. + +**Architecture:** Keep browser-option defaults and legacy clap compatibility at the CLI boundary, carry borrowed-root evidence through template inference, and reject unsafe overrides before rendering. Improve diagnostics and validation at their existing seams, then pin cross-language and documentation invariants with focused tests. + +**Tech Stack:** Rust 2024, clap 4 derive, `url`, `toml`, embedded JavaScript, mdBook/VitePress documentation. + +--- + +## File Map + +- `crates/trusted-server-cli/src/commands/audit/collector.rs`: generation browser default constants and option defaults. +- `crates/trusted-server-cli/src/commands/audit/mod.rs`: hidden legacy browser arguments, early TOML validation, conversion to generation arguments. +- `crates/trusted-server-cli/src/run.rs`: clap contract tests. +- `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: collector defaults and formatting. +- `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs`: borrowed-root inference metadata and root-gap refusal reasons. +- `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: redirect output, profile-scoped notes, merge-policy validation, explicit-pattern refusal. +- `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs`: timestamp-shaped volatile token recognition. +- `crates/trusted-server-cli/src/commands/audit/browser.rs`: Rust/JavaScript evidence-cap invariant test. +- `crates/trusted-server-cli/src/commands/audit/page.rs`: accurate final-URL/terminal-escaping test claims. +- `docs/guide/cli.md` and the volatile-collision design/plan: operator and historical documentation corrections. + +### Task 1: Restore generation browser defaults and legacy clap isolation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add failing clap and default tests** + +Add parser coverage proving that `ts audit --help` does not advertise generation +browser flags, `ts audit --chrome /tmp/chrome generate ...` is rejected, and the +legacy `ts audit --chrome ... --settle-max-ms ...` form still parses and +reaches `GenerateArgs`. Add a generation-option default assertion for 750 ms and +12,000 ms. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin run::tests::audit_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::tests::legacy_ -- --nocapture +``` + +Expected: the hidden/help and 12-second assertions fail on the current branch. + +- [ ] **Step 3: Implement one generation-default source and legacy mirror** + +Define generation-specific constants in `collector.rs` and use them in clap +attributes and `GenerateBrowserOpts::default`: + +```rust +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +``` + +Use those constants in `BrowserAuditCollector::default`. Replace the flattened +`GenerateBrowserOpts` under `LegacyGenerateArgs` with `LegacyBrowserOpts`, whose +seven fields each use `hide = true, requires = "legacy_url"`. Implement +`From<&LegacyBrowserOpts> for GenerateBrowserOpts` and use it in +`legacy_generate_args`. Add the missing blank line between collector methods. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run the Step 2 commands and the focused collector default test. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/run.rs +git commit -m "Preserve generation browser option contracts" +``` + +### Task 2: Enforce borrowed-root and merge-policy safety + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference and end-to-end tests** + +Add tests proving: + +- `InferenceOutcome` identifies `ad-sidebar` as borrowing the root witnessed by + another slot; +- explicit `--page-pattern` values cause `run_update_slots` to fail before the + source config changes when any rendered template borrowed the root; +- no-policy inference gives affected multi-path slots the root-witness reason; +- a configured `section_segment = 1` with no `section_root` refuses inferred + segment 0 when preserved `{section}` slots exist; +- the same segment, or an unset segment, allows adopting the inferred root. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::unit_template::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::merge_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::explicit_ -- --nocapture +``` + +Expected: borrowed stems are unavailable, explicit patterns are accepted, and +the configured-segment mismatch is accepted. + +- [ ] **Step 3: Carry borrowed stems and reject unsafe overrides** + +Add an ordered `borrowed_section_root: Vec` field to +`InferenceOutcome`. Populate it only when `RootUnwitnessed` successfully becomes +a template. Before building render slots, reject non-empty explicit patterns if +that vector is non-empty: + +```rust +return cli_error(format!( + "cannot apply --page-pattern to slot(s) {} because their {{section}} templates borrow section_root; remove --page-pattern so patterns can be derived from observed paths", + borrowed.join(", ") +)); +``` + +On the no-policy path, replace the generic multi-path refusal reason for +structurally valid root-unwitnessed slots with the specific missing-root-witness +reason. Preserve structural refusal reasons unchanged. + +Update `validate_merge_policy` so an explicit configured segment is compared +before the empty-root adoption return. Keep the guard limited to preserved +`{section}` slots and allow `--replace`. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Protect borrowed section templates during generation" +``` + +### Task 3: Make redirects, warnings, and config errors actionable + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` + +- [ ] **Step 1: Add failing diagnostic tests** + +Strengthen the HTTPS-upgrade assertion to require +`http://publisher.example/` and `https://publisher.example/`. Add a two-profile +warning test whose output names desktop and mobile separately. Add a malformed +whole-document TOML test while retaining tests for unknown valid settings and an +unreadable `[creative_opportunities]` section. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots_accepts_a_same_host_https_upgrade -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin profile_warning -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin creative_config -- --nocapture +``` + +- [ ] **Step 3: Implement scoped diagnostics and early parse failure** + +Render redirect endpoints as `origin.ascii_serialization() + path`. Thread the +profile label into `fold_collected`; keep the consent-stub warning global, label +page warnings/interstitials with path and profile, and retain the existing +site-wide discovery-warning dedupe. + +Replace `.ok()` in `creative_config` with an error mapping that identifies a +malformed existing TOML document and explains that generation did not start. +Continue parsing into `toml::Value`, not runtime `Settings`, so valid unknown +settings remain tolerated. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/mod.rs +git commit -m "Clarify audit generation diagnostics" +``` + +### Task 4: Pin detector and embedded-collector invariants + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` + +- [ ] **Step 1: Add failing invariant tests** + +Add a negative volatile-token test for `promo-20260820a-sidebar`, retain a +positive timestamp-shaped control with at least ten leading digits, and add the +embedded-JavaScript constant assertion: + +```rust +assert!( + AD_TEMPLATE_COLLECTOR_JS.contains(&format!( + "const __ts_max_entries = {MAX_EVIDENCE_ENTRIES}" + )), + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" +); +``` + +In the page summary test, assert the exact percent-encoded final URL line and +limit the raw-control assertion's comment to title and warning fields. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin per_render_token -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin evidence_entries -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin page_controlled_text -- --nocapture +``` + +- [ ] **Step 3: Tighten the token shape and correct the test claim** + +Require at least ten leading digits in `is_per_render_token`. Keep the rest of +the recognizer unchanged. Add the evidence-cap test and page assertion without +removing final-URL escaping. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/page.rs +git commit -m "Pin audit evidence recognition invariants" +``` + +### Task 5: Align documentation and local style + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Modify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] In the volatile-collision design example, remove the section-varying + sidebar from the list of omitted/explained slots because root-less templating + now writes it with a borrowed-root diagnostic. +- [ ] In the volatile-collision implementation plan, state that a recognized + render token must have a non-empty family prefix before it and placement + content after it; remove the broader "in any position" claim. +- [ ] Update the guide to say that a configured segment without a root is + preserved for existing templates, and document the explicit-pattern refusal + for borrowed-root slots. +- [ ] Add the missing `GenerateArgs.browser` doc comment, change the `expect` + message to the required `"should ..."` form, and retain the method-separation + blank line from Task 1. +- [ ] Run `cd docs && npm run format` and `cargo fmt --all -- --check`. +- [ ] Commit: + +```bash +git add docs/guide/cli.md docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +git commit -m "Align ad-template generation documentation" +``` + +### Task 6: Verify the complete review resolution + +**Files:** + +- Verify all files above. + +- [ ] Run focused audit generation tests: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate -- --nocapture +``` + +- [ ] Run the complete host CLI suite: + +```bash +./scripts/test-cli.sh aarch64-apple-darwin +``` + +- [ ] Run lint and formatting gates: + +```bash +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +cargo fmt --all -- --check +cd docs && npm run format +git diff --check +``` + +- [ ] Inspect `git status --short`, `git log --oneline -6`, and the complete + diff from `073d5644` to ensure only the approved review resolution is present. +- [ ] Do not push or post GitHub replies without separate user authorization. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md new file mode 100644 index 000000000..98af3e176 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -0,0 +1,842 @@ +# Server-Side Ad Template CLI Design + +**Date:** 2026-06-26 +**Status:** Draft design +**Scope:** Static and browser-backed CLI diagnostics for server-side ad templates + +## 1. Goal + +Add Trusted Server CLI support for server-side ad-template onboarding and +verification without resurrecting the stale standalone `ts-config` design. + +The CLI must answer two operator questions: + +1. Given an effective `trusted-server.toml`, which configured ad-template slots + match this path? +2. Given one or more live publisher URLs, are the configured slots for the + final navigated paths actually present on the page according to DOM, GPT, + and provider evidence, and do any runtime gates explain why Trusted Server + would not inject or auction for that page? + +The command surface is split by whether the command is local-config-only or +browser-backed: + +```bash +ts config ad-templates lint +ts config ad-templates match +ts config ad-templates check +ts config ad-templates explain + +ts audit ad-templates verify ... +``` + +Static commands live under `ts config` because they only load local effective +app config. Browser-backed verification lives under `ts audit` because it loads +public publisher pages in Chrome/Chromium and observes live page behavior. + +## 2. Context + +This design replaces the stale PR #724 direction. + +PR #724 designed a standalone `ts-config` binary around a +`creative-opportunities.toml` file. That is no longer the project shape: + +- Trusted Server configuration now flows through the unified `ts` CLI from PR + #799. +- Server-side ad-template slots live under `[creative_opportunities]` / + `[[creative_opportunities.slot]]` in `trusted-server.toml`. +- Effective config can include EdgeZero app-config environment overlays unless + `--no-env` is passed. +- Operator-owned `trusted-server.toml` is ignored; the repository tracks + `trusted-server.example.toml`. + +PR #799 is the CLI base. It owns the `ts` binary, EdgeZero lifecycle delegates, +and typed app-config validation/push/diff behavior. + +PR #800 is the audit dependency. It adds the generic browser-backed +`ts audit ` collector using local Chrome/Chromium. At the time this spec +was written, PR #800 was stale relative to the latest #799 head, so this work +depends on the #800 audit collector after it is rebased onto the latest #799 +typed blob-config model. + +## 3. Non-Goals + +- Do not add a standalone `ts-config` binary. +- Do not reintroduce `creative-opportunities.toml`. +- Do not implement browser-backed generation in Phase 1. +- Do not mutate `trusted-server.toml` from `verify`. +- Do not probe PBS, GAM, or APS management APIs. +- Do not require EdgeZero platform adapters for local static diagnostics. +- Do not make `ts audit ad-templates verify` push, provision, deploy, or update + platform resources. +- Do not rely on real GPT or APS network calls in tests. + +Browser-backed generation is a later phase: + +```bash +ts audit ad-templates generate ... +``` + +That phase needs separate rules for slot ID derivation, page-pattern inference, +multi-URL merging, TOML ordering, and whether the command emits a patch, a draft +file, or full config blocks. + +## 4. Command Surface + +### 4.1 Shared Config Flags + +All `ts config ad-templates ...` commands and +`ts audit ad-templates verify` accept the same local app-config flags: + +```bash +--app-config +--manifest +--no-env +``` + +Defaults match PR #799: + +| Option | Default | +| -------------- | ------------------------------------------------ | +| `--app-config` | `.toml`, resolved from `edgezero.toml` | +| `--manifest` | `edgezero.toml` | +| `--no-env` | `false`; app-config env overlay is applied | + +If an explicit `--app-config` path is supplied and missing, the command reports +that path as the error. It must not silently fall back to an environment or +manifest-derived path. + +### 4.2 Static Config Diagnostics + +```bash +ts config ad-templates lint [--app-config ] [--manifest ] [--no-env] +``` + +Reports whether `[creative_opportunities]` is configured, how many slots exist, +GAM network ID, auction timeout, auction enablement, configured auction +providers, and whether current EdgeZero routing will fall back to the legacy +path when configured slots are present. + +```bash +ts config ad-templates match [--details] ... +``` + +Normalizes a path or full URL to a path and reports the slots matched by the +runtime `creative_opportunities::match_slots` logic. `--details` includes slot +div ID, GAM unit path, page patterns, formats, and configured providers. + +```bash +ts config ad-templates check \ + (--expected-slot ... | --expect-no-slots) \ + [--allow-extra-slots] ... +``` + +CI-friendly assertion wrapper around the same matching logic. + +```bash +ts config ad-templates explain \ + [--method GET] \ + [--non-navigation] \ + [--prefetch] \ + [--bot] \ + [--consent-denied] \ + [--edgezero-enabled] ... +``` + +Explains the major runtime gates that decide whether the server-side ad stack +would run for a page request. This is a local model, not a live request replay. + +### 4.3 Browser-Backed Verification + +```bash +ts audit ad-templates verify ... \ + [--app-config ] \ + [--manifest ] \ + [--no-env] \ + [--strict] \ + [--json] \ + [--scroll] +``` + +Behavior: + +- Accept one or more `http` or `https` URLs. +- Reject all other schemes before launching a browser. +- Load the effective Trusted Server app config. +- For each URL, navigate first, collect the final URL, normalize the final URL + to a path, and call `creative_opportunities::match_slots`. +- Preserve the requested URL/path separately from the final URL/path. +- Emit a redirect warning when the final path differs from the requested path. +- Expect only the slots matched for the final URL path to be present on that + live page. +- Report live DOM/GPT/APS ad-slot evidence that does not correspond to a + matched configured slot as structured extra evidence. +- Launch Chrome/Chromium through the audit collector from the rebased #800 work. +- Inject a read-only ad-template collector before publisher scripts run. +- Compare configured matched slots against DOM, GPT, and APS evidence. +- Report runtime ad-stack gate evidence separately from placement evidence. +- Print human output by default. +- Emit stable machine-readable output with `--json`. +- Exit `0` by default for missing or partial live evidence; this is an + auditor-assist mode. +- Exit non-zero under `--strict` when a matched configured slot is missing or + only partially confirmed. + +`--scroll` performs a deterministic scroll pass after initial load and settle. +It is opt-in because it is slower and can trigger additional page behavior. +Slots first observed during scroll count as confirmed when the GPT evidence is +otherwise sufficient. + +## 5. Confirmation Model + +The verifier compares configured expected slots to live page evidence. + +It must keep three concepts separate: + +1. **Static slot matching:** which configured slots match a URL path according + to `creative_opportunities::match_slots`. +2. **Runtime ad-stack eligibility:** whether Trusted Server would run its + server-side ad stack for the audited navigation. This mirrors + `should_run_server_side_ad_stack` for the initial publisher request and the + `/__ts/page-bids` kill-switch/consent behavior for SPA route updates. +3. **Live placement evidence:** what the browser actually observes on the + rendered page through DOM, GPT, and APS evidence. + +`verify` is primarily a live placement verifier. `--strict` fails when matched +configured slots for an eligible page are missing or partial. Runtime gates are +reported so operators can distinguish "the slot is not on the page" from "the +current request/config would intentionally suppress Trusted Server ad-template +injection or page-bids slot output". + +### 5.1 Expected Slots + +For each input URL: + +1. Navigate the browser to the requested URL. +2. Record `requested_url`, `requested_path`, `final_url`, and `final_path`. +3. Match configured slots through the core runtime matcher using `final_path`. +4. Build an expected-slot record for each matched slot: + - slot ID; + - resolved div ID; + - resolved GAM unit path; + - configured formats; + - configured providers; + - matching page patterns. + +Only these expected slots are verified for that page. For example, slots whose +only pattern is `/` are expected for the homepage path, not for `/news/story`. + +When a navigation redirects, `verify` uses the final path for expected slots and +reports the requested path in output. This matches runtime behavior: Trusted +Server evaluates the actual publisher request path it handles, not the URL the +operator typed before redirects. + +### 5.2 Runtime Gate Evidence + +For each page result, `verify` reports a local runtime-gate model: + +| Gate | Source | +| ------------------------ | -------------------------------------------------------------------------------------------------------- | +| `method_get` | Browser navigation request; expected to pass for normal `verify`. | +| `navigation` | Browser navigation request; expected to pass for normal `verify`. | +| `not_prefetch` | Browser request headers; expected to pass unless the collector is extended with prefetch simulation. | +| `not_bot` | Browser User-Agent checked against the runtime bot fragments. | +| `matched_slots` | Final-path slot matching. | +| `auction_enabled` | Effective `[auction].enabled` / orchestrator enablement from app config. | +| `consent_allows_auction` | `unknown` unless the collector can prove a consent-allowed or consent-denied state for the live request. | + +`runtime_ad_stack_expected` is a three-state value: `yes`, `no`, or `unknown`. +Known blocking gates produce page warnings and set +`runtime_ad_stack_expected = "no"`. Unknown gates set +`runtime_ad_stack_expected = "unknown"` but do not by themselves fail +`--strict`. + +If `runtime_ad_stack_expected = "no"` because of a known config/request gate +such as `[auction].enabled = false`, strict mode does not fail missing GPT/APS +evidence for that page. The page result is reported as skipped for runtime +verification while still showing the static expected slots and any live +placement evidence that was observed. + +If `runtime_ad_stack_expected = "yes"` or `"unknown"`, strict mode applies the +normal missing/partial placement rules from §5.6. + +For SPA routes, `/__ts/page-bids` returns no slots when the ad-stack kill switch +or consent gate blocks the stack. Browser verification should report observed +page-bids responses when available, but it must not require real partner bids in +tests. + +Live ad-slot evidence that does not map to a matched expected slot is reported +as structured extra evidence. Extra evidence can identify publisher-owned slots +that have not yet moved into server-side ad templates, slots whose +`page_patterns` are too narrow, or slots that should stay outside Trusted +Server. It does not make `--strict` fail in Phase 1. + +### 5.3 DOM Slot Resolution + +The verifier must mirror the runtime GPT bootstrap's slot-root resolution: + +1. Try `document.getElementById(slot.div_id)`. +2. If absent, find the first element with an ID that starts with `slot.div_id`. +3. Ignore elements whose ID ends with `-container`. + +This is required because `div_id` may intentionally be a stable prefix for +framework-generated IDs, for example `ad-header-0-`. + +### 5.4 GPT Evidence + +A slot is confirmed by GPT evidence when the live page exposes a GPT slot whose: + +- ad unit path equals the configured resolved GAM unit path; +- slot element ID equals the resolved DOM element ID or an existing + `${resolved_dom_id}-container` element used by Trusted Server when defining + its own slot; +- configured sizes are compatible with the observed GPT sizes. + +The collector should observe both direct `googletag.defineSlot` calls and +post-load `googletag.pubads().getSlots()` state. + +Size compatibility is defined for Phase 1 as follows: + +- Normalize configured sizes from `CreativeOpportunityFormat` values where + `media_type = "banner"` into `(width, height)` pairs. +- Normalize observed GPT sizes from `defineSlot` input and `getSizes()` output: + - `[300, 250]` becomes one `(300, 250)` pair. + - `[[300, 250], [728, 90]]` becomes two pairs. + - non-numeric values such as `"fluid"` are ignored for numeric matching and + reported as warnings. +- A GPT slot's sizes are compatible when the configured banner size set and the + observed numeric GPT size set have at least one pair in common. +- Extra observed GPT sizes do not block confirmation, but they are reported as + warnings so auditors can decide whether to add formats to config. +- Configured banner sizes that are not observed do not block confirmation when + at least one configured size was observed, but they are reported as warnings. +- If ad unit path and div match but no numeric size overlap exists, the slot is + `partial`, not `confirmed`. +- Configured `video` and `native` formats are not used for Phase 1 GPT size + confirmation. A matched slot with only non-banner formats is `unconfirmable` + with an unsupported-format warning and does not fail `--strict`. +- A sizeless live GPT slot is `partial` when the config declares banner sizes, + because that is observable drift and must fail `--strict`. + +### 5.5 APS Evidence + +Phase 1 does not wrap or collect `apstag.fetchBids`: APS is server-side provider +configuration and client-side calls are neither required nor authoritative for +the runtime ad-template decision. + +### 5.6 Statuses + +| Status | Meaning | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | +| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | +| `missing` | No DOM or GPT evidence confirms the configured slot. | +| `unconfirmable` | The checker cannot evaluate the configured format with Phase 1 evidence, such as a non-banner-only slot. This is reported but does not fail strict mode. | + +In `--strict` mode: + +- `missing` fails. +- `partial` fails. +- `unconfirmable` does not fail. + +Provider issues are not statuses. They are warnings attached to the slot result. +For example, a slot can be `confirmed` and still carry a warning that configured +APS evidence was missing or ambiguous. Provider warnings do not fail `--strict` +unless a future `--strict-providers` flag is added. + +## 6. Architecture + +The architecture should keep command parsing thin and move ad-template behavior +into pure, testable modules. + +```text +crates/trusted-server-cli/src/ + app_config.rs + ad_templates/ + mod.rs + expected.rs + compare.rs + output.rs + config_ad_templates.rs + audit/ + page.rs + browser.rs + ad_templates.rs +``` + +### 6.1 `app_config.rs` + +Shared loader for effective Trusted Server app config. + +Responsibilities: + +- read `edgezero.toml` through EdgeZero manifest helpers; +- resolve the default `.toml` path; +- apply EdgeZero app-config env overlay unless `--no-env`; +- return `TrustedServerAppConfig` / `Settings`; +- report errors in the same terms as #799 config commands. + +This avoids duplicating config path and env-overlay behavior between +`ts config ad-templates ...` and `ts audit ad-templates verify`. + +The current branch already has a private loader in `config_ad_templates.rs`. +Before adding browser-backed verification, move that behavior into this shared +module and route the existing static commands through it so both command +families load the same effective config. + +### 6.2 `ad_templates::expected` + +Pure local expected-slot model. + +Responsibilities: + +- normalize path-or-URL input; +- call `creative_opportunities::match_slots`; +- convert matched slots into stable expected-slot structs; +- preserve deterministic ordering by slot order from config. + +This module must not compile glob patterns independently or duplicate matching +semantics. + +If richer pattern diagnostics are needed, add a small helper to +`trusted-server-core::creative_opportunities` and use it from both runtime and +CLI. + +### 6.3 `ad_templates::compare` + +Pure comparison between expected slots and collected browser evidence. + +Responsibilities: + +- implement DOM prefix matching rules; +- compare GPT path, div, and size evidence; +- compare APS evidence; +- collect unmatched live DOM/GPT/APS ad-slot evidence as structured + `extra_evidence`; +- assign `confirmed`, `partial`, `missing`, and provider warning details; +- decide strict failure status. + +This module should be testable without launching Chrome. + +### 6.4 `ad_templates::output` + +Human and JSON output model. + +Responsibilities: + +- serialize stable JSON output; +- keep arrays ordered by input URL, then configured slot order, then provider + name; +- render concise human summaries; +- avoid leaking page HTML, cookies, local storage, or arbitrary page data. + +### 6.5 `config_ad_templates.rs` + +Thin Clap adapter for `ts config ad-templates ...`. + +Responsibilities: + +- parse command arguments; +- call `app_config` and `ad_templates::expected`; +- delegate formatting to `ad_templates::output`; +- keep no browser-specific logic. + +### 6.6 `audit::browser` + +Shared browser utility extracted from or aligned with the rebased #800 audit +collector. + +Responsibilities: + +- locate Chrome/Chromium; +- launch an isolated profile; +- reject non-HTTP(S) URLs before navigation; +- set bounded navigation and settle timeouts; +- run optional init scripts; +- perform optional deterministic scroll; +- collect final URL, title, rendered scripts, resource entries, and optional + ad-template evidence. + +The generic `ts audit ` command from #800 should continue to work without +ad-template verification enabled. + +### 6.7 `audit::ad_templates` + +Browser-backed verifier orchestration. + +Responsibilities: + +- parse `ts audit ad-templates verify`; +- load effective config through `app_config`; +- compute expected slots for each URL; +- run the browser collector with ad-template evidence enabled; +- call `ad_templates::compare`; +- print human or JSON output; +- apply default auditor-assist exit behavior and `--strict` behavior. + +## 7. Browser Collector + +The ad-template collector is injected before page scripts run. It is read-only: +it records evidence and calls original page functions with unchanged arguments. + +The rebased #800 collector must grow a pre-navigation init-script hook before it +can satisfy this spec. The stale #800 collector only navigates, waits, and reads +post-load page state; that is insufficient for GPT/APS call evidence. + +Instrumentation requirements: + +- install the collector through the browser's "evaluate on new document" / + init-script mechanism before navigation; +- serialize only configured div prefixes and provider IDs needed for matching; +- observe pages that create `window.googletag = { cmd: [] }` after injection; +- wrap `googletag.cmd.push` callbacks without changing callback order; +- record direct `googletag.defineSlot` calls and calls executed from the GPT + command queue; +- read final `googletag.pubads().getSlots()` state after settle and after + scroll; +- observe pages that assign `window.apstag` after injection and wrap + `apstag.fetchBids` when present; +- tolerate pages that never load GPT or APS and report warnings instead of + throwing collector errors. + +Evidence to collect: + +- DOM elements with IDs relevant to configured slot div prefixes; +- calls to `googletag.defineSlot`; +- final `googletag.pubads().getSlots()` state after settle and after scroll; +- calls to `apstag.fetchBids`; +- timestamps or phases indicating whether evidence was observed during + `initial_load` or `scroll`. + +The collector must not: + +- block, rewrite, or suppress publisher scripts; +- override `navigator.webdriver`; +- capture cookies, local storage, session storage, request bodies, or arbitrary + page data; +- require real GPT/APS network calls in test fixtures. + +## 8. JSON Output Contract + +`--json` emits deterministic JSON. Shape: + +```json +{ + "ok": true, + "strict": false, + "pages": [ + { + "url": "https://www.example.com/news/story", + "final_url": "https://www.example.com/news/story", + "requested_path": "/news/story", + "path": "/news/story", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "atf", + "status": "confirmed", + "phase": "initial_load", + "configured": { + "div_id": "ad-atf-", + "gam_unit_path": "/123/news/atf", + "formats": [ + { "width": 300, "height": 250, "media_type": "banner" } + ], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-atf-0", + "gpt": { + "gam_unit_path": "/123/news/atf", + "div_id": "ad-atf-0", + "sizes": [[300, 250]] + } + }, + "warnings": [] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +Warning entries are objects with stable `code` and human-readable `message` +fields. Human output may print only the message. JSON consumers must not need to +parse warning strings. + +Extra live evidence is structured: + +```json +{ + "kind": "gpt", + "phase": "initial_load", + "dom_id": "ad-right-rail-0", + "gam_unit_path": "/123/publisher/right-rail", + "sizes": [[300, 250]], + "reason": "no_configured_slot_matched" +} +``` + +Allowed `kind` values for Phase 1 are `dom` and `gpt`. + +Strict-mode failures with page results use the same shape and set `ok` to +`false`. Example partial slot: + +```json +{ + "ok": false, + "strict": true, + "pages": [ + { + "url": "https://www.example.com/", + "final_url": "https://www.example.com/", + "requested_path": "/", + "path": "/", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "homepage-header", + "status": "partial", + "phase": "initial_load", + "configured": { + "div_id": "ad-header-0-", + "gam_unit_path": "/123/homepage/header", + "formats": [{ "width": 728, "height": 90, "media_type": "banner" }], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-header-0-_R_abc123", + "gpt": null + }, + "warnings": [ + { + "code": "dom_without_gpt", + "message": "DOM element matched, but no GPT slot evidence was observed" + } + ] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +For errors that occur before any page result can be produced, the command exits +non-zero and prints the normal CLI error. JSON error output can be added later +if the base CLI standardizes it. + +For multi-URL runs, browser/navigation failures after argument validation are +page-level failures when possible. The command continues to the remaining URLs, +sets top-level `ok` to `false`, and includes a page result: + +```json +{ + "url": "https://www.example.com/broken", + "final_url": null, + "requested_path": "/broken", + "path": null, + "error": { + "code": "navigation_failed", + "message": "failed to read main document navigation response" + }, + "slots": [], + "extra_evidence": [], + "warnings": [] +} +``` + +Invalid schemes are still rejected before browser launch for the whole command, +because they are argument errors rather than page collection results. + +## 9. Error Handling + +Static commands fail when: + +- config cannot be loaded; +- `[creative_opportunities]` is malformed; +- CLI assertions in `check` fail. + +Browser verification fails when: + +- config cannot be loaded; +- any URL is not HTTP(S); +- Chrome/Chromium cannot be found or launched; +- all navigations fail before any page result can be collected; +- at least one page-level error occurs in a multi-URL run; +- command output cannot be written; +- `--strict` is set, runtime verification is not skipped by a known gate, and + at least one matched slot is missing or partial. `unconfirmable` is excluded. + +Browser collection can still produce a page result with warnings when: + +- page settle times out; +- a navigation redirects before final URL matching; +- scroll evidence is incomplete; +- GPT is not loaded; +- extra live DOM/GPT ad-slot evidence has no matched configured slot; +- no slots match the URL. + +## 10. Testing + +Static tests: + +- parse every `ts config ad-templates` command; +- load temp `edgezero.toml` and temp `trusted-server.toml`; +- verify `--app-config`, `--manifest`, and `--no-env` behavior; +- verify `/`, `/news/*`, and full URL normalization behavior; +- verify `check` success and failure output. +- verify the existing static command loader uses the shared `app_config` module. + +Pure comparison tests: + +- exact DOM ID match; +- prefix DOM ID match for framework-generated suffixes; +- ignore `-container` elements; +- GPT confirms by GAM unit path, div ID, and compatible sizes; +- DOM-only creates `partial`; +- no DOM/GPT creates `missing`; +- APS match creates no provider warning; +- APS missing/ambiguous creates provider warnings; +- `--strict` fails only missing and partial slots. + +Browser fixture tests: + +- local HTML fixture with direct `googletag.defineSlot`; +- fixture using `googletag.cmd.push`; +- fixture assigning `window.googletag` after collector injection; +- fixture with delayed/lazy slot observed only with `--scroll`; +- fixture with APS `fetchBids`; +- fixture assigning `window.apstag` after collector injection; +- redirect fixture that matches expected slots on final path; +- multi-URL fixture where one URL fails and one URL returns page results; +- fixture where `[auction].enabled = false` reports runtime skipped instead of + strict missing-slot failure; +- invalid non-HTTP(S) URL rejection before browser launch; +- JSON contract tests for warning codes, `extra_evidence`, page errors, + deterministic ordering, `partial`, `missing`, and strict failures; +- fixture with no real GPT/APS network dependency. + +Verification commands: + +```bash +cargo test --workspace +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --package trusted-server-cli --target +``` + +## 11. Branch And PR Plan + +The implementation should not be built on stale #724. + +Recommended dependency order: + +1. Land or rebase PR #799 as the CLI base. +2. Rebase PR #800 onto the latest #799 head so `ts audit` uses the current typed + blob app-config model. +3. Harden and refactor the existing static `ts config ad-templates ...` + diagnostics on top of the current server-side ad-template branch and #799: + extract the private config loader into `app_config`, move pure expected-slot + logic into `ad_templates::expected`, and keep existing behavior covered by + tests. +4. Extend the rebased #800 collector with pre-navigation init scripts, + ad-template evidence hooks, optional scroll, page-level errors, and bounded + structured output. +5. Build `ts audit ad-templates verify` on top of that collector and the + server-side ad-template branch. +6. Keep `generate` for a separate Phase 2 spec and PR. + +If delivery needs to be split, static diagnostics can land before browser-backed +verification. Browser-backed verification should not duplicate the #800 browser +collector. + +## 12. CLI Namespace Decision + +`ts audit ad-templates verify` is the final command shape for browser-backed +ad-template verification. + +When this work is combined with the rebased #800 audit command, `ts audit` +should become a subcommand namespace: + +```bash +ts audit page +ts audit generate +ts audit ad-templates verify ... +``` + +The existing #800 `ts audit ` behavior should be preserved as a +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. + +Parsing contract: + +- `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. +- `ts audit ad-templates verify ...` is the canonical ad-template verifier. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. +- `ts audit ad-templates` must never be treated as a legacy URL positional. +- `ts audit page` without a URL must fail with the normal Clap missing-argument + error. + +Implementation shape: + +```rust +#[derive(Debug, clap::Args)] +struct AuditArgs { + #[command(subcommand)] + command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + legacy_url: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), +} +``` + +If Clap cannot enforce the optional-subcommand plus hidden positional contract +cleanly, implement a small custom dispatcher for the `audit` argv tail and test +it directly. Required parser tests: + +- `ts audit https://www.example.com/` dispatches to artifact generation; +- `ts audit page https://www.example.com/` dispatches to page audit; +- `ts audit ad-templates verify https://www.example.com/` dispatches to + ad-template verification; +- `ts audit ad-templates` does not parse as a URL; +- `ts audit ftp://www.example.com/` fails before browser launch. + +JSON error output is intentionally left to the broader CLI output contract. This +spec only standardizes successful verification result JSON and strict-mode +verification failure JSON where page results exist. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md new file mode 100644 index 000000000..031ecc50c --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -0,0 +1,864 @@ +# The Cacheable Root: Latency Diagnosis and Stage 0 Design + +_Filename retains its original `esi-` prefix; the commit history and every +cross-reference point at it. The subject moved, the path did not._ + +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · + +> **HISTORICAL RECORD — NOT THE CURRENT IMPLEMENTATION.** This document preserves the +> measurement and feasibility investigation. Every executable ESI tag, parser, and +> subrequest described below belongs to a rejected spike; do not use those sections to +> infer current runtime behavior. The final branch retains `assembly_mode = "esi"` only as +> the operator spelling for Fastly C2 plus exact byte-seam assembly. See +> [the merge-hardening design](./2026-08-12-1009-esi-merge-hardening-design.md). + +**Revised:** 2026-08-10 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3`. + +> ## ⚠️ Correction, 2026-08-10 — this document's original ESI verdict was wrong +> +> The first revision concluded that ESI was **structurally blocked**: that it +> presupposed a TS-owned template cache which did not exist, and that such a cache was +> in turn blocked on purge capability the platform did not offer. **Both claims are +> false**, and an external review was right to reject them. +> +> Verified against the pinned `fastly` 0.12.1: +> +> - **The cache boundary is native.** `fastly::cache::core` provides +> `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / +> `found()` to read them back, and `Transaction` with `must_insert()` for request +> collapsing. The two-stage design needs no separate KV or template service. +> - **Purge exists in-process.** `fastly::http::purge::purge_surrogate_key` purges from +> inside Compute; the management-API token scope cited in the original is irrelevant to +> it. Note which cache, though: `InsertBuilder::surrogate_keys([...])` is the **Core +> Cache** API and keys the transformed-template cache (C2). It does **not** key the HTTP +> read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied +> keys or the HTTP cache's own surrogate-key surface. +> - **The original pipeline ordering was backwards.** It said "order esi → lol*html, +> never the reverse." `lol_html` \_emits* the ESI include tags, so ESI must run after it. +> Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> +> The error was inspecting what this repository does and reporting it as what the +> platform permits — the same mistake this document criticises #1009 for making in the +> other direction. +> +> **ESI is therefore feasible and unvalidated, not rejected.** Validating it is +> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing +> and are **not** an answer to #1009. + +## Document map — read this first + +#1009 is answered across three documents, not one. This is the only place that says +which owns what. + +| Document | Owns | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | + +**If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, +[§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it +gets validated. Everything else here is Stage 0 and the latency analysis behind it. + +**Decision requested:** approve the four items in §1. + +> **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments +> so that cacheable publisher HTML is separated from per-user ad state, recovering a +> TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router +> publisher running on Fastly Compute. ESI can do this; whether it should is not settled +> here. Separately, the regression has a cheaper cause than the issue assumes. +> +> **This document deliberately carries no performance measurements.** Every conclusion +> below is derived from code at the pinned baseline, so it can be checked by reading the +> repository rather than by trusting a benchmark. Where a quantity is needed and unknown, +> it is named as unknown and [§3](#3-monday-morning) says how to obtain it. +> +> Terms used throughout: **the hold** = TS holding the HTTP response open at `` +> until the server-side auction (SSAT) resolves. **React #418** = the React +> hydration-mismatch error raised when `adInit()` mutates ad-slot subtrees during +> hydration; it is why bid application is deferred to `window.load`. It is a React error +> number, **not** a repository issue — the tracker is +> [#938](https://github.com/IABTechLab/trusted-server/issues/938). **The SSAT price +> defect** = a live mispricing bug named in #1009 (prices reading 100× high) — cited +> from #1009 and prior investigation, not re-verified here. + +--- + +## 1. Decision requested + +| # | Decision | Owner needed | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | + +Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ +doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower +detail than the work it recommends. + +--- + +## 2. Why — the three findings + +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits executable ESI +include tags into a shared template; `fastly::cache::core` stores that template; the +`esi` crate assembles per request on the way out. Everything that requires is +already a dependency. The real open questions are empirical, not architectural: does it +beat a plain client fetch by enough to justify a Fastly-only rendering path, and can +per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment +failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. + +Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it +is a per-platform accelerator rather than the architecture, and its maintenance cost +belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** +— bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP +could embed an ESI include targeting an arbitrary URL and make the edge fetch it. Details in +[Appendix E](#appendix-e--esi-notes-condensed). + +**The auction is already out of band; the hold is ~free.** It is dispatched _before_ +the origin fetch and does not block — dispatched at `publisher.rs:2751-2755`, sent at `:2870` — +with a 500 ms budget. The actual cost is `with_cache_bypass` +(`publisher.rs:2867`), +which forces every ad-eligible navigation to miss the Fastly readthrough cache. + +**The two fixes are multiplicative.** Removing the bypass alone lets the previously +hidden auction surface as the new bottleneck. Removing the hold alone changes nothing, +because the auction was never the bottleneck. **Shipping the hold removal without the +bypass removal will measure no improvement and will read as the effort having failed** — +the most likely way this work gets judged unfairly. + +**Ordering is established; magnitude is not.** The ordering above follows from code and +needs no measurement. The _size_ of the win does — and the one quantity it depends on, +the origin build time under `Pass`, has never been measured. #1009's timings do not +supply it: they compare cached fetches against each other, not against an origin build. +**Quote no figure to a publisher until §3 Step C runs.** Full reasoning in +[§6](#6-the-analysis). + +--- + +## 3. Monday morning + +Three checks, ordered cheapest-first. Each needs a named owner before starting. + +**Step A — origin `Vary` and cookie check (minutes for the first pass).** `curl` the +origin with and without `RSC`, `Next-Router-*`, and the experiment header; inspect `Vary`, +`Cache-Control`, and `Set-Cookie`. **This first pass yields a `PROVISIONAL PASS` only** — +it is not what gates the flip. A `FINAL PASS` additionally requires a real authenticated +session, Basic Auth through TS, the experiment variant, representative routes, and +cached-hit render attribution. Do the cheap pass first because it is the +cheapest thing that unblocks anything. + +**Step B — what consumes TS's own response headers (under a day).** Request a TS-served +path that already emits `public, s-maxage` +(`http_util.rs:294-311`) +twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b +split** — see [§7](#7-deferred-work-specified-not-scheduled). + +**Step C — measure the hold directly (1 day + a measurement window).** + +The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at +`publisher.rs:793`, plus the +two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two +`Instant`s around it yield **`hold_wait_ms`** — the number this entire document is +arguing about, measured rather than modelled. + +Emit two timings per ad-eligible navigation: + +| Metric | Why | +| ----------------- | ---------------------------------------------------------------------- | +| `hold_wait_ms` | **The decision.** How long the response was actually held for bids. | +| `origin_fetch_ms` | Attribution — how much of the win Stage 0 can claim. Origin TTFB only. | + +`hold_wait_ms` replaces the proxy comparison an earlier draft proposed. Comparing `O` +against `A` was an indirect way of asking "does the hold block?"; this asks it directly, +costs less to build, and removes the modelling error corrected in +[§6.2](#62-what-the-hold-actually-costs). + +Deliberately not measured: auction collect duration is already instrumented +(`OrchestrationResult::total_time_ms`, `auction/orchestrator.rs:285`, flowing to +`auction_events_raw`) — read it, don't rebuild it. Rewrite duration decides nothing and +would mean touching two finalizers. + +- **Mechanism: a `log::info!` line behind a debug flag, not `Server-Timing`.** A response + header would in fact work for the origin-fetch figure — that value is known before + headers commit — but a server-side log needs no browser harness to collect it, `log` is + this project's instrumentation crate, and the auction path already measures itself with + `web_time::Instant`. Gate it behind config: one line per eligible navigation is real log + spend and the instrumentation is temporary. +- **Sample: enough navigations per arm to separate the medians with confidence**, across + both page types, and state the N alongside any result. #1009's sample was small enough + that its conclusion did not survive contact with the code; replacing it with another + underpowered sample would repeat the error. + +**Step C has two outcomes, both actionable:** + +| `hold_wait_ms` median | Meaning | Effect on staging | +| --------------------- | ----------------------- | ------------------------------------------------------------ | +| Near zero | The hold is free | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| Materially non-zero | The hold **is** costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | + +The work does not change; its order and justification do. **The staging in §7 is +conditional on this measurement**, and the second outcome is a live possibility rather +than a formality — §6.2's argument for the first is weaker than an earlier draft claimed. + +Stage 1's bids-fetch timeout still needs a measured client-side figure rather than an +invented constant, but Step C is server-side and does not supply it. Capture it from the +browser harness when Stage 1 is actually scheduled. + +--- + +## 4. Stage 0 — the only build item recommended now + +Stop bypassing the read-through cache on ad-eligible navigations +(`publisher.rs:2867`). + +**Ship it as an operator flag, not a deletion.** Add +`publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as +the Step C instrumentation. Then turn it off with `ts config push`. + +The diff is slightly larger than deleting a line, and that is the point. The risk being +gated here is **cache poisoning** — serving one representation in response to a request +for another. For that class of failure, rollback speed dominates diff size: a config push +reverts the read path in seconds where a release does not — but a config push **evicts +nothing**, so full rollback is flip, then purge or roll a versioned key namespace, then +observe past the origin TTL. The flag also buys an A/B on a byte-identical +build, removing build difference as a confound in the very measurement this depends on, +and allows flipping for a tester-cookie population before all traffic. + +Retire the flag once the change has held: flip the default, then delete the setting and +its branch. A temporary flag left in place becomes permanent configuration surface. + +### What to watch after the flip + +Two regression signals, both checked before the win is: + +- **`unexpected_origin_304` abandonment rate.** That reason + (`publisher.rs:2894-2916`, + emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack + path refuses cached and conditional origin responses. Re-enabling the cache is what + could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching + TS that the conditional-header strip was supposed to make impossible. +- **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch means the `Vary` risk materialized + despite a PASS verdict. Roll back immediately; this is cache poisoning, not a + performance regression. + +**Why it is safe in principle.** The conditional-header strip runs 34 lines earlier +under the same gate (`publisher.rs:2832-2836`, +which also strips `Range`/`If-Range`), so the request already reaches the cache +unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) +added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The +strip alone satisfies its invariant. + +**But it carries a risk that design never considered — and this is the blocking +precondition.** RSC fetches are not navigations +(`is_navigation_request` +requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow +through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass +puts both representations under one cache key. #1009 states the origin varies on +`rsc`, `next-router-*`, and a publisher-specific experiment header — if that variance is +not declared via `Vary`, the +cache can serve a flight payload to an HTML navigation. + +The classification is also not airtight: `is_navigation_request` falls back to the +`Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is +weaker — `fetch()` can set Accept: text/html"_ +(`http_util.rs:84-88`). + +**A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches +already transit the read-through cache today, because they never set the bypass. If the +origin varies on `Next-Router-*` without declaring it, TS is cross-serving RSC variants +right now. On a FAIL, file that immediately and treat "ask the origin to declare `Vary`" +as urgent rather than as the cheaper of two options. + +**The `Vary` check is necessary but not sufficient.** Turning the read-through cache on +for HTML navigations exposes three things a representation check does not cover, and all +three are a larger class than the RSC split: + +- **Client `Cookie`.** TS forwards client cookies to origin unchanged — there is no + `COOKIE` strip on the publisher path. Any cookie-personalized HTML (logged-in state, + paywall meter, publisher-side A/B assignment) becomes cross-servable unless the origin + declares `Vary: Cookie` or marks those responses private. +- **Origin `Set-Cookie`.** If the origin emits `Set-Cookie` alongside a shared-cacheable + `Cache-Control`, the read-through cache can replay one visitor's cookie to the next. + TS's own privacy net downgrades **TS's** response — it runs after the cache has already + stored the origin's. +- **`Authorization`.** #1009 describes a basic-auth-gated deployment. Responses to + authorized requests entering a shared cache needs its own check. + +So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with +and without a session cookie. Same minutes of work; closes the bigger hole. + +**Two effort branches, and Step A's `Vary` result decides which** — note this selects the +_shape_ of Stage 0, while the `FINAL PASS` conditions decide _whether it ships at all_: + +| Step A result | Stage 0 is… | Effort | +| ---------------------- | --------------------------------------------- | ------ | +| Origin declares `Vary` | the flag, its tests, then a config push | 1–2 d | +| Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | + +The discriminator is the safer design either way, because it keys on the headers that +actually distinguish the representations rather than on the navigation classification. + +**Two benefits beyond TTFB, worth stating to a publisher:** + +- **Origin load drops.** The 304-prevention design explicitly accepted _"increasing + origin load"_ as a cost. This reverses it. +- **`stale-if-error` becomes reachable.** Under `Pass` an origin outage is a hard + failure. This needs a decision rather than a default: stale HTML carries stale slot + markup, and whether that beats an error is a product call. + +--- + +## 5. The trap in the deferred work — read this before scheduling Stages 1–2 + +The hold is load-bearing for something other than latency. The invariant is: + +> `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. + +The end-tag handler (`html_processor.rs:381-395`) +locks that mutex once and falls back to `build_empty_bids_script()` on `None`. + +**Removing the hold without relocating collection renders a normal page with +`tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On +Axum, Cloudflare, and Spin the loss is fully silent: +`publisher.rs:2248` holds a +bare `Option` with no guard, so not even a drop warning fires. **The +SSPs are billed regardless.** + +This is why Stage 2 is gated on three companions and a production soak, and why slot +fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled). + +--- + +## 6. The analysis + +### 6.1 Corrections to #1009's premises + +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | Partly. `tsjs.adSlots` **content** is per-URL — `build_slot_json` emits config- and path-derived fields only. But its **presence** is gated on `should_run_ad_stack` (consent, bot, prefetch, kill switch), so it is request-dependent and **must not live in a shared template**. See §6.7. | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | + +Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its +two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). + +**Credit where due.** #1009 names the hold as blocker 1 and states it correctly. What +changes here is its _causal weight_. Likewise, #1009's own observation that TS _"shifts +the auction cost from client-side to server-side rather than adding new work"_ is the +argument for client-fill, which the issue then declines in favour of ESI. + +### 6.2 What the hold actually costs + +**An earlier draft of this section claimed a stronger argument than the code supports. +It was wrong, and the correction matters.** + +The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` +(`publisher.rs:2190-2202`) +scans the **decoded origin input** for ` Dispatch precedes the origin fetch, so the hold costs `max(0, A − T)`, where `A` is the +> auction collect duration and `T` is origin TTFB plus body transfer up to the `` +> byte. Since `` sits at the end of a document, `T` is close to the full download. + +`A` is bounded by `auction_timeout_ms`, resolved as +`creative_opportunities.auction_timeout_ms` falling back to `auction.timeout_ms` +(`publisher.rs:2680-2684`) +— check the resolution order against your own config rather than trusting a number; the +shipped example sets different values at each level. + +**This is a claim requiring measurement, not a proof.** §3 Step C measures the hold's +cost directly rather than inferring it. + +A finding that does survive, and belongs with [the ceiling](#64-the-ceiling): because +`HtmlWithPostProcessing` withholds all output until the final chunk, the streaming-prefix +design at `publisher.rs:1343-1348` +— whose comment promises "the client receives the document up to `` while the +auction rides alongside transfer" — is **inert on a Next.js publisher**. Every +`step.ready` yields empty bytes. That comment is misleading on exactly the publisher +under discussion. + +### 6.3 The quantity nobody has measured + +Write the fetch time under `Pass` as `O`. Recovery depends on it, and it has never been +captured. #1009's timings cannot supply it: they compare a POP hit against a +shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` +bypasses TS's read-through cache and its shield. + +Note `Pass` bypasses **TS's** caches only. It has no authority over any CDN the publisher +runs in front of their own origin — and #1009's `x-cache: MISS, MISS` on the TS-on arm +hints one may exist. So `O` may not be origin build time at all. Since `O` is the single +quantity this model depends on, that ambiguity is worth resolving in Step C rather than +assuming. + +What follows from code alone, without any number: + +| Configuration | Long pole after the change | Recovery | +| ------------------- | -------------------------- | ------------------------------ | +| Hold removal only | origin (still `PASS`) | **none** | +| Bypass removal only | the auction budget | partial — the auction surfaces | +| **Both** | the rewrite | **the full available win** | + +That ordering is what the staging rests on, and it is measurement-independent. The +magnitude of each row is not, and §3 Step C supplies it. + +### 6.4 The ceiling + +#1009 targets "approach the TS-off warm numbers." **Unreachable, structurally.** Those +numbers are TS-off _streaming_ a POP HIT. TS buffers the whole document before emitting +a byte (16 MB cap), so its floor is `full origin body download + full rewrite` — above a +streamed hit by construction, whatever the timings turn out to be. Set the target from +Step C's measured rewrite cost rather than from the TS-off baseline. Going below the +floor requires true origin streaming (#849), out of scope. A non-Next.js publisher with +no post-processor takes the streaming path and would see a lower floor. + +### 6.5 Confidence + +**High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the +the silent-empty-bids failure mode, the geo and `Vary` blockers, +and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone +can check them without running anything. + +**None on magnitude.** `O` is unmeasured and the rewrite cost is unmeasured. This +document does not estimate them, and no figure in it should be quoted as one. + +Worth stating plainly: #1009 reached the opposite causal conclusion from a small sample. +That is a caution about small samples generally, not only about that one — which is why +§3 Step C specifies the measurement rather than this document supplying a substitute +for it. + +### 6.6 The ESI pipeline, corrected + +An earlier revision of this document said "order esi → lol*html, never the reverse." +That is backwards. `lol_html` is what \_emits* the ESI include tags; ESI cannot process +tags that do not exist yet. The correct order: + +``` +origin → lol_html transform → fastly::cache::core → finalize headers → stream esi assembly → client + (one unconditional marker (shared template, (EC cookie, geo, (per request, + at the body-close seam; surrogate-keyed, unconditional fetch the + the head seam is NOT a TS-chosen TTL) private/no-store) fragment) + hole — adSlots presence + is request-gated, §6.7) nothing may change + after this point +``` + +The push/pull mismatch that the earlier revision treated as a blocker is real but +irrelevant: `lol_html` pushes, `esi` pulls, and **the cache is the buffer between them**. +That is not an obstacle to the two-stage design — it _is_ the two-stage design, which is +what #1009 proposed in the first place. + +Mechanism, all present in the pinned `fastly` 0.12.1: + +| Need | API | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate **C2 only** | `InsertBuilder::surrogate_keys([...])` (Core Cache) + `fastly::http::purge::purge_surrogate_key`. Does **not** key C1 — see the row below. | +| Invalidate C1 | Origin-supplied surrogate keys, or the HTTP cache's own surrogate-key surface. Not the Core Cache API. | + +Purge runs **inside Compute**. The management-API token scope cited under +[Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does +not gate this. + +**Three caches, kept distinct.** Conflating them is what produced the original error: + +1. **Origin read-through** — raw origin bytes. What Stage 0 turns back on. +2. **Shared transformed template** — post-`lol_html`, pre-ESI, no per-user data. The ESI + target, and new. +3. **Assembled-response delivery cache** — the final per-user output. **Must never + exist.** Nothing in this document or the spike proposes one. + +**Validation constraint.** Viceroy 0.17 cannot exercise the customized read-through hooks +end to end. Unit tests can cover the transform and the security properties; MISS / HIT / +stale / shielding behaviour must run against a real Fastly test service. + +--- + +### 6.7 What may and may not live in a shared template + +A correction to §6.1 row 1, and the constraint that governs any shared-template design. + +The original framing — "`adSlots` is per-URL, so there is one per-user hole, not two" — +is half right and dangerously so. `build_slot_json` really does emit only config- and +path-derived fields. But whether the script is emitted **at all** is gated on +`should_run_ad_stack` (`publisher.rs:2920-2927`), which is +`is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the _content_ is per-URL and the _presence_ is per-request. A shared object filled by +the first request would freeze that request's consent decision, bot classification, +prefetch status, and kill-switch state for every later reader. A consent-denied fill +serves a no-ads template to consenting users; a consenting fill serves ad markup to +someone who refused. + +**The rule for anything cached and shared:** + +| May live in the template | Must live in the per-request fragment | +| --------------------------------------- | ---------------------------------------------- | +| tsjs bundle script tag (content-hashed) | `tsjs.adSlots` — presence is request-gated | +| URL rewrites (per-host, in the key) | `tsjs.bids` | +| | GPT diagnostics bootstrap (cookie/query-gated) | +| | Integration head-inserts (request-scoped) | + +The test that catches this class is **byte-identity of the template across requests +differing in consent, bot classification, and prefetch status** — not an absence-of- +per-user-values scan, which the broken design would have passed. + +This applies to any shared-template work, ESI or client-fill alike. The +[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. + +--- + +## 7. Deferred work, specified not scheduled + +**The full sequence, in one place.** Stage 0 is specified in [§4](#4-stage-0--the-only-build-item-recommended-now) +rather than repeated here; everything below it is deferred. + +| Stage | What | Status | +| ----- | ----------------------------------------------- | ---------------------------------------------- | +| **0** | Operator flag disabling the origin cache bypass | Recommended now. Gated on a `FINAL PASS`. §4. | +| 1 | Bid delivery off the response body | Deferred behind the correctness defects | +| 2 | Delete the `` hold | Deferred; one-way, needs a Stage 1 soak | +| 3a | Browser caching (`private, max-age` + `ETag`) | Specified, low risk, unscheduled | +| 3b | Shared cacheability | Blocked on geo suppression, `Vary`, and Step B | +| 4 | Purge wiring | Prerequisite for any TS-owned cache | + +**ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the +rest. It no longer queues: it is feasible on the pinned SDK and is decided by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` +([§6.6](#66-the-esi-pipeline-corrected)), not a new service. + +Lower detail below is deliberate. Full specifications are in the appendices. + +**Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at +navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer +already exist. Three decisions must be made before planning: the `slots: []` precedence +rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event +emission point; and whether the dispatch/collect split survives at all. Plumbing detail +in [Appendix B](#appendix-b--stage-1-plumbing-condensed). Estimated 8–13 d, low-to-medium +confidence, uncertainty concentrated client-side. + +Three companions are mandatory, not optional: **suppress the server bids script +entirely** (not an empty one), **fail loud** (the end-tag handler takes bids by value so +a missing auction is a compile error), and **relocate telemetry** (navigation +`Completed` rows are emitted only from the collect functions, and the `ts-debug` dump +rides the same string). Behaviour change to accept: under client-fill the auction runs +only if the browser executes the fetch, so bots and JS-disabled clients stop triggering +server-side auctions — revenue-relevant, sign unknown. + +**Stage 2 — delete the hold.** 5–8 d. **Rollback is one-way**: it deletes the hold, the +dispatch/collect split, and twelve tests, so the only revert is a release. Ships only +after Stage 1 has run flag-on in production for a window defined _before_ Stage 1 +starts, with TS-attributed renders flat and `auction_events_raw` navigation rows intact. +Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six compression +imports, and the non-parser-context `` runs _all_ attempts and concatenates every non-failed output** — not + first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in + the crate. +- Single include, not per-slot: the auction is one operation producing all slots' bids. + +--- + +## Appendix F — deferred open items (condensed) + +Implementation-level, for unscheduled work only. Decisions needing a human are in +[§9](#9-decisions-needed-from-this-review). + +Should `collect_non_html_auction` (`publisher.rs:2388`) go with the hold or stay? Is +`body_close_hold_loop_stream` (`:2109`, no production caller) safe to delete, or is the +buffered-adapter streaming cutover (#495) still live? Does hidden-tab rAF behaviour +interact badly with a bids timeout? What are Fastly's pending-request semantics when a +`DispatchedAuction` drops mid-flight? Does `stale-if-error` on a cached root serve +acceptable content given stale slot markup? And the googletag shim discards listeners +queued before it loads (#1009 Part 1) — not filed, should be. + +--- + +## Appendix G — code-grounded seams + +All pinned to `cfb98f4`. + +| Concern | Location | +| --------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (content per-URL, presence request-gated) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | diff --git a/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md new file mode 100644 index 000000000..200a8abbe --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md @@ -0,0 +1,15 @@ +# Contiguous Generated Slot Tables Design + +## Problem + +`splice_creative_slots` parses rendered slots in a temporary `toml_edit::DocumentMut` and moves its `ArrayOfTables` into the target document. Parsed tables retain document-local numeric positions. Those positions collide with positions in the target document, so serialization can interleave generated slot and provider tables with unrelated top-level tables even though the resulting TOML remains semantically valid. + +## Design + +Before insertion, assign the generated slot tables and all nested provider tables the target `[creative_opportunities]` table's document position. `toml_edit` performs a stable position sort, so equal positions retain traversal order: the creative table, each slot, and that slot's provider tables remain contiguous. For a newly created creative section, allocate an anchor after the greatest existing parsed-table position. + +The update continues to preserve unrelated values, comments, line endings, and semantic table ownership. It does not reformat existing operator-authored content or modify slot inference. + +## Testing + +Add a regression fixture with a late `[creative_opportunities]` section and unrelated tables whose positions overlap those from the temporary generated document. Assert that the parent, generated slots, and provider subtables serialize contiguously before the next unrelated table. Retain the existing semantic-preservation and CRLF tests, then run the CLI test suite, formatting, and native CLI clippy. diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md new file mode 100644 index 000000000..5ff4d0707 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -0,0 +1,254 @@ +# PR 823 Review Resolution Design + +## Goal + +Resolve the actionable findings in review `4958563121` on PR 823 without +unrelated refactoring, verify the complete branch, publish the fixes, and reply +to every inline review thread with concrete resolution evidence. + +## Scope + +The implementation covers all 28 inline threads and all actionable items in the +review summary. The summary's explicitly out-of-scope pre-existing +partially-invalid `page_patterns` behavior is not expanded into this PR unless a +fix is required by another in-scope change. The PR description's stale legacy +alias sentence is corrected after the branch changes are published. + +Each reviewer suggestion is verified against the current code. A suggestion is +implemented when it is correct for this repository. Where repository evidence +contradicts a suggestion, the implementation retains the correct behavior and +the review response explains the evidence. + +## Design Principles + +- Preserve operator-authored configuration, comments, ordering, and unrelated + sections byte-for-byte wherever possible. +- Never print secrets or whole effective configuration documents as diagnostic + output. +- Never turn uncertain crawl evidence into a runnable fabricated ad-unit path. +- Treat browser navigation as a session, not a sequence of isolated launches. +- Keep `generate`, `verify`, static CLI commands, and runtime matching on shared + domain rules instead of parallel reimplementations. +- Bound all page-controlled data and browser operations. +- Use test-first changes for behavior corrections and minimal annotations for + code-quality-only corrections. + +## Component Design + +### 1. Configuration integrity and command output + +`slot_toml` will replace the line-oriented slot-boundary heuristic with a +TOML-aware edit strategy. The resulting document must preserve every top-level +item outside the managed creative-opportunity fields and preserve comments +adjacent to or between operator sections. Non-contiguous slot declarations, +multiline values, arrays whose continuation lines begin with `[`, trailing +comments, CRLF input, and inline-slot conversion receive regression coverage. +The updater will reject a candidate if preservation cannot be proven. + +Generation will re-read the source config immediately before the atomic write +and refuse to overwrite a concurrently edited file. `--dry-run` will emit only +the managed creative-opportunities change, never the complete config. Notes and +rollback warnings go to stderr so machine-readable stdout remains clean. Tests +will prove that dry-run leaves the source file byte-identical and does not expose +unrelated secret-bearing keys. + +Merge behavior remains add-only for operator-authored data: existing templated +unit paths are retained, newly observed formats are unioned, and multiple +discovered placements absorbed by one broad configured div prefix produce an +operator note. + +### 2. Crawl evidence and inference + +Inference will preserve evidence instead of silently collapsing it: + +- Non-ASCII shared-prefix computation uses UTF-8 byte boundaries. +- Same-page normalization collisions retain distinct raw placements and emit a + diagnostic rather than silently dropping formats. Numeric-only stable tokens + are not classified as hexadecimal hash noise. +- Multi-slot SRA request fallbacks are ignored when `dids` names more than one + slot. +- A page is considered empty only when no audited profile found slots there. +- Fragment detection requires stronger evidence: a useful shared prefix, or at + least three disjoint fragments. Ambiguous two-slot groups are retained with a + note. +- Locale landing paths are emitted literally when they are shorter than the + inferred section depth, and literal path segments are escaped before being + interpolated into globs. +- Refused template decisions are omitted from generated slots and surfaced with + their reasons. The documentation and tests will consistently describe these + cases as refusal, not literal fallback. +- The redundant witness rule is removed or made independently meaningful. The + actual crawler will support the section depth that inference can produce; + locale-prefixed behavior will not exist only in hand-built evidence tests. +- Dropped-section diagnostics are capped, percent-encoded paths are normalized + before filtering, and page-like extensions are classified consistently. + +The root page and section pages for a device profile are collected in one +browser session. Page analysis that parses full HTML is moved off the +current-thread CDP event pump. Each page/tab is closed on every success and +error path. + +### 3. Shared browser behavior + +The browser collectors will share executable discovery and launch/session +configuration. Browser options exposed to operators will have one meaning in +`page`, `verify`, and `generate`: Chrome override, settling, headful/headless +mode, device profile/viewport, proxy, consent assumption, cookies, and TLS +policy. + +`verify` will reuse one browser/runtime/profile across its URLs so clearance and +session state survive. The generic/legacy generator will default to the same +consent assumption as ad-template generation and expose the opt-out rather than +depending on `derive(Default)`. + +Cookie parameters are explicitly host-only with `Path=/`. A same-host +`http`-to-`https` upgrade is accepted with a redirect note; host changes, +downgrades, and unexpected port changes remain cross-origin refusals. Failure to +read or parse the final browser URL fails closed instead of substituting the +requested URL. + +Every post-navigation evaluation is time-bounded. The collector enlarges the +resource timing buffer before navigation, waits for an interactive or complete +document before accruing quiet time, honors sub-poll quiet windows, validates +`quiet <= max`, and reports saturation. Navigation load-event timeout is a +warning after a successful `goto`; it does not discard readable page evidence. +Evidence payload bytes and captured string lengths are capped before expensive +decode/allocation. + +Init-script and page-evaluation failures become explicit warnings or errors +rather than empty evidence. Promise-returning sitemap evaluation awaits its +result. Main-frame-only collection is disclosed when frames are skipped. + +The injected collector will be behavior-preserving: size pairs enforce the +`u32` range, the `googletag` setter is total, the unused non-variadic `cmd.push` +wrapper is removed, wrapping markers are closure-local/non-enumerable, and +page-derived warning text is terminal-safe. + +### 4. Runtime and static-command parity + +Expected-slot projection uses the runtime's renderability rule. Slots the +runtime omits for a path do not count as matched verification slots; diagnostics +state that the runtime omits the slot on that path rather than claiming the +whole config is rejected. + +Configured media type remains a typed `MediaType` through comparison and is +rendered to a string only at the output boundary. Slots that the phase-one +checker cannot confirm (video/native-only) are represented as unconfirmable and +do not fail `--strict`; genuinely partial or missing confirmable slots still +fail, including a live out-of-page slot with no sizes matched against +banner-configured formats, which is partial. Slot phase is absent when no +evidence exists. +The server-side APS compatibility field no longer creates unconditional +client-side `fetchBids` warnings. + +Collector warnings are included in page results. Human output includes the +runtime expectation, gate summary, matched count, extra evidence, and warnings +already present in JSON. Output escaping covers Unicode bidi controls and all +config-derived strings. + +`explain` reports exactly the shared runtime gate result. Provider configuration +is a separate advisory. The unsupported `--edgezero-enabled` model and stale +legacy-fallback claim are removed because no runtime condition backs them. +Gate diagnostics consume the shared gate result instead of rebuilding lists by +hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is +removed, and the consent tri-state is documented and exhaustively tested. + +`compile_page_pattern` becomes crate-private and a public validation-only API is +used by the CLI. `lint` explicitly reports every configured page pattern the +runtime would drop, while the broader pre-existing runtime acceptance policy +remains out of scope. Specific compile failures are retained in logs. HTTP +methods use `http::Method` parsing so CLI semantics match the runtime. + +Full URLs and bare path inputs pass through the same URL normalization rules: +percent-encoding, dot-segment resolution, query/fragment removal, and leading +slash behavior must be identical. Scheme detection is anchored to the path +portion before `?`, so an absolute URL inside a query value does not cause a +bare path to be parsed as a full URL. + +### 5. CLI contracts, documentation, and CI + +Clap owns argument validation: URL parsing happens at the value parser, the +audit namespace uses help-on-missing-subcommand, `check` uses an argument group +and conflicts, and settle bounds are rejected during parsing. Parser tests cover +the visible command shapes and legacy restrictions. + +CI-oriented assertion failures exit 1; tool/configuration/navigation failures +exit 2. Assertion text is written directly and cannot disappear behind a log +filter. The guide documents all four `ts config ad-templates` commands, all +flags, shared config-loading flags, browser flags, consent/profile behavior, +dry-run output, and exit codes. + +Browser fixture CI either installs/resolves Chrome and requires the tests to +execute, or explicitly opts into a mode that fails when Chrome is unavailable; +it may not report success after silently skipping every browser assertion. + +All real-looking customer identifiers and names introduced by this PR are +replaced with fictional values in tests, comments, and documentation. Stale +module-level lint suppressions, inaccurate docs, assertion messages, enum +ordering, dead query matching, and orphaned comments are corrected without +unrelated cleanup. + +## Inline Review Traceability + +| Thread | Resolution area | +| -------------------------- | ------------------------------------------------------------------ | +| `3802056460`, `3802056470` | TOML-aware splice and comment/value preservation | +| `3802056474` | Secret-safe dry-run and stderr diagnostics | +| `3802056481` | Omit and explain refused slots | +| `3802056488` | UTF-8-safe div prefix calculation | +| `3802056494` | Same-page normalized-div collisions | +| `3802056497` | Locale landing-page patterns | +| `3802056502` | Multi-profile empty-page accounting | +| `3802056508` | Close every browser tab | +| `3802056513` | Enforce JavaScript-to-Rust `u32` bounds | +| `3802056521`, `3802056529` | Total GPT hook and removal of behavior-changing `cmd.push` wrapper | +| `3802056539` | Shared faithful browser launch configuration | +| `3802056549`, `3802056555` | Correct settling and load-timeout handling | +| `3802056559` | Preserve injected collector warnings | +| `3802056564`, `3802056571` | Runtime renderability parity and accurate diagnostics | +| `3802056580`, `3802056584` | Unconfirmable status and removal of false APS warning | +| `3802056586` | Identical URL and bare-path normalization | +| `3802056593` | Fictional committed examples | +| `3802056599` | Browser fixture CI must execute or fail loudly | +| `3802056605` | Add-only merge of formats with broad-prefix diagnostics | +| `3802056614` | Consent parity for generic and legacy generation | +| `3802056623` | Refusal behavior, tests, and documentation agree | +| `3802056628` | Safe same-host HTTP-to-HTTPS redirect handling | +| `3802056638` | Remove ungrounded EdgeZero fallback model | + +## Error Handling and Compatibility + +All new Rust fallible paths use the repository's existing `CliResult` / +`error-stack` conventions. Browser failures identify the operation and URL but +do not include cookies, configuration values, or page payloads. Best-effort +cleanup must not replace an earlier collection error. + +JSON compatibility is preserved where possible. New distinctions are additive +or correct semantically invalid fields: unconfirmable status is explicit, and +phase may be omitted when there was no evidence. Documentation is updated with +the exact wire behavior. + +## Verification Strategy + +Each behavioral issue follows red-green-refactor: + +1. Add the smallest unit, parser, orchestration, or fixture test reproducing the + review finding. +2. Run the narrow test and confirm the expected failure. +3. Implement the minimal correction. +4. Re-run the narrow test and the affected crate suite. + +Final verification runs the repository-required commands relevant to the +changed surface: CLI tests through `scripts/test-cli.sh`, target-matched Rust +tests, JS tests when the collector script changes, `cargo fmt --all -- --check`, +all target-matched clippy aliases, documentation formatting, and browser fixture +tests with an available Chrome. Any environment-dependent test that cannot run +is reported explicitly and is not described as passing. + +## Review Replies and Publication + +Changes are grouped into reviewable commits by component, then pushed to the PR +branch after final verification. Each inline reply is posted in its existing +thread and states the concrete change, relevant test, or evidence-backed reason +for retaining behavior. Replies avoid generic acknowledgements. Threads are not +replied to as fixed until the corresponding commit is visible on GitHub. diff --git a/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md new file mode 100644 index 000000000..e9025174d --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md @@ -0,0 +1,15 @@ +# Pre-navigation Cookie Installation Design + +## Problem + +The audit collectors open `about:blank` so initialization scripts can be installed before publisher code runs. Cookies are explicitly scoped by domain and `/`, but `chromiumoxide::Page::set_cookie` rejects cookies without a URL while the page is still `about:blank`. Consequently, any audit using `--cookie` fails before navigation; audits without cookies are unaffected. + +## Design + +Build the same host-only, root-scoped `CookieParam` values, then install them through `Browser::set_cookies` before creating the page. Browser-level installation sends the explicit domain/path cookie directly to Chrome without deriving scope from the current page URL. Both verification and generation collectors use one shared helper so their behavior cannot drift. + +Cookie-installation errors remain fatal and identify the affected cookie without logging its value. Page initialization, first-request authentication, browser-session reuse, and cookie scope remain unchanged. + +## Testing + +Add a Chrome-backed regression test that starts with `about:blank`, installs a cookie through the shared browser helper, navigates to a local HTTP fixture, and verifies the cookie is visible on the first loaded document. Run the focused CLI tests, formatting, and lint checks required for the touched crate. diff --git a/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md new file mode 100644 index 000000000..8f75d0dcf --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md @@ -0,0 +1,59 @@ +# Ad-template generation progress design + +## Problem + +`ts audit ad-templates generate` audits up to the configured page budget for +each selected device profile (17 pages by default). Navigation and page settling +are intentionally bounded but can still take tens of seconds per page. The +browser collector buffers page results until the browser session closes, so the +command currently emits no output during most of that work and appears stuck. + +## Design + +Emit line-oriented progress on stderr while collection is running. Progress +must identify the device profile, current page, known total, and safe page +location. It must also identify non-page phases where a noticeable pause can +occur: launching the browser, planning the crawl after the root page, and +finalizing the browser session. + +Progress is an explicit collector callback rather than direct terminal output +inside the browser implementation. This keeps output policy in the command +layer, makes the behavior testable with in-memory writers, and lets non-browser +collectors preserve the same contract. Each line is flushed immediately. + +The first profile's root navigation has no final total because follow-up pages +are planned from the rendered root. It is reported as `1/?`; once planning +finishes, subsequent pages use a stable `current/total` count. Later profiles +receive the complete target list and report the root as `1/total`. Totals include +the root, and every attempted page advances the current count even if collection +fails. + +Progress never prints a full URL. It renders only the origin-free path, omitting +userinfo, query, and fragment data, then applies the CLI's existing terminal-text +sanitizer. An empty path is rendered as `/`. + +Stdout remains reserved for the generated diff or success summary. This keeps +`--dry-run` and shell redirection stable. Progress is intentionally plain text, +not an animated spinner, so it remains useful in logs and does not add a terminal +UI dependency. + +## Error handling + +Failure to write or flush progress is returned as a normal CLI output error. A +callback failure during a browser session stops further collection but does not +skip finalization, browser close, or process wait. An earlier collection or +planning error takes precedence over a later progress error; either takes +precedence over teardown errors. Close and wait are still attempted +independently. No cookie values, URL credentials, query values, fragments, or +browser credentials are included in progress. + +## Tests + +Unit tests will verify that progress is emitted before collection completes, +contains the specified profile-aware page counts, keeps stdout unchanged, +redacts URL credentials/query/fragment data, sanitizes paths, and reports +finalization. Writer tests will cover write failure, flush failure, and explicit +flush invocation. Collector tests will verify teardown still runs after progress +failure and that collection/planning errors, progress errors, and teardown errors +retain the stated precedence. The existing CLI and Chrome-backed suites will +verify the collector behavior and browser lifecycle remain intact. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md new file mode 100644 index 000000000..bdc9c5b9c --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -0,0 +1,95 @@ +# Refuse Volatile Div-ID Collisions + +## Problem + +GPT discovery normalizes per-render div IDs such as +`ad-in_content--in_content-0` to the stable prefix `ad-in_content`. +When several live elements on the same page normalize to that prefix, the +runtime cannot represent them safely: one prefix resolves at most one element, +while each exact raw ID changes on a later render. The current collision path +preserves the raw IDs, causing `--replace` to write unusable literal slots. + +## Design + +Treat a source-local normalized collision as ambiguous and refuse the entire +group. The first observation remains tentatively accepted. When a second raw div +ID that describes a _different element_ normalizes to the same prefix, remove +the first slot, record the group as ambiguous, and suppress every later member. +Emit one diagnostic when the group first becomes ambiguous, naming the +normalized prefix and explaining that neither a single prefix nor volatile exact +IDs are safe. Tell the operator to expose distinct stable div IDs or prefixes in +publisher markup before configuring the placements. + +Two raw IDs sharing a stem are not by themselves two elements. One element +re-rendered under a fresh framework token produces exactly that shape, and +absorbing it is what normalization is for: a React publisher reports +`ad-header-0-_R_3f_` from the server render and `ad-header-0-_r_0_` from the +client one, and refusing that pair would generate no slots at all. The two cases +are separated by comparing what the ephemeral markers did _not_ cover — the +marker spans are excised and the remaining parts compared, so identical +residues mean one element observed twice, while `-in_content-0` against +`-in_content-1` means two siblings and is refused. + +The verdict is site-wide, not page-local. Article pages carry several in-content +units and refuse the shared prefix while a landing page carries one, so a +page-local refusal would let crawl sampling decide whether the ambiguous prefix +reaches the config. `DiscoveredSlots` therefore carries the refused stems, +`EvidenceTable` unions them across pages, and the slot iterator the writer reads +suppresses them regardless of which page contributed them. + +Registry and request-derived evidence retain separate collision maps, matching +the current source precedence: even an ambiguous registry stem continues to +suppress request fallback for that stem. Network-ID discovery is unaffected. + +`DiscoveredSlots` records whether any otherwise usable GPT slot evidence was +seen independently of how many safe slots remain. `EvidenceTable::fold_page` +uses that signal when classifying empty pages, so a collision-only page is not +mistaken for a bot challenge. Cross-page slot inference, merging, and +`--replace` otherwise remain unchanged because ambiguous slots never enter +those stages. + +Some ad stacks build IDs as `__`, where the +render token — at least ten leading digits followed by more alphanumerics, +that is, a millisecond timestamp plus entropy — sits _before_ the part that +distinguishes one placement from the next. Such an ID can be written neither +literally nor as a prefix: the only stable prefix stops at the token and reaches +every placement in the family at once. Discovery refuses a single otherwise +usable registry or request observation of that shape, preserves the page/network +evidence, and emits one diagnostic naming the family prefix. The shape decides +rather than a vendor name, so any stack with this layout is covered without a +code change, and every placement after the token is covered rather than an +enumerated few. A token in trailing position is _not_ this case — everything +before it still identifies the element — and is left to normalization and the +collision check. + +## Safety and Output + +The generator prefers omission over a configuration that cannot match future +renders. For an observed desktop crawl of a site with this mix, replacement +output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` +slots, while the in-content collision group and the volatile-token family are +explained in notes. + +## Tests + +- A two-element same-page normalization collision yields no slots and one + diagnostic containing the prefix, both unsafe alternatives, and operator + action. +- Two renders of one element (identical residues either side of the marker, + including a React server/client pair) collapse to one slot with no diagnostic. +- Repeats of the first and second IDs plus a third distinct ID after a collision + remain suppressed and do not create additional diagnostics. +- Request-derived collisions follow the same policy. +- An ambiguous registry stem still suppresses request fallback, and network-ID + discovery survives when every collided slot is omitted. +- A stem refused on one page stays refused after a later page contributes a + single member of the group. +- A collision-only page is recorded as having evidence rather than as an empty + challenge page. +- Single registry- and request-derived render-token observations are omitted + while retaining evidence and any parseable network ID, for every placement + suffix after the token. +- IDs with no render token, with a bare digit run, or with a trailing token stay + eligible. +- Existing normalization, request fallback, fragment detection, and full CLI + tests remain green. diff --git a/docs/superpowers/specs/2026-08-19-trusted-client-ip-header-design.md b/docs/superpowers/specs/2026-08-19-trusted-client-ip-header-design.md new file mode 100644 index 000000000..125be4a89 --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-trusted-client-ip-header-design.md @@ -0,0 +1,223 @@ +# Trusted Client IP Header Design + +## Summary + +Trusted Server's Fastly adapter currently treats +`fastly::Request::get_client_ip_addr()` as the reader address. On a direct +request that is correct, but on a request chained through another Fastly +service it is the fronting edge node. Geo lookup, EC generation, cluster +classification, consent jurisdiction, auction device IPs, and downstream +integration forwarding then all consume the edge-node address. + +Fastly preserves the original address in `Fastly-Client-IP`, but the header is +caller-controlled at the public edge unless the fronting service overwrites it. +Trusted Server must therefore not trust that header based on presence alone. + +This change adds an optional authenticated client-IP header. Existing +deployments remain unchanged until an operator explicitly configures both the +forwarded-IP and authentication headers with a shared secret. + +Related issues: #1040 and #1041. + +## Goals + +- Allow a CDN-fronted Fastly deployment to supply the reader's IP address. +- Establish an explicit trust boundary before using a forwarded address. +- Use one resolved address consistently for geo and all `ClientInfo` consumers. +- Preserve current peer-IP behavior when the feature is absent or authentication + fails. +- Remove trust headers before routing so they cannot leak downstream or be + interpreted by unrelated code. + +## Non-goals + +- Automatically infer whether a request came from another Fastly service. +- Trust `Fastly-Client-IP`, `Fastly-FF`, or `X-Forwarded-For` by presence. +- Change client-IP resolution in Cloudflare, Spin, or Axum adapters. Those + adapters still remove configured trust headers so a shared configuration + cannot expose the authentication value downstream. +- Add rotating or time-limited request signatures in this change. +- Make Fastly no-code request routing preserve a value it does not expose. + +## Configuration Contract + +Add an optional top-level section: + +```toml +[trusted_client_ip] +ip_header = "fastly-client-ip" +auth_header = "x-ts-client-ip-auth" +shared_secret = "replace-with-a-random-shared-secret" +``` + +All three fields are required when the section is present. `shared_secret` uses +the existing `Redacted` type so debug representations do not disclose +it. Configuration validation rejects invalid header names, identical header +names, secrets containing bytes outside the ASCII graphic range +`0x21..=0x7e`, and secrets shorter than 32 ASCII bytes. This is intentionally +stricter than the request reader's `HeaderValue::to_str` contract, which also +accepts horizontal tab: excluding all whitespace prevents an intermediary from +normalizing a configured secret into a different request value. The shared +`reject_placeholder_secrets` startup gate also +rejects the placeholder secret published in the example configuration and +guides, so a copied config cannot ship a publicly known secret. + +To ensure request entry can remove the fields without deleting a required HTTP +field, `ip_header` must either be `fastly-client-ip` or begin with `x-`, and +`auth_header` must begin with `x-`. Comparison is case-insensitive after parsing +through `http::HeaderName`. The two Fastly-injected TLS bridge fields +(`x-ts-tls-protocol` and `x-ts-tls-cipher`) are forbidden for either setting +because the entry point owns and re-injects them after sanitization. These rules +exclude framing, routing, representation, cookie, and authorization fields such +as `host`, `content-length`, `accept`, `cookie`, and `authorization`. A fronting +CDN whose native client-IP field does not meet this contract must copy it into a +dedicated `x-` field before forwarding. + +The section is absent by default. Because the runtime configuration uses strict +unknown-key validation, the example and configuration reference must document +the exact field names. + +## Request Processing + +The Fastly entry point resolves the client address after application state has +loaded but before spoofable headers are sanitized: + +1. Capture `req.get_client_ip_addr()` as the fallback peer address. +2. If `trusted_client_ip` is absent, select the peer address. +3. If configured, require exactly one authentication-header field value. It + must be representable by `HeaderValue::to_str` and match the configured + ASCII-graphic secret byte-for-byte, without trimming or other normalization. + Compare fixed-size SHA-256 digests using a constant-time comparison. A + missing, duplicated, non-ASCII, empty, or mismatched authentication value + fails authentication. +4. Only after authentication succeeds, require exactly one IP-header field + value and parse it directly as `std::net::IpAddr`. Do not trim or normalize + the value. This accepts canonical or otherwise Rust-supported IPv4 and IPv6 + spellings but rejects whitespace, ports, IPv6 zone identifiers, + comma-separated lists, empty values, non-UTF-8 bytes, and duplicate fields. +5. Select the parsed forwarded address on success. Missing headers, a wrong + secret, a malformed header value, or a non-IP value all select the peer + address without rejecting the request. +6. Remove the configured IP and authentication headers, then run the existing + forwarded-header sanitizer. Steps 1-6 sit behind a single + `resolve_and_sanitize_client_ip` call so resolution cannot be reordered + after the sanitization that removes the headers it reads. +7. Pass the selected address into `client_info_from_request` and use the same + value for entry-point geo response finalization. + +`Fastly-Client-IP` is added to the static spoofable-header list. This ensures it +is stripped even when the feature is not configured. A configured header is +read before sanitization and removed explicitly, so configuring +`fastly-client-ip` remains valid. + +The core request sanitizer removes the two configured trust headers on every +adapter. Only Fastly consumes them, but applying the removal invariant across +adapters prevents a shared multi-platform configuration from forwarding an +authentication value to routing or integrations. Cross-adapter treatment of +client-supplied `X-Forwarded-For` is separate from this feature. + +Authentication failures do not log supplied secrets or IP values. A debug-level +message may record only the reason category (missing authentication, mismatch, +or invalid IP) and that the peer fallback was used. + +## Component Boundaries + +### Core settings + +`trusted-server-core/src/settings.rs` owns the serializable +`TrustedClientIpConfig`, validation, redaction, and constant-time secret +verification. Keeping the security rule with the configuration type prevents +adapter call sites from comparing variable-length secrets directly. + +### Fastly client-IP resolution + +`trusted-server-adapter-fastly/src/platform.rs` owns a small resolver that reads +Fastly request headers and returns either the authenticated forwarded IP or the +SDK peer IP. `client_info_from_request` accepts the already-resolved address so +it cannot accidentally re-read the immediate peer. + +### Fastly entry point and sanitization + +`trusted-server-adapter-fastly/src/main.rs` invokes the resolver once and shares +its result with request services and response geo finalization. +`trusted-server-adapter-fastly/src/compat.rs` removes the configured dynamic +headers and continues applying the static spoofable-header list. + +The Axum, Cloudflare, and Spin outer request middleware invokes the shared core +sanitizer before routing. Those adapters do not use this configuration to +resolve their client address. + +No core EC, consent, auction, or integration logic changes: those consumers +already use `RuntimeServices::client_info().client_ip` correctly. + +## Security Model + +The fronting CDN must overwrite both configured headers on every request sent to +Trusted Server. It must never preserve caller-provided values. The shared secret +must be generated randomly, stored in both the fronting service and Trusted +Server configuration, and excluded from responses and origin requests. + +`Redacted` prevents the secret from appearing in debug output and +validation errors; it does not move the value into a platform secret store. The +secret is serialized in the Trusted Server application-config blob, so access +to that configuration store must be restricted. + +This design protects against callers that can reach the Trusted Server hostname +and inject an arbitrary IP header, provided they do not know the shared secret. +It does not provide replay protection: any party that learns the static secret +can authenticate arbitrary IP values. A timestamped HMAC would address replay +and secret reuse but is outside #1041's requested scope. + +Direct requests remain supported. Without valid trust headers they use the +direct peer address, which is the reader address on a one-hop request. + +## Testing + +Tests follow red-green-refactor and cover: + +- absent configuration uses the peer IP; +- valid authentication plus an IPv4 header selects the forwarded IP; +- valid authentication plus an IPv6 header selects the forwarded IP; +- missing, empty, incorrect, non-UTF-8, or duplicate authentication falls back + to the peer IP; +- whitespace-padded, port-bearing, zone-qualified, comma-separated, non-UTF-8, + empty, or duplicate IP input falls back to the peer IP; +- configured headers are removed after resolution; +- configured headers are removed by every adapter before routing; +- `Fastly-Client-IP` is stripped when configuration is absent; +- settings parse, validation, secret redaction, and default behavior; +- a 31-byte secret is rejected and a 32-byte ASCII-graphic secret is accepted; +- non-ASCII, horizontal-tab, space, DEL, and other control-character shared + secrets fail configuration validation without appearing in the error; +- `client_info_from_request` and entry-point geo finalization receive the same + selected address. + +Targeted Fastly and core tests run after each change. Final verification uses +the repository's Fastly/core test alias, formatting, and target-matched Clippy. +The existing local Viceroy certificate-keychain failure may require running the +Fastly integration suite in an environment with native certificates available; +native unit tests and Wasm compilation still provide local evidence. + +## Documentation + +- Add a commented `[trusted_client_ip]` example to + `trusted-server.example.toml` using only `example.com`-safe material. +- Add the new section to `docs/guide/configuration.md`. +- Add Fastly front-door setup guidance to `docs/guide/fastly.md`, emphasizing + that both headers must be overwritten and that enabling the reader without a + correctly configured front door creates a spoofing vulnerability. +- Document the no-code request-routing limitation from #1041. + +## Acceptance Criteria + +- Configuration absent: runtime behavior remains peer-IP based and + `Fastly-Client-IP` is stripped. +- Valid configured authentication: geo and every `ClientInfo` consumer use the + forwarded reader IP. +- Missing, wrong, or malformed authentication: the request succeeds using the + peer IP. +- Invalid forwarded IP: the request succeeds using the peer IP. +- Trust headers never reach routing or downstream origins. +- Existing direct Fastly deployments require no configuration migration. +- Configuration accepts only shared secrets of 32 or more ASCII graphic bytes + (`0x21..=0x7e`), excluding whitespace and non-ASCII values. diff --git a/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md new file mode 100644 index 000000000..8962f4862 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md @@ -0,0 +1,98 @@ +# PR 823 Round-5 Review Resolution + +## Goal + +Resolve review `4989897698` on PR 823 without weakening the generator's safety +rules, silently changing existing CLI defaults, or expanding the change beyond +the audit CLI and its documentation. + +## Browser and CLI Compatibility + +The hidden `ts audit ` compatibility form keeps accepting the same browser +flags as `ts audit generate `, but those flags must remain hidden and must +require the legacy URL positional. A dedicated `LegacyBrowserOpts` mirrors the +seven generation browser fields and converts into `GenerateBrowserOpts` when the +legacy command is dispatched. Consequently, flags placed before a real audit +subcommand are rejected instead of parsed and ignored. + +Generation retains its established 750 ms quiet period and 12-second maximum +settle wait. Generation defaults have one source of truth shared by clap, +`GenerateBrowserOpts::default`, and `BrowserAuditCollector::default`; applying +parsed options must not silently shorten the collector's maximum. The generic +page/verification collector keeps its existing independent 10-second default. + +Redirect notes show the origin and path for both requested and final URLs. This +makes scheme and host changes visible without exposing URL userinfo, queries, or +fragments. + +## Root-Less Template Safety + +Template inference records which slot stems borrowed the config-level +`section_root` because those slots were never witnessed on a path without the +configured section segment. Such a template is safe only while its page patterns +are derived from the paths where the slot was observed. + +Operator-supplied `--page-pattern` values replace those derived patterns for +every slot. If inference contains any borrowed-root slot and explicit patterns +were supplied, generation fails before rendering or writing a candidate config. +The error identifies the affected slots, explains that explicit patterns cannot +prove the borrowed-root invariant, and directs the operator to remove +`--page-pattern`. Failing the command is preferable to silently omitting real +inventory or attempting an unsound glob intersection. + +When no config-level section policy can be inferred because every otherwise +templatable slot lacks a root witness, each affected slot's refusal reason names +that crawl gap rather than claiming that its paths failed to generalize. + +## Merge Policy + +An explicitly configured `section_segment` is operator intent even when +`section_root` is currently unset. If preserved `{section}` slots exist and an +inferred policy would change that configured segment, merge fails and requires +`--replace` for the migration. If the configured segment matches, or is unset, +the inferred `section_root` may be adopted so the previously incomplete config +becomes loadable. + +## Diagnostics and Early Validation + +Warnings produced while folding a collected page include the device-profile +label as well as the path. Identical warnings from desktop and mobile therefore +remain distinguishable. The consent-stub warning remains a single unscoped +run-level note, and site-wide discovery warnings remain deduplicated. + +The existing config is parsed as TOML before Chrome starts. A whole-document +syntax error is returned immediately; a valid document with settings unknown to +the CLI still permits extraction of `[creative_opportunities]`; and a present +but unreadable creative section remains an error. + +The volatile div-id token recognizer requires at least ten leading digits plus +an alphanumeric suffix. This continues to recognize timestamp-like generated +tokens while preventing an eight-digit calendar date followed by a stable +letter from causing a single-observation family refusal. + +## Consistency Corrections + +Tests pin the Rust evidence cap to the embedded JavaScript collector constant. +The terminal-escaping test claims only controls it can actually inject; URL's +own percent-encoding is covered by an exact final-URL assertion rather than +presented as evidence for terminal escaping. Existing code escaping the final +URL remains as defense in depth. + +The affected guide, prior volatile-collision spec and plan, documentation +comments, `expect` message, and method spacing are corrected to describe the +implemented behavior exactly. The root-less templating behavior and this review +resolution are documented by this design and its paired implementation plan. + +## Testing and Delivery + +Every behavioral correction starts with a focused regression test that fails on +the current branch. Tests cover hidden legacy flags, the 12-second generation +default, complete redirect notes, borrowed-root rejection with explicit +patterns, configured-segment preservation, profile-specific warnings, +whole-document TOML failure, the evidence-cap invariant, and the calendar-date +token control. + +After focused tests pass, verification runs the host-target CLI suite and +audit/generate tests, CLI clippy with warnings denied, Rust formatting, docs +formatting, and `git diff --check`. No GitHub replies or push are part of this +change unless separately requested. diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index 96d6cb7ff..b5300da84 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -328,7 +328,7 @@ check_hit_is_private() { -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ "http://127.0.0.1:$TS_PORT/article") check "cache hit is not shared-cacheable" \ - "$(echo "$hdrs" | grep -ci 'cache-control: no-store, private' || true)" "1" + "$(echo "$hdrs" | grep -ci 'cache-control: private, no-store' || true)" "1" } check_post_reaches_origin() { diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..adc4ec0f0 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,20 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +export TS_AUDIT_BROWSER_TESTS=1 +AUDIT_BROWSER_TEST_FILTERS=( + "commands::audit::browser::tests::" + "commands::audit::generate::browser_collector::tests::progress_failure_still_finalizes_browser_session" +) +for AUDIT_BROWSER_TEST_FILTER in "${AUDIT_BROWSER_TEST_FILTERS[@]}"; do + AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list + } | awk '/: test$/ { count += 1 } END { print count + 0 }')" + if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 + fi + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 +done diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 5bbdfb857..735115c9a 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -11,6 +11,13 @@ origin_url = "https://origin.example.com" # origin_host_header_override = "www.example.com" proxy_secret = "change-me-proxy-secret" +# Optional: trust a fronting CDN's reader IP only when it also supplies the +# matching shared secret. The front door must overwrite both headers. +# [trusted_client_ip] +# ip_header = "fastly-client-ip" +# auth_header = "x-ts-client-ip-auth" +# shared_secret = "replace-with-a-random-shared-secret" + [ec] passphrase = "trusted-server-placeholder-secret" ec_store = "ec_identity_store" @@ -184,6 +191,9 @@ debug = false # inventory_page_origin = "https://www.publisher.example" # Script creatives require separate security validation before opt-in. allow_script_creatives = false +# Default: Trusted Server's opaque static renderer route. Set publisher_native only +# for the controlled publisher-origin friendly-frame experiment. +rendering_mode = "trusted_server" [integrations.google_tag_manager] enabled = false @@ -223,6 +233,14 @@ verbosity = "redacted" format = "compact" [creative_opportunities] +# Set to false to disable server-side ad templates while retaining slot definitions +# and direct POST /auction callers. Structurally inactive templates use the +# browser-only Cache-Control: private, max-age=60 policy; this setting is one cause. +# Origin private/no-store policies and bot, prefetch, or consent-denied requests +# retain their origin policy. CDN-specific cache headers remain unchanged. +# Before rolling back to a binary that predates this setting, restore true, re-push +# and finalize the config first. +enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on