From 2b326e778c4c76404b75602793f38173527ee1b9 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Wed, 15 Apr 2026 20:45:20 +0200 Subject: [PATCH 001/195] Add server-side ad templates design spec Co-Authored-By: Claude Sonnet 4.6 --- ...6-04-15-server-side-ad-templates-design.md | 363 ++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md new file mode 100644 index 000000000..454f37641 --- /dev/null +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -0,0 +1,363 @@ +# Server-Side Ad Templates Design + +*April 2026* + +--- + +## 1. Problem Statement + +Today's display ad pipeline on most publisher sites is structurally sequential +and browser-bound: + +1. Page HTML arrives at browser +2. Prebid.js (~300KB) downloads and parses +3. Smart Slots SDK scans the DOM to discover ad placements +4. `addAdUnits()` registers slot definitions +5. Prebid auction fires from the browser (~80–150ms RTT to SSPs) +6. Bids return (~1,000–1,500ms window) +7. GPT `setTargeting()` + `refresh()` fires +8. GAM creative renders + +**Total time to ad visible: ~3,100ms.** + +The browser is the slowest possible place to run an auction. It must first download and parse +multiple SDKs, scan the DOM to discover what ad slots exist, and then fire SSP requests over +a consumer internet connection with high and variable latency. + +Trusted Server sits at the Fastly edge — milliseconds from the user, with data-center-to-data-center +RTT to Prebid Server (~20–30ms vs ~80–150ms from a browser). The server knows, from the request +URL alone, exactly which ad slots are available on any given page. There is no reason to wait for +the browser. + +--- + +## 2. Goal + +Enable Trusted Server to: + +1. Match an incoming page request URL against a set of pre-configured slot templates +2. Immediately fire the full server-side auction (all providers: PBS, APS, future wrappers) in + parallel with the origin HTML fetch — before the browser receives a single byte +3. Inject GPT slot definitions into `` so the client can define slots without any SDK +4. Return pre-collected winning bids to the browser's lightweight `/auction` POST before the + browser would have even finished parsing Prebid.js +5. Eliminate Prebid.js from the client entirely + +**Target time to ad visible: ~1,200ms. Net saving: ~2,000ms.** + +--- + +## 3. Non-Goals + +- Eliminating client-side GPT / Google Ad Manager — GAM remains in the rendering pipeline + for Phase 1. The GAM call (`securepubads.g.doubleclick.net`) moves server-side in a future phase. +- Dynamic slot discovery (reading the DOM) — this design commits to pre-defined, URL-matched + slot templates. Smart Slots' dynamic injection behavior is replaced by server knowledge. +- Changing the `AuctionOrchestrator` internally — the orchestrator already handles parallel + provider fan-out. This design adds a new trigger point, not new auction logic. + +--- + +## 4. Architecture + +### 4.1 New File: `creative-opportunities.toml` + +A new config file at the repo root, alongside `trusted-server.toml`. It holds all slot templates: +page pattern matching rules, ad formats, floor prices, and GAM targeting key-values. Bidder-level +params (placement IDs, account IDs) live in Prebid Server stored requests, keyed by slot ID — not +in this file. + +Loaded at build time via `include_str!()`, parsed into `Vec` at startup. +Ad ops can edit this file independently of server configuration. + +`floor_price` is the publisher-owned hard floor per slot — the source of truth for the minimum +acceptable bid price, enforced at the edge before bids reach the ad server. Any bid below the +floor is discarded at the orchestrator level before it enters `__ts_bids`. SSPs may apply their +own dynamic floors independently within their platforms; this floor is the publisher's baseline +that supersedes all other floor logic by virtue of being enforced earliest in the pipeline. + +**Schema:** + +```toml +[[slot]] +id = "atf_sidebar_ad" +page_patterns = ["/20*/"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[[slot]] +id = "below-content-ad" +page_patterns = ["/20*/"] +formats = [{ width = 300, height = 250 }, { width = 728, height = 90 }] +floor_price = 0.25 + +[slot.targeting] +pos = "btf" +zone = "belowContent" + +[[slot]] +id = "ad-homepage-0" +page_patterns = ["/", "/index.html"] +formats = [{ width = 970, height = 250 }, { width = 728, height = 90 }] +floor_price = 1.00 + +[slot.targeting] +pos = "atf" +zone = "homepage" +slot_index = "0" +``` + +**Rust type:** + +```rust +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CreativeOpportunitySlot { + pub id: String, + pub page_patterns: Vec, + pub formats: Vec, + pub floor_price: Option, + pub targeting: HashMap, +} +``` + +### 4.2 URL Pattern Matching + +At request time, TS matches the request path against each slot's `page_patterns`. Patterns are +glob-style strings: + +- `/20*/` — matches all date-prefixed article paths (e.g., `/2024/01/my-article/`) +- `/` — matches the homepage exactly +- `/index.html` — exact match + +Multiple slots can match a single URL. All matching slots are collected and fed into a single +auction as separate impressions. Pattern matching is purely in-memory against the pre-parsed +config — sub-millisecond. + +### 4.3 Auction Trigger + +When slots are matched, TS immediately calls `AuctionOrchestrator::run_auction()` with the +matched slots converted to `AdSlot` objects. This happens at request receipt time — in parallel +with the origin fetch. + +The orchestrator's existing behaviour is unchanged: +- All providers (PBS, APS, any configured wrappers) are dispatched simultaneously +- Per-provider timeout budgets are enforced from the remaining auction deadline +- Floor price filtering, bid unification, and winning bid selection are applied as today +- PBS resolves bidder params from its stored requests by slot ID — no bidder params travel + through TS or the browser + +**On NextJS 14 (buffered mode):** TS must buffer the full origin response before forwarding. +This gives the auction the entire origin response time (~150–400ms typical) to run before +any HTML is forwarded. In practice, bids are often collected before origin even responds. + +**On NextJS 16 (streaming mode):** TS streams HTML chunks to the browser immediately. The +auction runs in parallel. Bid injection into `` must complete before the `` tag +is forwarded. If the auction has not returned by the time `` is encountered, TS waits +up to the remaining auction budget, then flushes with whatever bids have arrived (partial +results) or no targeting if timed out. Content after `` is never held. + +### 4.4 Head Injection + +TS injects two separate ``, not +> raw string interpolation. -Prebid.js is eliminated. The client-side ad bootstrap is replaced by a small inline script -(~20 lines) that reads `__ts_ad_slots` and `__ts_bids` and drives GPT directly: +> **Cache contract:** Any response with `__ts_bids` injected is per-user data and must +> not be cached. TS sets `Cache-Control: private, no-store` on the response before +> forwarding, overriding any conflicting cache headers from the publisher origin. +> `Surrogate-Control` and `Fastly-Surrogate-Control` are also stripped. + +### 4.5 Win Notifications + +Win notification responsibilities are split by where the truth lives: + +**`nurl` (SSP win event) — fired server-side.** When the orchestrator selects a winning +bid, TS fires a fire-and-forget background HTTP request to `nurl` from the edge +(edge→SSP RTT ~20–30ms, no auction-path latency cost). A per-integration switch +(`[integrations.prebid].fire_nurl_at_edge`, default `true`) handles cases where the PBS +deployment already fires win events internally to avoid double-firing. APS win +notification follows its own spec. + +**`burl` (billing event) — fired client-side.** `burl` is embedded per slot in +`__ts_bids` (see §4.4). The `__tsAdInit` script registers a GPT `slotRenderEnded` +listener after defining slots. On render: if `!event.isEmpty` and +`event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid`, the client fires `burl` +via `navigator.sendBeacon`. This confirms both that the ad rendered and that our specific +Prebid bid (not a direct deal or backfill) won the GAM line item match. + +### 4.6 Client Residual + +Prebid.js is eliminated. The client-side ad bootstrap is replaced by a small inline +script (~30 lines) that reads `__ts_ad_slots` and `__ts_bids`, drives GPT directly, and +handles billing notifications: ```javascript -window.__tsAdInit = function() { - var slots = window.__ts_ad_slots || []; - var bids = window.__ts_bids || {}; - googletag.cmd.push(function() { - slots.forEach(function(slot) { - var gptSlot = googletag.defineSlot(slot.id, slot.formats, slot.id) - .addService(googletag.pubads()); +window.__tsAdInit = function () { + var slots = window.__ts_ad_slots || [] + var bids = window.__ts_bids || {} + googletag.cmd.push(function () { + slots.forEach(function (slot) { + var gptSlot = googletag + .defineSlot(slot.gam_unit_path, slot.formats, slot.div_id) + .addService(googletag.pubads()) // Apply static targeting from config - Object.entries(slot.targeting).forEach(function([k, v]) { - gptSlot.setTargeting(k, v); - }); + Object.entries(slot.targeting).forEach(function ([k, v]) { + gptSlot.setTargeting(k, v) + }) // Apply pre-won bid targeting if available - var bidTargeting = bids[slot.id] || {}; - Object.entries(bidTargeting).forEach(function([k, v]) { - gptSlot.setTargeting(k, v); - }); - }); - googletag.pubads().enableSingleRequest(); - googletag.enableServices(); - googletag.pubads().refresh(); - }); -}; + var bidData = bids[slot.id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (key) { + if (bidData[key]) gptSlot.setTargeting(key, bidData[key]) + }) + }) + googletag.pubads().enableSingleRequest() + googletag.enableServices() + // Fire burl on confirmed render + googletag.pubads().addEventListener('slotRenderEnded', function (event) { + var slotId = event.slot.getSlotElementId() + var bidData = bids[slotId] || {} + if ( + !event.isEmpty && + bidData.burl && + event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid + ) { + navigator.sendBeacon(bidData.burl) + } + }) + googletag.pubads().refresh() + }) +} ``` -This script is part of the `tsjs-gpt` integration bundle, injected by TS into every matching -page response alongside the existing GPT integration. +This script is part of the existing `gpt` integration bundle +(`crates/js/lib/src/integrations/gpt/index.ts`), extending the existing GPT shim. +Injected via the `gpt` head injector alongside `window.__ts_ad_slots`. --- @@ -238,21 +440,26 @@ t=0ms GET ts.publisher.com/article arrives at Fastly edge t=1ms URL matched against creative-opportunities.toml Slots matched: [atf_sidebar_ad, below-content-ad, section_ad] + Consent check: TCF consent present → auction proceeds t=2ms AuctionOrchestrator.run_auction() called - PBS + APS dispatched in parallel + PBS + APS dispatched in parallel via send_async() Edge→PBS RTT: ~20–30ms -t=2ms Origin fetch dispatched in parallel +t=2ms Origin fetch dispatched via send_async() in parallel + +t=2ms window.__ts_ad_slots script assembled from config (no auction needed) t=150ms Origin HTML arrives at edge (NextJS 14: buffered) + Auction still running; origin response held at edge -t=502ms Auction timeout fires (500ms budget) - Winning bids collected +t=502ms Auction deadline fires (500ms budget) + Winning bids collected; nurl fired as background requests -t=502ms injection assembled: - - window.__ts_ad_slots (from config, available at t=1ms) - - window.__ts_bids (from auction results) +t=502ms HtmlProcessorConfig constructed with bid results captured + injection assembled: + - window.__ts_ad_slots (from config, ready at t=2ms) + - window.__ts_bids (from auction results; Cache-Control: private, no-store set) t=502ms HTML forwarded to browser with injected @@ -270,7 +477,7 @@ t=822ms GET /gampad/ads t=922ms Creative fetch -t=1222ms Creative sub-resources + paint +t=1222ms Creative sub-resources + paint; burl fired via slotRenderEnded AD VISIBLE ~1200ms ``` @@ -279,18 +486,23 @@ t=1222ms Creative sub-resources + paint ## 6. Performance Summary -| Stage | Client-side today | With TS templates | Saving | -|---|---|---|---| -| Script load chain | ~700ms | ~40ms (tsjs only) | -660ms | -| Script parse/JIT | ~280ms | ~10ms | -270ms | -| Sequential SDK hops | ~200ms | 0 | -200ms | -| Auction window | ~1,500ms | ~500ms | -1,000ms | -| GAM + creative | ~570ms | ~570ms | — | -| **Total** | **~3,250ms** | **~1,200ms** | **~2,000ms** | +| Stage | Client-side today | With TS templates | Saving | +| ------------------- | ----------------- | ----------------- | ------------ | +| Script load chain | ~700ms | ~40ms (tsjs only) | -660ms | +| Script parse/JIT | ~280ms | ~10ms | -270ms | +| Sequential SDK hops | ~200ms | 0 | -200ms | +| Auction window | ~1,500ms | ~500ms | -1,000ms | +| GAM + creative | ~570ms | ~570ms | — | +| TTFB penalty¹ | 0 | up to +350ms | - | +| **Total** | **~3,250ms** | **~1,200ms** | **~2,000ms** | + +¹ Buffered mode only: the origin response is held until the auction resolves. For fast +origins (<150ms) and a 500ms auction deadline, TTFB may increase by up to 350ms. This +tradeoff is net-positive on revenue. The streaming mode (NextJS 16) has no TTFB penalty. -Auction RTT improvement: browser fires SSP requests at 80–150ms RTT; edge fires at 20–30ms. -Auction timeout can drop from 1,000–1,500ms to 500ms while still collecting more complete -results, because edge→PBS latency is ~5–7x lower. +Auction RTT improvement: browser fires SSP requests at 80–150ms RTT; edge fires at +20–30ms. Auction timeout can drop from 1,000–1,500ms to 500ms while still collecting +more complete results, because edge→PBS latency is ~5–7x lower. --- @@ -299,24 +511,42 @@ results, because edge→PBS latency is ~5–7x lower. ### New - `creative-opportunities.toml` — slot template config file -- `crates/trusted-server-core/src/creative_opportunities.rs` — config types, TOML parsing, - URL pattern matching, slot-to-`AdSlot` conversion -- `build.rs` update — `include_str!()` for `creative-opportunities.toml` -- Request handler modification — match slots at request receipt, trigger orchestrator immediately, - hold result for head injection -- `tsjs-gpt` integration update — `__tsAdInit` bootstrap replaces Prebid.js ad unit setup +- `crates/trusted-server-core/src/creative_opportunities.rs` — config types, TOML + parsing, URL glob matching, slot-to-`AdSlot` conversion, price bucketing +- `crates/trusted-server-core/build.rs` — `include_str!()` for + `creative-opportunities.toml`; startup slot-ID validation +- `crates/trusted-server-core/src/price_bucket.rs` — Prebid price granularity tables + (dense default; publisher-configurable); converts raw CPM `f64` to `hb_pb` string ### Modified -- `crates/trusted-server-core/src/integrations/prebid.rs` head injector — emit - `window.__ts_ad_slots` from matched slots -- `crates/trusted-server-core/src/html_processor.rs` — inject `window.__ts_bids` once auction - results are available, before `` -- `trusted-server.toml` — add `creative_opportunities_path` config key pointing to the new file +- **`crates/trusted-server-core/src/publisher.rs`** — primary structural change: + - Convert `handle_publisher_request` from `fn` to `async fn` + - Switch origin fetch from `.send()` to `.send_async()` (returns + `PlatformPendingRequest`) + - Add `orchestrator: &AuctionOrchestrator` parameter + - Match slots, check consent, fire auction and origin fetch concurrently + - Await both and construct `HtmlProcessorConfig` with resolved bid results +- **`crates/trusted-server-adapter-fastly/src/main.rs`** — update `route_request` call + site to `.await` the now-async publisher handler; pass orchestrator reference +- **`crates/trusted-server-core/src/html_processor.rs`** — inject `window.__ts_bids` + before `` via `el.on_end_tag()` on the `` element; set + `Cache-Control: private, no-store` header on injection; HTML-escape bid JSON +- **`crates/trusted-server-core/src/integrations/gpt.rs`** — extend head injector to + emit `window.__ts_ad_slots` from matched slots (not `prebid.rs`); emit `__tsAdInit` + bootstrap script +- **`crates/js/lib/src/integrations/gpt/index.ts`** — add `__tsAdInit` function and + `slotRenderEnded` burl-firing logic to the existing GPT shim +- **`crates/trusted-server-core/src/integrations/prebid.rs`** — add + `fire_nurl_at_edge` config key; add nurl fire-and-forget call in orchestrator result + handling +- **`trusted-server.toml`** — add `[creative_opportunities]` section +- **`crates/trusted-server-core/src/settings.rs`** — add `CreativeOpportunitiesConfig` + to `Settings` ### Unchanged -- `AuctionOrchestrator` — no internal changes; new call site only +- `AuctionOrchestrator` internals — no changes; new call site only - PBS stored request configuration — bidder params remain in PBS, keyed by slot ID - GAM line item configuration — targeting key-values pass through unchanged @@ -324,40 +554,66 @@ results, because edge→PBS latency is ~5–7x lower. ## 8. Edge Cases -**No slots match the URL** — auction is not fired. Head injection emits neither global. GPT -bootstrap detects empty `__ts_ad_slots` and skips initialization. Page loads normally with no -ad stack. +**No slots match the URL** — auction is not fired. Neither global is emitted. The page +loads with no TS ad stack; existing client-side Prebid/GPT flow runs unmodified (for +publishers in dual-mode rollout). + +**Consent absent or denied** — auction is not fired. Neither global is emitted. +`Cache-Control: private, no-store` is still set (to prevent caching the consent-negative +response if personalised ads were previously served). Page loads normally; GAM runs its +own auction without Prebid targeting. + +**Auction times out with partial results** — `__ts_bids` is populated with whatever bids +arrived before the deadline. Slots with no bid are omitted. GPT fires without pre-set +targeting for those slots; GAM falls back to its own auction for them. + +**Auction times out with zero results** — `__ts_bids` is an empty object `{}`. All slots +fire GAM without bid targeting. No revenue impact beyond the timeout scenario itself. -**Auction times out with partial results** — `__ts_bids` is populated with whatever bids arrived -before the deadline. Slots with no bid omitted. GPT fires without pre-set targeting for those slots; -GAM falls back to its own auction. +**Origin is slow (NextJS 14, buffered)** — auction has more time; results more likely to +be complete. TTFB impact is bounded by the origin latency, not additive to it. -**Auction times out with zero results** — `__ts_bids` is an empty object `{}`. All slots fire -GAM without bid targeting. No revenue impact beyond the timeout scenario itself (same as today's -fallback). +**NextJS 16 streaming** — `el.on_end_tag()` on `` gates injection. TS waits up to +the remaining `auction_timeout_ms` budget, then flushes. Content after `` is never +held. If the auction resolves before `` is encountered (common case), injection is +zero-latency. -**Origin is slow (NextJS 14, buffered)** — auction has more time; results more likely to be -complete. No change to streaming behavior. +**`creative-opportunities.toml` missing or malformed** — startup fails with a clear +error. No silent degradation. -**NextJS 16 streaming** — TS must flush `` before `` tag passes through. If auction -not yet complete, TS waits up to `auction_timeout_ms` from the config, then flushes. Content -streaming resumes immediately after `` regardless of bid state. +**Config empty (zero slots)** — treated as "no match" for all URLs; auction never fires. +No error. Useful as a kill-switch: deploying an empty `creative-opportunities.toml` +disables the feature without a code change. -**`creative-opportunities.toml` missing or malformed** — startup fails with a clear error. -No silent degradation. +**Slot ID not found in PBS stored requests** — PBS returns a no-bid for that slot. Slot +is omitted from `__ts_bids`. The remaining slots proceed normally. --- ## 9. Open Questions -1. **URL pattern coverage** — does `/20*/` cover all article paths, or are there +1. **URL pattern coverage** — does `/20**` cover all article paths, or are there non-date-prefixed article URLs? Publisher to confirm. 2. **PBS stored request setup** — slot IDs in `creative-opportunities.toml` must have - corresponding stored requests configured in the publisher's PBS instance before this goes live. -3. **Homepage slot count** — the example shows slots 0 and 1. Are there slots 2–5 following - the same pattern? Slot IDs and count to be confirmed with ad ops. -4. **Auction timeout for server-side trigger** — current `[integrations.prebid].timeout_ms` - is 1,000ms. Recommend reducing to 500ms for server-side triggered auctions given the - lower edge→PBS RTT. Separate config key or override on the new trigger path? -5. **`tsjs-gpt` bootstrap delivery** — the `__tsAdInit` script needs to fire after GPT.js - loads. Confirm injection order with the existing GPT integration head injection. + corresponding stored requests configured in the publisher's PBS instance before this + goes live. +3. **Homepage slot count** — the example shows slots 0 and 1. Are there additional slots + following the same pattern? Slot IDs and count to be confirmed with ad ops. +4. **Auction timeout** — ✅ Resolved: new dedicated key + `[creative_opportunities].auction_timeout_ms` with fallback to `[auction].timeout_ms`. + Per-provider ceilings (`[integrations.prebid].timeout_ms`, + `[integrations.aps].timeout_ms`) remain unchanged; the orchestrator's existing + `min(remaining_budget, provider_timeout)` logic applies. +5. **KV-backed config migration path** — Phase 1 ships with `include_str!()` for + simplicity and cost. When ad ops require live slot edits between deploys, the migration + path is: load from `services.kv_store()` at request time with a compiled-in fallback. + Design tracked as a follow-up before Phase 2. +6. **Phase 2 server-side GAM** — The real latency ceiling is the GAM call + (`securepubads.g.doubleclick.net`). Phase 2 routes the GAM ad request through the edge + (securepubads proxy + creative bundling), eliminating the last browser→Google hop. The + Phase 1 architecture is designed to be shape-compatible with this: `__ts_ad_slots` + gives the edge the full slot inventory it needs to build a server-side GAM request. +7. **`tsjs-gpt` bootstrap delivery** — ✅ Resolved: `__tsAdInit` is part of the existing + `gpt` integration bundle, not a new integration. Injection order: `window.__ts_ad_slots` + → existing GPT shim → `__tsAdInit` — all emitted by the `gpt` head injector in a single + `".to_string() + ), + ad_bids_script: None, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"T", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots"); + } + + #[test] + fn injects_bids_before_end_of_head() { + let bids_script = ""; + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_script: Some(bids_script.to_string()), + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"T", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("window.__ts_bids"), "should inject bids"); + let bids_pos = html.find("window.__ts_bids").expect("should find bids"); + let end_head_pos = html.find("").expect("should find "); + assert!(bids_pos < end_head_pos, "bids script should appear before "); + } + ``` + + Run: `cargo test -p trusted-server-core html_processor` + Expected: compile error (no `ad_slots_script`/`ad_bids_script` fields, no `empty_for_tests()`) + +- [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** + + In `registry.rs`, add: + + ```rust + #[cfg(test)] + impl IntegrationRegistry { + pub fn empty_for_tests() -> Self { + // Minimal registry with no integrations for unit testing html_processor + Self { + inner: Arc::new(RegistryInner { + proxies: Default::default(), + attribute_rewriters: Default::default(), + script_rewriters: Vec::new(), + html_post_processors: Vec::new(), + head_injectors: Vec::new(), + metadata: Default::default(), + }) + } + } + } + ``` + + (Adjust field names to match the actual `RegistryInner` struct.) + +- [ ] **Step 3: Add fields to `HtmlProcessorConfig`** + + ```rust + pub struct HtmlProcessorConfig { + pub origin_host: String, + pub request_host: String, + pub request_scheme: String, + pub integrations: IntegrationRegistry, + /// Pre-computed `` for matched slots. + /// Injected at open, before integration head inserts. `None` when no slots matched. + pub ad_slots_script: Option, + /// Pre-computed `` for winning bids. + /// Injected immediately before via on_end_tag(). `None` when auction not run. + pub ad_bids_script: Option, + } + ``` + + Update `from_settings` to initialize `ad_slots_script: None, ad_bids_script: None`. + +- [ ] **Step 4: Inject `__ts_ad_slots` at head-open AND register `on_end_tag` for `__ts_bids`** + + In `create_html_processor`, within the EXISTING single `element!("head", ...)` handler, make two changes: + 1. Prepend the ad slots script BEFORE the existing integration inserts: + + ```rust + // NEW: inject __ts_ad_slots first + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } + // ... existing: for insert in integrations.head_inserts(&ctx) { ... } + ``` + + 2. After `el.prepend(...)`, register the end-tag handler for `__ts_bids`: + ```rust + // Register on_end_tag handler for __ts_bids injection before + if let Some(bids_script) = ad_bids_script.clone() { + el.on_end_tag(move |end_tag| { + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + })?; + } + ``` + + Both changes live inside the same `element!("head", ...)` closure — no second handler needed. + + Capture `ad_slots_script` and `ad_bids_script` into the closure the same way as `injected_tsjs`: + + ```rust + let ad_slots_script = config.ad_slots_script.clone(); + let ad_bids_script = config.ad_bids_script.clone(); + ``` + + > **lol_html `on_end_tag` API note:** `Element::on_end_tag(handler)` is available in lol_html ≥2.0. The handler receives `&mut EndTag` and must return `Result<(), Box>`. Use `ContentType::Html` so the injected `", escaped) + } + + pub(crate) fn build_ad_bids_script( + winning_bids: &std::collections::HashMap, + price_granularity: crate::price_bucket::PriceGranularity, + ) -> String { + let bids_map: serde_json::Map = winning_bids + .iter() + .filter_map(|(slot_id, bid)| { + let cpm = bid.price?; + let entry = serde_json::json!({ + "hb_pb": price_bucket(cpm, price_granularity), + "hb_bidder": bid.bidder, + "hb_adid": bid.ad_id.as_deref().unwrap_or(""), + "burl": bid.burl, + }); + Some((slot_id.clone(), entry)) + }) + .collect(); + let json = serde_json::to_string(&serde_json::Value::Object(bids_map)) + .expect("should serialize bids"); + let escaped = html_escape_for_script(&json); + format!("", escaped) + } + + /// HTML-escape a JSON string for safe inline `" + .to_string(), + // __tsAdInit definition — reads window.__ts_ad_slots / __ts_bids at call time. + concat!( + "" + ).to_string(), + ] + } + } + ``` + +- [ ] **Step 3: Run tests** + + Run: `cargo test -p trusted-server-core integrations::gpt` + Expected: all pass including new test + +- [ ] **Step 4: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt.rs + git commit -m "Emit __tsAdInit function definition from GPT head injector" + ``` + +--- + +## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` + +**Files:** + +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` + +The TypeScript version is the authoritative implementation; it must mirror the Rust inline string from Task 9 exactly. + +- [ ] **Step 1: Write a failing test** + + In `crates/js/lib/src/integrations/gpt/index.test.ts`: + + ```typescript + import { describe, it, expect, vi, beforeEach } from 'vitest' + + describe('installTsAdInit', () => { + beforeEach(() => { + delete (window as any).__ts_ad_slots + delete (window as any).__ts_bids + delete (window as any).__tsAdInit + }) + + it('defines googletag slots from __ts_ad_slots and calls refresh', () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + getTargeting: vi.fn().mockReturnValue([]), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: { pos: 'atf' }, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + burl: 'https://ssp/bill', + }, + } + + // Must import installTsAdInit from the module + const { installTsAdInit } = require('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect((window as any).googletag.defineSlot).toHaveBeenCalledWith( + '/123/atf', + [[300, 250]], + 'atf' + ) + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') + expect(mockPubads.refresh).toHaveBeenCalled() + }) + + it('fires burl via sendBeacon on slotRenderEnded when our bid won', () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) + // ... setup and trigger slotRenderEnded event + // Verify: navigator.sendBeacon called with burl + beaconSpy.mockRestore() + }) + }) + ``` + + Run: `cd crates/js/lib && npx vitest run` + Expected: FAIL — `installTsAdInit` not exported + +- [ ] **Step 2: Add `installTsAdInit` to `index.ts`** + + Add to `crates/js/lib/src/integrations/gpt/index.ts` (bottom of file): + + ```typescript + interface TsAdSlot { + id: string + gam_unit_path: string + div_id: string + formats: Array + targeting: Record + } + + interface TsBidData { + hb_pb?: string + hb_bidder?: string + hb_adid?: string + burl?: string + } + + type TsWindow = Window & { + __ts_ad_slots?: TsAdSlot[] + __ts_bids?: Record + __tsAdInit?: () => void + } + + /** + * Install `window.__tsAdInit` — reads `window.__ts_ad_slots` and `window.__ts_bids` + * (injected by the edge into ), defines GPT slots, applies pre-won bid targeting, + * registers a `slotRenderEnded` listener to fire `burl` via `sendBeacon`, then calls + * `refresh()`. + */ + export function installTsAdInit(): void { + const w = window as TsWindow + w.__tsAdInit = function () { + const slots = w.__ts_ad_slots ?? [] + const bids = w.__ts_bids ?? {} + const g = (window as GptWindow).googletag + if (!g) return + g.cmd.push(() => { + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!gptSlot) return + gptSlot.addService(g.pubads()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + }) + g.pubads().enableSingleRequest() + g.enableServices() + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + if ( + !event.isEmpty && + bid.burl && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + ) { + navigator.sendBeacon(bid.burl) + } + }) + g.pubads().refresh() + }) + } + } + ``` + + Call `installTsAdInit()` from the integration's initialization path so it's set up when the bundle loads. + +- [ ] **Step 3: Run JS tests** + + Run: `cd crates/js/lib && npx vitest run` + Expected: new tests pass + +- [ ] **Step 4: Build JS bundle** + + Run: `cd crates/js/lib && node build-all.mjs` + Expected: clean build + +- [ ] **Step 5: Commit** + + ```bash + git add crates/js/lib/src/integrations/gpt/ + git commit -m "Add __tsAdInit and slotRenderEnded burl firing to GPT integration" + ``` + +--- + +## Task 11: `nurl` fire-and-forget + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Write failing test** + + ```rust + #[test] + fn prebid_config_fire_nurl_defaults_to_true() { + let config = PrebidConfig::default(); + assert!(config.fire_nurl_at_edge, "should fire nurl at edge by default"); + } + ``` + + Run: `cargo test -p trusted-server-core integrations::prebid` + Expected: FAIL + +- [ ] **Step 2: Add `fire_nurl_at_edge` to `PrebidConfig`** + + ```rust + #[serde(default = "default_fire_nurl_at_edge")] + pub fire_nurl_at_edge: bool, + ``` + + ```rust + fn default_fire_nurl_at_edge() -> bool { true } + ``` + +- [ ] **Step 3: Fire nurls in publisher.rs after auction** + + After `auction_result` is obtained, add: + + ```rust + if let Some(ref result) = auction_result { + fire_winning_nurls(result, settings); + } + ``` + + Add helper (no `.await` — fire-and-forget): + + ```rust + fn fire_winning_nurls( + result: &crate::auction::orchestrator::OrchestrationResult, + settings: &Settings, + ) { + use crate::backend::BackendConfig; + + let fire_nurl = settings + .integrations + .get_typed::("prebid") + .map(|c| c.fire_nurl_at_edge) + .unwrap_or(true); + + if !fire_nurl { + return; + } + + for bid in result.winning_bids.values() { + let Some(ref nurl) = bid.nurl else { continue }; + let backend_name = match BackendConfig::from_url(nurl, false) { + Ok(name) => name, + Err(e) => { + log::warn!("nurl: cannot create backend for {nurl}: {e:?}"); + continue; + } + }; + match fastly::Request::get(nurl).send_async(&backend_name) { + Ok(_) => log::debug!("nurl: fired for slot {}", bid.slot_id), + Err(e) => log::warn!("nurl: failed for slot {}: {e}", bid.slot_id), + } + } + } + ``` + +- [ ] **Step 4: Run tests** + + Run: `cargo test --workspace` + Expected: all pass + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/prebid.rs \ + crates/trusted-server-core/src/publisher.rs + git commit -m "Fire winning bid nurl fire-and-forget from edge; add fire_nurl_at_edge config" + ``` + +--- + +## Task 12: End-to-end integration tests + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (test module) + +Tests use `pub(crate)` helpers from Task 8 directly. + +- [ ] **Step 1: Write tests** + + In `publisher.rs` test module: + + ```rust + #[cfg(test)] + mod creative_opportunities_tests { + use super::{build_ad_slots_script, build_ad_bids_script, html_escape_for_script}; + use crate::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, CreativeOpportunityFormat, + CreativeOpportunitiesFile, match_slots, + }; + use crate::auction::types::{Bid, MediaType}; + use crate::price_bucket::PriceGranularity; + use std::collections::HashMap; + + fn make_config() -> CreativeOpportunitiesConfig { + CreativeOpportunitiesConfig { + gam_network_id: "21765378893".to_string(), + auction_timeout_ms: Some(500), + price_granularity: PriceGranularity::Dense, + } + } + + fn make_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "atf_sidebar_ad".to_string(), + gam_unit_path: Some("/21765378893/publisher/atf-sidebar".to_string()), + div_id: Some("div-atf-sidebar".to_string()), + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, height: 250, media_type: MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: [("pos".to_string(), "atf".to_string())].into_iter().collect(), + providers: Default::default(), + } + } + + #[test] + fn ad_slots_script_is_safe_and_parseable() { + let slots = vec![make_slot()]; + let config = make_config(); + let script = build_ad_slots_script(&slots, &config); + assert!(script.contains("window.__ts_ad_slots=JSON.parse"), "should use JSON.parse"); + assert!(script.contains("atf_sidebar_ad"), "should include slot id"); + // Verify no raw < or > that could break HTML parser + let inner = script.trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in script content"); + assert!(!inner.contains('>'), "no unescaped > in script content"); + } + + #[test] + fn ad_bids_script_uses_price_bucket_and_ad_id() { + let mut winning_bids = HashMap::new(); + winning_bids.insert("atf_sidebar_ad".to_string(), Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(2.53), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, height: 250, + nurl: None, + burl: Some("https://ssp.example/billing?id=abc123".to_string()), + ad_id: Some("prebid-uuid-abc123".to_string()), + metadata: HashMap::new(), + }); + let script = build_ad_bids_script(&winning_bids, PriceGranularity::Dense); + assert!(script.contains("\"hb_pb\":\"2.53\""), "should bucket 2.53 as 2.53 (dense)"); + assert!(script.contains("\"hb_bidder\":\"kargo\""), "should include bidder"); + assert!(script.contains("\"hb_adid\":\"prebid-uuid-abc123\""), "should use ad_id not creative markup"); + assert!(script.contains("burl"), "should include burl for billing"); + } + + #[test] + fn html_escape_neutralizes_xss_in_json() { + let malicious = r#"{"zone":""), "should escape "); + assert!(escaped.contains("\\u003c"), "should unicode-escape <"); + assert!(escaped.contains("\\u003e"), "should unicode-escape >"); + } + + #[test] + fn url_matching_end_to_end() { + let file = CreativeOpportunitiesFile { slots: vec![make_slot()] }; + assert_eq!(match_slots(&file.slots, "/2024/01/my-article").len(), 1, "should match article"); + assert_eq!(match_slots(&file.slots, "/about").len(), 0, "should not match /about"); + assert_eq!(match_slots(&file.slots, "/").len(), 0, "should not match root"); + } + } + ``` + +- [ ] **Step 2: Run tests** + + Run: `cargo test -p trusted-server-core creative_opportunities_tests` + Expected: all pass + +- [ ] **Step 3: Run full suite + CI gates** + + ```bash + cargo test --workspace + cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo fmt --all -- --check + cd crates/js/lib && npx vitest run + cd crates/js/lib && npm run format + cd docs && npm run format + ``` + + Expected: all clean + +- [ ] **Step 4: Commit** + + ```bash + git add crates/trusted-server-core/src/publisher.rs + git commit -m "Add integration tests for creative opportunities pipeline (slots, bids, XSS)" + ``` + +--- + +## Manual Verification Checklist + +Run `fastly compute serve` and verify: + +- [ ] **No match:** Request `/about` — no `__ts_ad_slots` or `__ts_bids` in response HTML, no `Cache-Control: private, no-store` +- [ ] **Match:** Request `/2024/01/article` — both globals present in ``, `Cache-Control: private, no-store` set +- [ ] **Empty file kill-switch:** Empty `creative-opportunities.toml` → no globals injected on any URL +- [ ] **Auction timeout:** Set `auction_timeout_ms = 1` → `__ts_bids` injects as `{}`, no slot entries +- [ ] **XSS check:** Add `targeting = { zone = " +``` + +> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped +> before insertion into the ``, not -> raw string interpolation. +- If the auction has already completed for ``, response returns immediately + with cached results (cache hit). Typical case for non-trivial origin times. +- If the auction is still in flight, the request blocks until completion or `A_deadline`, + whichever fires first. Long-poll semantics, capped by the auction timeout. +- If `` is unknown (cache miss, expired TTL, or never created), returns + `404`. Client falls back to firing GPT without pre-set targeting. +- If no slot received a bid above floor, returns `{}`. Client fires GPT without targeting. +- Response carries `Cache-Control: private, no-store`. -> **Cache contract:** Any response with `__ts_bids` injected is per-user data and must -> not be cached. TS sets `Cache-Control: private, no-store` on the response before -> forwarding, overriding any conflicting cache headers from the publisher origin. -> `Surrogate-Control` and `Fastly-Surrogate-Control` are also stripped. +**Storage:** auction results cached in-process (per-edge-instance) keyed by request ID +with a 30-second TTL. Sized small (a few KB per entry) and short-lived; no Fastly KV +write on the hot path. + +**Security:** request IDs are 128-bit unguessable UUIDs. Even if a request ID leaks, the +worst-case impact is reading bid metadata that's already destined for that session's +GPT slots — no cross-user data exposure. ### 4.5 Win Notifications @@ -386,119 +455,357 @@ Prebid bid (not a direct deal or backfill) won the GAM line item match. ### 4.6 Client Residual Prebid.js is eliminated. The client-side ad bootstrap is replaced by a small inline -script (~30 lines) that reads `__ts_ad_slots` and `__ts_bids`, drives GPT directly, and -handles billing notifications: +script that reads `__ts_ad_slots`, fetches bids from `/ts-bids`, drives GPT directly, +and handles billing notifications. Slot definition happens immediately; bid targeting +and `refresh()` happen after `/ts-bids` resolves: ```javascript window.__tsAdInit = function () { var slots = window.__ts_ad_slots || [] - var bids = window.__ts_bids || {} + var rid = window.__ts_request_id + + // Kick off bid fetch as early as possible. Fires in parallel with GPT setup. + var bidsPromise = rid + ? fetch('/ts-bids?rid=' + encodeURIComponent(rid), { credentials: 'omit' }) + .then(function (r) { + return r.ok ? r.json() : {} + }) + .catch(function () { + return {} + }) + : Promise.resolve({}) + googletag.cmd.push(function () { - slots.forEach(function (slot) { + // Define slots immediately — no auction wait + var gptSlots = slots.map(function (slot) { var gptSlot = googletag .defineSlot(slot.gam_unit_path, slot.formats, slot.div_id) .addService(googletag.pubads()) - // Apply static targeting from config Object.entries(slot.targeting).forEach(function ([k, v]) { gptSlot.setTargeting(k, v) }) - // Apply pre-won bid targeting if available - var bidData = bids[slot.id] || {} - ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (key) { - if (bidData[key]) gptSlot.setTargeting(key, bidData[key]) - }) + return { id: slot.id, gptSlot: gptSlot } }) + googletag.pubads().enableSingleRequest() googletag.enableServices() - // Fire burl on confirmed render - googletag.pubads().addEventListener('slotRenderEnded', function (event) { - var slotId = event.slot.getSlotElementId() - var bidData = bids[slotId] || {} - if ( - !event.isEmpty && - bidData.burl && - event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid - ) { - navigator.sendBeacon(bidData.burl) - } + + // Apply bid targeting and refresh once /ts-bids resolves. + bidsPromise.then(function (bids) { + gptSlots.forEach(function ({ id, gptSlot }) { + var bidData = bids[id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (key) { + if (bidData[key]) gptSlot.setTargeting(key, bidData[key]) + }) + }) + + // Fire burl on confirmed render + googletag.pubads().addEventListener('slotRenderEnded', function (event) { + var slotId = event.slot.getSlotElementId() + var bidData = bids[slotId] || {} + if ( + !event.isEmpty && + bidData.burl && + event.slot.getTargeting('hb_adid')[0] === bidData.hb_adid + ) { + navigator.sendBeacon(bidData.burl) + } + }) + + googletag.pubads().refresh() }) - googletag.pubads().refresh() }) } ``` +**Why slot definition happens before bid fetch resolves:** GPT slot definition is +synchronous and cheap. Defining slots early lets GPT prepare iframes and start any +internal work that doesn't require ad server response. `refresh()` is the call that +actually triggers the GAM ad request — that's the one we delay until bids arrive. + +**Failure modes:** + +- `/ts-bids` returns 404 (unknown rid, TTL expired) → `bidsPromise` resolves to `{}`, + `refresh()` fires without bid targeting, GAM falls back to its own auction. Same + graceful degradation as no-bid case. +- `/ts-bids` network failure → caught, resolves to `{}`, same fallback. +- Auction times out server-side → `/ts-bids` returns `{}`, same fallback. + This script is part of the existing `gpt` integration bundle (`crates/js/lib/src/integrations/gpt/index.ts`), extending the existing GPT shim. Injected via the `gpt` head injector alongside `window.__ts_ad_slots`. +### 4.7 Caching Behavior + +Page assets and bid results have very different cacheability properties. The +architecture is designed so that everything that can be cached, is. + +**What gets cached where:** + +| Asset | Cached at | Cacheability | +| ------------------------ | -------------------------------- | --------------------------------------------------------- | +| Origin HTML | Fastly edge HTTP cache | Yes, if origin sends `Cache-Control: public, max-age=...` | +| Origin CSS / fonts / JS | Fastly edge + browser | Yes (typically hashed URLs, immutable) | +| `tsjs` bundle | Fastly edge + browser | Yes (already content-hashed via `bundle.rs`, immutable) | +| `__ts_ad_slots` payload | Could be precomputed per pattern | In-memory match is sub-millisecond — not worth caching | +| `__ts_request_id` | **Never** | Per-request UUID, minted at request receipt | +| Bid results (`/ts-bids`) | In-process `bid_cache`, 30s TTL | Per-request, never shared across users | + +**Architecture:** + +1. Fastly's built-in HTTP cache stores the **origin response** keyed by URL. TS + does not implement its own HTML caching layer — it leverages the existing + Fastly cache. +2. On request: TS reads from cache (cache hit, ~5ms) or fetches from origin + (cache miss, ~150ms typical). +3. TS injects `__ts_ad_slots` + `__ts_request_id` at the `` open via the + existing `el.prepend()` head handler. This injection is per-request — origin + HTML in cache is unmodified. +4. TS forces `Transfer-Encoding: chunked` and streams the assembled response + to the browser. +5. The auction runs in parallel regardless of HTML cache state — bids land in + `bid_cache` keyed by `request_id`, served via `/ts-bids` when the client + fetches. + +The `bid_cache` (per-request bid results) and Fastly's HTML cache are +**independent systems**. HTML cache hit/miss does not affect auction firing; +auction firing does not affect HTML caching. + +**`Cache-Control` handling:** + +TS preserves the origin's `Cache-Control` header on the response sent to the +browser, with one override: when `__ts_request_id` is injected (any matched +page), TS sets `Cache-Control: private, no-store` on the **browser-facing** +response to prevent intermediate caches or the browser from caching the +per-user assembled HTML. The Fastly edge cache for the **origin** response is +unaffected — TS reads the cached origin HTML and assembles a fresh per-request +response on every hit. + +`Surrogate-Control` and `Fastly-Surrogate-Control` headers from origin are +preserved (they control Fastly's cache, not the browser's). + +**When caching doesn't apply:** + +- **Logged-in users** — origin typically returns `Cache-Control: private`. Falls + back to cache-miss timing (full origin fetch). +- **Personalized SSR** (per-user content, A/B test variants) — same. +- **Dynamic NextJS routes without ISR** — origin sends `Cache-Control: no-store` + or short max-age. Falls back to cache-miss timing. +- **First request after deploy or cache purge** — cold cache, full origin fetch. +- **Long-tail URLs** — low cache hit rate, treat as cache-miss case. + +For typical news / content publisher sites with anonymous visitors on stable +content pages, expect 70–90%+ edge cache hit rate. The cache-hit timing in §5 +is the realistic common case, not the optimistic best case. + --- ## 5. Request-Time Sequence +Sequence applies to all origins (WordPress, Drupal, Rails, NextJS 14/16, static sites). +TS forces chunked encoding on every response, so origin format is invisible from the +browser's perspective. + +### 5.1 Visual Sequence (full content + creative flow) + +```mermaid +sequenceDiagram + autonumber + participant B as Browser + participant E as TS Edge
(Fastly) + participant C as Fastly HTTP Cache + participant O as Publisher Origin
(WP / NextJS / etc) + participant A as Auction
(PBS + APS) + participant S as SSPs
(Kargo / Index / etc) + participant G as GAM
(securepubads) + + Note over B,G: t=0ms — Navigation start + + B->>E: GET ts.publisher.com/article + + Note over E: t=1ms — URL → slots match
Mint request_id (UUID)
Check consent + + par Auction kicks off server-side + E->>A: POST bid requests
(PBS + APS in parallel) + A->>S: Fan out to all SSPs + S-->>A: Bids return + A-->>E: Aggregated bid responses
(t=502ms) + Note over E: Cache bids in bid_cache
(keyed by request_id, 30s TTL) + E->>S: Fire nurl (fire-and-forget)
for winning bids + and Origin HTML lookup + E->>C: Lookup origin HTML by URL + alt Cache HIT (typical for content pages) + C-->>E: Cached HTML (~5ms) + else Cache MISS (cold / dynamic / logged-in) + C->>O: GET origin HTML + O-->>C: HTML response (~150ms) + C-->>E: HTML response + end + end + + Note over E: Force Transfer-Encoding: chunked
Inject __ts_ad_slots + __ts_request_id
at open
Set Cache-Control: private, no-store + + E-->>B: Stream HTML chunks (no auction wait) + + Note over B: TTFB: ~10ms (hit) / ~155ms (miss)
Browser parses
CSS, fonts, tsjs download
(also from Fastly + browser cache) + + Note over B: flushes immediately
Body parsing begins
🎨 FCP: ~80ms (hit) / ~250ms (miss) + + Note over B: tsjs bundle executes
t=130ms (hit) / t=300ms (miss)
__tsAdInit() defines GPT slots
(no GAM call yet) + + B->>E: GET /ts-bids?rid= + + alt Auction already complete (typical on cache-hit pages) + Note over E: bid_cache hit — return immediately + E-->>B: Bid targeting JSON
(hb_pb, hb_bidder, hb_adid, burl) + else Auction still running + Note over E: Long-poll — block until
auction completes or A_deadline + A-->>E: Bids arrive + E-->>B: Bid targeting JSON
(or {} on timeout) + end + + Note over B: Bids received (~30ms RTT)
setTargeting(hb_*) per slot
Register slotRenderEnded listener
googletag.pubads().refresh() fires + + B->>G: GET /gampad/ads
with hb_* key-values + + Note over G: GAM matches hb_pb against
Prebid line items, selects winner + + G-->>B: Ad markup
(iframe HTML or creative URL) + + Note over B: Creative iframe loads in slot
Fetches sub-resources
(images, scripts, viewability pixels) + + Note over B: 🎯 Creative paints
slotRenderEnded event fires
__tsAdInit checks hb_adid match + + alt Our Prebid bid won the GAM line item match + B->>S: Fire burl (navigator.sendBeacon)
SSP confirms billable impression + else Direct deal / backfill won (hb_adid mismatch or empty) + Note over B: No burl fired — our bid lost
(correct behavior — different creative rendered) + end + + Note over B: window.load fires
(page fully loaded) + + Note over B,G: ✅ AD VISIBLE
Cache hit: ~900ms total
Cache miss: ~1,050ms total
FCP: ~80ms (hit) / ~250ms (miss)

vs client-side today: ~3,250ms ad-visible / FCP ~500ms+ +``` + +### 5.2 Cache-Hit Sequence (typical for content publisher pages) + +This is the common case for anonymous visitors on cacheable content pages. + ``` t=0ms GET ts.publisher.com/article arrives at Fastly edge t=1ms URL matched against creative-opportunities.toml Slots matched: [atf_sidebar_ad, below-content-ad, section_ad] Consent check: TCF consent present → auction proceeds + Request ID minted: 550e8400-e29b-41d4-a716-446655440000 -t=2ms AuctionOrchestrator.run_auction() called +t=2ms AuctionOrchestrator.run_auction() dispatched (parallel) PBS + APS dispatched in parallel via send_async() Edge→PBS RTT: ~20–30ms + Fastly cache lookup dispatched in parallel + __ts_ad_slots + __ts_request_id ".to_string() + r#""# + .to_string() ), - ad_bids_script: None, }; let mut processor = create_html_processor(config); let output = processor .process_chunk(b"T", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); - assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots"); + assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots at head-open"); + assert!(html.contains("window.__ts_request_id"), "should inject request_id at head-open"); } #[test] - fn injects_bids_before_end_of_head() { - let bids_script = ""; + fn does_not_hold_end_of_head() { + // Verify: no bid data appears before — that hold was rejected by spec §4.3 let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, - ad_bids_script: Some(bids_script.to_string()), }; let mut processor = create_html_processor(config); let output = processor .process_chunk(b"T", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); - assert!(html.contains("window.__ts_bids"), "should inject bids"); - let bids_pos = html.find("window.__ts_bids").expect("should find bids"); - let end_head_pos = html.find("").expect("should find "); - assert!(bids_pos < end_head_pos, "bids script should appear before "); + assert!(!html.contains("__ts_bids"), "must not inject bids into head"); } ``` Run: `cargo test -p trusted-server-core html_processor` - Expected: compile error (no `ad_slots_script`/`ad_bids_script` fields, no `empty_for_tests()`) + Expected: compile error (no `ad_slots_script` field, no `empty_for_tests()`) - [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** @@ -888,7 +886,6 @@ Adding the two new fields to `HtmlProcessorConfig` and the injection logic is in #[cfg(test)] impl IntegrationRegistry { pub fn empty_for_tests() -> Self { - // Minimal registry with no integrations for unit testing html_processor Self { inner: Arc::new(RegistryInner { proxies: Default::default(), @@ -905,7 +902,9 @@ Adding the two new fields to `HtmlProcessorConfig` and the injection logic is in (Adjust field names to match the actual `RegistryInner` struct.) -- [ ] **Step 3: Add fields to `HtmlProcessorConfig`** +- [ ] **Step 3: Add single field to `HtmlProcessorConfig`** + + Replace any existing `ad_slots_script`/`ad_bids_script` fields with: ```rust pub struct HtmlProcessorConfig { @@ -913,56 +912,47 @@ Adding the two new fields to `HtmlProcessorConfig` and the injection logic is in pub request_host: String, pub request_scheme: String, pub integrations: IntegrationRegistry, - /// Pre-computed `` for matched slots. - /// Injected at open, before integration head inserts. `None` when no slots matched. + /// Pre-computed ``. + /// Injected at `` open, before integration head inserts. `None` when no slots matched. pub ad_slots_script: Option, - /// Pre-computed `` for winning bids. - /// Injected immediately before via on_end_tag(). `None` when auction not run. - pub ad_bids_script: Option, } ``` - Update `from_settings` to initialize `ad_slots_script: None, ad_bids_script: None`. + Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_slots_script: None`. -- [ ] **Step 4: Inject `__ts_ad_slots` at head-open AND register `on_end_tag` for `__ts_bids`** +- [ ] **Step 4: Inject `ad_slots_script` at head-open** - In `create_html_processor`, within the EXISTING single `element!("head", ...)` handler, make two changes: - 1. Prepend the ad slots script BEFORE the existing integration inserts: + In `create_html_processor`, within the EXISTING `element!("head", ...)` handler, build the full snippet string with `ad_slots_script` first (so it appears first in output — lol_html `prepend` inserts before children, with **last-prepend-wins** ordering, so we call `prepend` exactly once with the full combined string): - ```rust - // NEW: inject __ts_ad_slots first - if let Some(ref slots_script) = ad_slots_script { - snippet.push_str(slots_script); - } - // ... existing: for insert in integrations.head_inserts(&ctx) { ... } - ``` + ```rust + let ad_slots_script = config.ad_slots_script.clone(); + // ... existing captures ... - 2. After `el.prepend(...)`, register the end-tag handler for `__ts_bids`: - ```rust - // Register on_end_tag handler for __ts_bids injection before - if let Some(bids_script) = ad_bids_script.clone() { - el.on_end_tag(move |end_tag| { - end_tag.before(&bids_script, ContentType::Html); - Ok(()) - })?; - } - ``` + element!("head", |el| { + let mut snippet = String::new(); - Both changes live inside the same `element!("head", ...)` closure — no second handler needed. + // ad_slots_script first so __ts_ad_slots + __ts_request_id appear before + // integration inserts. DO NOT call prepend multiple times — lol_html stacks + // prepend calls in reverse order, so a single prepend with the full string + // guarantees correct ordering. + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } - Capture `ad_slots_script` and `ad_bids_script` into the closure the same way as `injected_tsjs`: + // ... existing: for insert in integrations.head_inserts(&ctx) { snippet.push_str(...) } - ```rust - let ad_slots_script = config.ad_slots_script.clone(); - let ad_bids_script = config.ad_bids_script.clone(); + if !snippet.is_empty() { + el.prepend(&snippet, ContentType::Html); + } + // DO NOT register on_end_tag — flushes immediately per spec §4.3 + Ok(()) + }) ``` - > **lol_html `on_end_tag` API note:** `Element::on_end_tag(handler)` is available in lol_html ≥2.0. The handler receives `&mut EndTag` and must return `Result<(), Box>`. Use `ContentType::Html` so the injected `", escaped) + let slots_json_str = serde_json::to_string(&slots_json) + .expect("should serialize ad slots"); + let escaped_slots = html_escape_for_script(&slots_json_str); + // request_id is a UUID (hex + hyphens only) — safe to embed without escaping. + format!( + r#""# + ) } - pub(crate) fn build_ad_bids_script( + /// Build the `BidMap` stored in `bid_cache` and returned by `/ts-bids`. + /// + /// Keyed by slot ID. Values contain `hb_pb`, `hb_bidder`, `hb_adid`, `burl`. + pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, price_granularity: crate::price_bucket::PriceGranularity, - ) -> String { - let bids_map: serde_json::Map = winning_bids + ) -> crate::bid_cache::BidMap { + winning_bids .iter() .filter_map(|(slot_id, bid)| { let cpm = bid.price?; - let entry = serde_json::json!({ - "hb_pb": price_bucket(cpm, price_granularity), - "hb_bidder": bid.bidder, - "hb_adid": bid.ad_id.as_deref().unwrap_or(""), - "burl": bid.burl, - }); - Some((slot_id.clone(), entry)) + let entry: std::collections::HashMap = [ + ("hb_pb".to_string(), serde_json::Value::String(price_bucket(cpm, price_granularity))), + ("hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone())), + ("hb_adid".to_string(), serde_json::Value::String( + bid.ad_id.as_deref().unwrap_or("").to_string() + )), + ("burl".to_string(), bid.burl.as_deref() + .map(serde_json::Value::from) + .unwrap_or(serde_json::Value::Null)), + ].into_iter().collect(); + Some((slot_id.clone(), entry.into_iter() + .map(|(k, v)| (k, v)) + .collect::>() + .into())) }) - .collect(); - let json = serde_json::to_string(&serde_json::Value::Object(bids_map)) - .expect("should serialize bids"); - let escaped = html_escape_for_script(&json); - format!("", escaped) + .collect() } /// HTML-escape a JSON string for safe inline `" .to_string(), - // __tsAdInit definition — reads window.__ts_ad_slots / __ts_bids at call time. + // __tsAdInit: fetches /ts-bids for bid targeting, then drives GPT. + // window.__ts_ad_slots and window.__ts_request_id are injected at head-open by TS. + // bidsPromise resolves concurrently with page rendering — never blocks FCP. concat!( "" @@ -1394,20 +1825,20 @@ The `HtmlProcessorConfig` fields now exist (Task 7). This task wires the auction ```bash git add crates/trusted-server-core/src/integrations/gpt.rs - git commit -m "Emit __tsAdInit function definition from GPT head injector" + git commit -m "Emit __tsAdInit with /ts-bids fetch pattern from GPT head injector" ``` --- -## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` +## Task 12: `gpt/index.ts` — TypeScript `__tsAdInit` with `/ts-bids` fetch **Files:** - Modify: `crates/js/lib/src/integrations/gpt/index.ts` -The TypeScript version is the authoritative implementation; it must mirror the Rust inline string from Task 9 exactly. +The TypeScript version mirrors the Rust inline string from Task 11. It uses the `bidsPromise` pattern — fetching `/ts-bids` concurrently with GPT slot definition. -- [ ] **Step 1: Write a failing test** +- [ ] **Step 1: Write failing tests** In `crates/js/lib/src/integrations/gpt/index.test.ts`: @@ -1417,20 +1848,21 @@ The TypeScript version is the authoritative implementation; it must mirror the R describe('installTsAdInit', () => { beforeEach(() => { delete (window as any).__ts_ad_slots - delete (window as any).__ts_bids + delete (window as any).__ts_request_id delete (window as any).__tsAdInit }) - it('defines googletag slots from __ts_ad_slots and calls refresh', () => { + it('fetches /ts-bids with request_id and applies bid targeting before refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue([]), } const mockPubads = { enableSingleRequest: vi.fn(), addEventListener: vi.fn(), refresh: vi.fn(), - getTargeting: vi.fn().mockReturnValue([]), } ;(window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, @@ -1447,45 +1879,131 @@ The TypeScript version is the authoritative implementation; it must mirror the R targeting: { pos: 'atf' }, }, ] - ;(window as any).__ts_bids = { - atf: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'abc', - burl: 'https://ssp/bill', - }, - } + ;(window as any).__ts_request_id = 'test-rid-123' + + const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + burl: 'https://ssp/bill', + }, + }), + } as Response) - // Must import installTsAdInit from the module - const { installTsAdInit } = require('./index') + const { installTsAdInit } = await import('./index') installTsAdInit() - ;(window as any).__tsAdInit() + await (window as any).__tsAdInit() - expect((window as any).googletag.defineSlot).toHaveBeenCalledWith( - '/123/atf', - [[300, 250]], - 'atf' + expect(fetchSpy).toHaveBeenCalledWith( + expect.stringContaining('/ts-bids?rid=test-rid-123'), + expect.objectContaining({ credentials: 'omit' }) ) expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') expect(mockPubads.refresh).toHaveBeenCalled() + + fetchSpy.mockRestore() + }) + + it('calls refresh with empty bids when fetch fails', async () => { + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [] + ;(window as any).__ts_request_id = 'rid-fail' + + vi.spyOn(global, 'fetch').mockRejectedValue(new Error('network error')) + + const { installTsAdInit } = await import('./index') + installTsAdInit() + await (window as any).__tsAdInit() + + expect(mockPubads.refresh).toHaveBeenCalled() }) - it('fires burl via sendBeacon on slotRenderEnded when our bid won', () => { + it('fires burl via sendBeacon on slotRenderEnded when our bid won', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) - // ... setup and trigger slotRenderEnded event - // Verify: navigator.sendBeacon called with burl + let capturedListener: ((e: any) => void) | undefined + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['abc']), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn + }), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ] + ;(window as any).__ts_request_id = 'rid-burl-test' + + vi.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + burl: 'https://ssp/bill', + }, + }), + } as Response) + + const { installTsAdInit } = await import('./index') + installTsAdInit() + await (window as any).__tsAdInit() + + // Trigger slotRenderEnded — slot has our winning hb_adid + expect(capturedListener).toBeDefined() + capturedListener!({ + isEmpty: false, + slot: mockSlot, + }) + + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') beaconSpy.mockRestore() }) }) ``` Run: `cd crates/js/lib && npx vitest run` - Expected: FAIL — `installTsAdInit` not exported + Expected: FAIL — `installTsAdInit` not exported or fetches wrong endpoint - [ ] **Step 2: Add `installTsAdInit` to `index.ts`** - Add to `crates/js/lib/src/integrations/gpt/index.ts` (bottom of file): + Add to `crates/js/lib/src/integrations/gpt/index.ts`: ```typescript interface TsAdSlot { @@ -1505,60 +2023,87 @@ The TypeScript version is the authoritative implementation; it must mirror the R type TsWindow = Window & { __ts_ad_slots?: TsAdSlot[] - __ts_bids?: Record + __ts_request_id?: string __tsAdInit?: () => void } /** - * Install `window.__tsAdInit` — reads `window.__ts_ad_slots` and `window.__ts_bids` - * (injected by the edge into ), defines GPT slots, applies pre-won bid targeting, - * registers a `slotRenderEnded` listener to fire `burl` via `sendBeacon`, then calls - * `refresh()`. + * Install `window.__tsAdInit`. + * + * Reads `window.__ts_ad_slots` and `window.__ts_request_id` (both injected by + * the edge at `` open). Fetches bid results from `/ts-bids?rid=` + * concurrently with GPT slot definition. Applies targeting and calls `refresh()` + * after the fetch resolves. Registers `slotRenderEnded` to fire `burl` via + * `sendBeacon` when our specific Prebid bid wins the GAM line item match. */ export function installTsAdInit(): void { const w = window as TsWindow w.__tsAdInit = function () { const slots = w.__ts_ad_slots ?? [] - const bids = w.__ts_bids ?? {} + const rid = w.__ts_request_id + + const bidsPromise: Promise> = rid + ? fetch(`/ts-bids?rid=${encodeURIComponent(rid)}`, { + credentials: 'omit', + }) + .then((r) => (r.ok ? r.json() : {})) + .catch(() => ({})) + : Promise.resolve({}) + const g = (window as GptWindow).googletag if (!g) return + g.cmd.push(() => { - slots.forEach((slot) => { - const gptSlot = g.defineSlot?.( - slot.gam_unit_path, - slot.formats, - slot.div_id - ) - if (!gptSlot) return - gptSlot.addService(g.pubads()) - Object.entries(slot.targeting ?? {}).forEach(([k, v]) => - gptSlot.setTargeting(k, v) - ) - const bid = bids[slot.id] ?? {} - ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + const gptSlots = slots + .map((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!gptSlot) return null + gptSlot.addService(g.pubads()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + return { id: slot.id, gptSlot } }) - }) + .filter(Boolean) as Array<{ + id: string + gptSlot: NonNullable> + }> + g.pubads().enableSingleRequest() g.enableServices() - g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? '' - const bid = bids[slotId] ?? {} - if ( - !event.isEmpty && - bid.burl && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid - ) { - navigator.sendBeacon(bid.burl) - } + + bidsPromise.then((bids) => { + gptSlots.forEach(({ id, gptSlot }) => { + const bid = bids[id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + }) + + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + if ( + !event.isEmpty && + bid.burl && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + ) { + navigator.sendBeacon(bid.burl) + } + }) + + g.pubads().refresh() }) - g.pubads().refresh() }) } } ``` - Call `installTsAdInit()` from the integration's initialization path so it's set up when the bundle loads. + Call `installTsAdInit()` from the integration's initialization path. - [ ] **Step 3: Run JS tests** @@ -1574,12 +2119,12 @@ The TypeScript version is the authoritative implementation; it must mirror the R ```bash git add crates/js/lib/src/integrations/gpt/ - git commit -m "Add __tsAdInit and slotRenderEnded burl firing to GPT integration" + git commit -m "Add installTsAdInit with /ts-bids fetch pattern and slotRenderEnded burl firing" ``` --- -## Task 11: `nurl` fire-and-forget +## Task 13: `nurl` fire-and-forget **Files:** @@ -1610,9 +2155,9 @@ The TypeScript version is the authoritative implementation; it must mirror the R fn default_fire_nurl_at_edge() -> bool { true } ``` -- [ ] **Step 3: Fire nurls in publisher.rs after auction** +- [ ] **Step 3: Fire nurls in publisher.rs after bid_cache.put()** - After `auction_result` is obtained, add: + After the `bid_cache.put(...)` call (Task 9 Step 3), add: ```rust if let Some(ref result) = auction_result { @@ -1620,7 +2165,7 @@ The TypeScript version is the authoritative implementation; it must mirror the R } ``` - Add helper (no `.await` — fire-and-forget): + Add helper: ```rust fn fire_winning_nurls( @@ -1671,13 +2216,13 @@ The TypeScript version is the authoritative implementation; it must mirror the R --- -## Task 12: End-to-end integration tests +## Task 14: End-to-end integration tests **Files:** - Modify: `crates/trusted-server-core/src/publisher.rs` (test module) -Tests use `pub(crate)` helpers from Task 8 directly. +Tests use `pub(crate)` helpers from Task 9 directly. - [ ] **Step 1: Write tests** @@ -1686,7 +2231,7 @@ Tests use `pub(crate)` helpers from Task 8 directly. ```rust #[cfg(test)] mod creative_opportunities_tests { - use super::{build_ad_slots_script, build_ad_bids_script, html_escape_for_script}; + use super::{build_head_globals_script, build_bid_map, html_escape_for_script}; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunitySlot, CreativeOpportunityFormat, CreativeOpportunitiesFile, match_slots, @@ -1719,20 +2264,32 @@ Tests use `pub(crate)` helpers from Task 8 directly. } #[test] - fn ad_slots_script_is_safe_and_parseable() { + fn head_globals_script_contains_ad_slots_and_request_id() { let slots = vec![make_slot()]; let config = make_config(); - let script = build_ad_slots_script(&slots, &config); - assert!(script.contains("window.__ts_ad_slots=JSON.parse"), "should use JSON.parse"); + let rid = "550e8400-e29b-41d4-a716-446655440000"; + let script = build_head_globals_script(&slots, rid, &config); + assert!(script.contains("window.__ts_ad_slots=JSON.parse"), "should use JSON.parse for slots"); assert!(script.contains("atf_sidebar_ad"), "should include slot id"); - // Verify no raw < or > that could break HTML parser - let inner = script.trim_start_matches(""); + assert!(script.contains(&format!("window.__ts_request_id=\"{rid}\"")), "should include request_id"); + assert!(!script.contains("__ts_bids"), "must NOT contain bids — bids come from /ts-bids"); + } + + #[test] + fn head_globals_script_is_xss_safe() { + let slots = vec![make_slot()]; + let config = make_config(); + let script = build_head_globals_script(&slots, "safe-rid", &config); + // Strip outer "); assert!(!inner.contains('<'), "no unescaped < in script content"); assert!(!inner.contains('>'), "no unescaped > in script content"); } #[test] - fn ad_bids_script_uses_price_bucket_and_ad_id() { + fn bid_map_uses_price_bucket_and_ad_id() { let mut winning_bids = HashMap::new(); winning_bids.insert("atf_sidebar_ad".to_string(), Bid { slot_id: "atf_sidebar_ad".to_string(), @@ -1747,11 +2304,23 @@ Tests use `pub(crate)` helpers from Task 8 directly. ad_id: Some("prebid-uuid-abc123".to_string()), metadata: HashMap::new(), }); - let script = build_ad_bids_script(&winning_bids, PriceGranularity::Dense); - assert!(script.contains("\"hb_pb\":\"2.53\""), "should bucket 2.53 as 2.53 (dense)"); - assert!(script.contains("\"hb_bidder\":\"kargo\""), "should include bidder"); - assert!(script.contains("\"hb_adid\":\"prebid-uuid-abc123\""), "should use ad_id not creative markup"); - assert!(script.contains("burl"), "should include burl for billing"); + let bid_map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let slot_bids = bid_map.get("atf_sidebar_ad").expect("should have slot bids"); + assert_eq!( + slot_bids.get("hb_pb").and_then(|v| v.as_str()), + Some("2.53"), + "should bucket 2.53 as 2.53 (dense)" + ); + assert_eq!( + slot_bids.get("hb_bidder").and_then(|v| v.as_str()), + Some("kargo"), + "should include bidder" + ); + assert_eq!( + slot_bids.get("hb_adid").and_then(|v| v.as_str()), + Some("prebid-uuid-abc123"), + "should use ad_id not creative markup" + ); } #[test] @@ -1795,7 +2364,7 @@ Tests use `pub(crate)` helpers from Task 8 directly. ```bash git add crates/trusted-server-core/src/publisher.rs - git commit -m "Add integration tests for creative opportunities pipeline (slots, bids, XSS)" + git commit -m "Add integration tests for creative opportunities pipeline (head globals, bid map, XSS)" ``` --- @@ -1804,19 +2373,27 @@ Tests use `pub(crate)` helpers from Task 8 directly. Run `fastly compute serve` and verify: -- [ ] **No match:** Request `/about` — no `__ts_ad_slots` or `__ts_bids` in response HTML, no `Cache-Control: private, no-store` -- [ ] **Match:** Request `/2024/01/article` — both globals present in ``, `Cache-Control: private, no-store` set -- [ ] **Empty file kill-switch:** Empty `creative-opportunities.toml` → no globals injected on any URL -- [ ] **Auction timeout:** Set `auction_timeout_ms = 1` → `__ts_bids` injects as `{}`, no slot entries -- [ ] **XSS check:** Add `targeting = { zone = " -``` - -> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped -> before insertion into the `, ContentType::Html)`. + +> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped +> before insertion into the `"# - .to_string() + r#""#.to_string() ), + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), }; let mut processor = create_html_processor(config); let output = processor - .process_chunk(b"T", true) + .process_chunk(b"Tcontent", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots at head-open"); - assert!(html.contains("window.__ts_request_id"), "should inject request_id at head-open"); + assert!(!html.contains("__ts_request_id"), "must NOT inject request_id — body-injection arch has no request_id"); } #[test] - fn does_not_hold_end_of_head() { - // Verify: no bid data appears before — that hold was rejected by spec §4.3 + fn injects_ts_bids_before_body_close() { + let bids_script = r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new( + Some(bids_script.to_string()) + )); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + ad_bids_state: state, }; let mut processor = create_html_processor(config); let output = processor - .process_chunk(b"T", true) + .process_chunk(b"content", true) .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); - assert!(!html.contains("__ts_bids"), "must not inject bids into head"); + assert!(html.contains("window.__ts_bids"), "should inject bids before "); + let bids_pos = html.find("window.__ts_bids").expect("bids should be in output"); + let body_close_pos = html.find("").expect(" should be in output"); + assert!(bids_pos < body_close_pos, "bids must appear before "); + } + + #[test] + fn injects_empty_ts_bids_when_state_is_none() { + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("__ts_bids=JSON.parse(\"{}\""), "should inject empty bids on None state"); } ``` Run: `cargo test -p trusted-server-core html_processor` - Expected: compile error (no `ad_slots_script` field, no `empty_for_tests()`) + Expected: compile error (no `ad_bids_state` field yet) - [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** @@ -902,9 +930,7 @@ The `hb_pb` value in bid responses is a discretized bucket string from Prebid's (Adjust field names to match the actual `RegistryInner` struct.) -- [ ] **Step 3: Add single field to `HtmlProcessorConfig`** - - Replace any existing `ad_slots_script`/`ad_bids_script` fields with: +- [ ] **Step 3: Update `HtmlProcessorConfig`** ```rust pub struct HtmlProcessorConfig { @@ -912,362 +938,104 @@ The `hb_pb` value in bid responses is a discretized bucket string from Prebid's pub request_host: String, pub request_scheme: String, pub integrations: IntegrationRegistry, - /// Pre-computed ``. - /// Injected at `` open, before integration head inserts. `None` when no slots matched. + /// Pre-computed ``. + /// Injected at `` open. `None` when no slots matched. pub ad_slots_script: Option, + /// Shared auction result script — written by the auction task before HTML processing + /// begins. Handler reads this in `el.on_end_tag()` on the body element. + /// `None` means no auction ran (consent denied, bot UA, no slot match, etc.); + /// inject empty `__ts_bids = {}` as graceful fallback. + pub ad_bids_state: std::sync::Arc>>, } ``` - Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_slots_script: None`. + Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_bids_state: Arc::new(RwLock::new(None))`. - [ ] **Step 4: Inject `ad_slots_script` at head-open** - In `create_html_processor`, within the EXISTING `element!("head", ...)` handler, build the full snippet string with `ad_slots_script` first (so it appears first in output — lol_html `prepend` inserts before children, with **last-prepend-wins** ordering, so we call `prepend` exactly once with the full combined string): + In `create_html_processor`, within the existing `element!("head", ...)` handler: ```rust let ad_slots_script = config.ad_slots_script.clone(); - // ... existing captures ... + // existing captures... element!("head", |el| { let mut snippet = String::new(); - - // ad_slots_script first so __ts_ad_slots + __ts_request_id appear before - // integration inserts. DO NOT call prepend multiple times — lol_html stacks - // prepend calls in reverse order, so a single prepend with the full string - // guarantees correct ordering. if let Some(ref slots_script) = ad_slots_script { snippet.push_str(slots_script); } - - // ... existing: for insert in integrations.head_inserts(&ctx) { snippet.push_str(...) } - + // existing integration head inserts... if !snippet.is_empty() { el.prepend(&snippet, ContentType::Html); } - // DO NOT register on_end_tag — flushes immediately per spec §4.3 + // DO NOT register on_end_tag — flushes immediately Ok(()) }) ``` -- [ ] **Step 5: Run tests** - - Run: `cargo test -p trusted-server-core html_processor` - Expected: all tests pass (including the new ones; no bids injection test must also pass) - -- [ ] **Step 6: Run full suite** - - Run: `cargo test --workspace` - Expected: clean - -- [ ] **Step 7: Commit** - - ```bash - git add crates/trusted-server-core/src/html_processor.rs \ - crates/trusted-server-core/src/integrations/registry.rs - git commit -m "Add ad_slots_script injection to HtmlProcessorConfig at head-open; no hold" - ``` - ---- - -## Task 8: `bid_cache.rs` — In-process auction result cache - -**Files:** - -- Create: `crates/trusted-server-core/src/bid_cache.rs` -- Modify: `crates/trusted-server-core/src/lib.rs` - -The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL. It is shared across concurrent Fastly request handlers via `std::sync::Mutex`. The `/ts-bids` endpoint (Task 10) uses `wait_for()` to block-poll until results arrive or the deadline fires. - -> **WASM note:** `std::time::Instant` and `std::thread::sleep` are both supported in Viceroy and Fastly Compute. The Mutex is uncontested in practice — requests are handled cooperatively with brief lock windows. - -- [ ] **Step 1: Write failing tests** +- [ ] **Step 5: Inject `__ts_bids` before `` via `el.on_end_tag()`** - Create `crates/trusted-server-core/src/bid_cache.rs` with only the tests: + Add a new handler in `create_html_processor`. The shared state is already populated by the time lol_html reaches `` (Task 9 awaits the auction before starting HTML processing): ```rust - #[cfg(test)] - mod tests { - use super::*; - use std::time::{Duration, Instant}; - - fn make_bids() -> BidMap { - let mut m = std::collections::HashMap::new(); - m.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); - m - } - - #[test] - fn returns_not_found_for_unknown_rid() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let result = cache.try_get("unknown-rid"); - assert!(matches!(result, CacheResult::NotFound), "should return NotFound"); - } - - #[test] - fn returns_pending_before_put() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-1", deadline); - let result = cache.try_get("rid-1"); - assert!(matches!(result, CacheResult::Pending), "should be Pending"); - } - - #[test] - fn returns_bids_after_put() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-2", deadline); - cache.put("rid-2", make_bids()); - match cache.try_get("rid-2") { - CacheResult::Complete(bids) => { - assert!(bids.contains_key("atf"), "should contain atf bid"); + let ad_bids_state = config.ad_bids_state.clone(); + + element!("body", |el| { + let state = ad_bids_state.clone(); + el.on_end_tag(move |end_tag| { + let script = state.read().expect("should read bid state"); + let bids_script = match &*script { + Some(s) => s.clone(), + None => { + r#""#.to_string() } - other => panic!("expected Complete, got {:?}", other), - } - } - - #[test] - fn returns_not_found_for_expired_entry() { - let cache = BidCache::new(Duration::from_millis(1), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-3", deadline); - cache.put("rid-3", make_bids()); - std::thread::sleep(Duration::from_millis(5)); - let result = cache.try_get("rid-3"); - assert!(matches!(result, CacheResult::NotFound), "should expire after TTL"); - } - - #[test] - fn wait_for_returns_bids_immediately_when_complete() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending("rid-4", deadline); - cache.put("rid-4", make_bids()); - let result = cache.wait_for("rid-4", deadline); - assert!(matches!(result, WaitResult::Bids(_)), "should return bids immediately"); - } - - #[test] - fn wait_for_returns_not_found_for_unknown_rid() { - let cache = BidCache::new(Duration::from_secs(30), 100); - let deadline = Instant::now() + Duration::from_millis(50); - let result = cache.wait_for("never-registered", deadline); - assert!(matches!(result, WaitResult::NotFound), "should return NotFound"); - } - } - ``` - - Run: `cargo test -p trusted-server-core bid_cache` - Expected: compile error (module not exported yet) - -- [ ] **Step 2: Implement bid_cache.rs** - - ```rust - //! In-process auction result cache keyed by request ID. - //! - //! Shared across concurrent Fastly request handlers via a global `Mutex`. - //! Entries expire after a configurable TTL (30 seconds by default). - - use std::collections::HashMap; - use std::sync::Mutex; - use std::time::{Duration, Instant}; - - pub type BidMap = HashMap; - - #[derive(Debug)] - enum EntryState { - Pending { auction_deadline: Instant }, - Complete { bids: BidMap }, - } - - struct CacheEntry { - state: EntryState, - inserted_at: Instant, - } - - struct BidCacheInner { - entries: HashMap, - insertion_order: std::collections::VecDeque, - capacity: usize, - ttl: Duration, - } - - impl BidCacheInner { - fn evict_expired(&mut self) { - let now = Instant::now(); - self.insertion_order.retain(|rid| { - self.entries.get(rid) - .map(|e| now.duration_since(e.inserted_at) < self.ttl) - .unwrap_or(false) - }); - self.entries.retain(|_, e| now.duration_since(e.inserted_at) < self.ttl); - } - - fn evict_oldest_if_full(&mut self) { - while self.entries.len() >= self.capacity { - if let Some(oldest) = self.insertion_order.pop_front() { - self.entries.remove(&oldest); - } else { - break; - } - } - } - } - - /// Outcome of a non-blocking cache lookup. - #[derive(Debug)] - pub enum CacheResult { - /// Auction complete; bids are ready. - Complete(BidMap), - /// Auction registered but not yet complete. - Pending, - /// Request ID never registered, or TTL expired. - NotFound, - } - - /// Outcome of a blocking `wait_for` call. - #[derive(Debug)] - pub enum WaitResult { - /// Auction completed within the deadline. - Bids(BidMap), - /// Deadline passed; bids not available. - Empty, - /// Request ID never registered (caller should return 404). - NotFound, - } - - /// In-process cache for auction results, shared across request handlers. - pub struct BidCache { - inner: Mutex, - } - - impl BidCache { - /// Create a new `BidCache`. - /// - /// # Arguments - /// - `ttl`: how long to keep entries before expiry - /// - `capacity`: max number of concurrent entries (oldest evicted when full) - pub fn new(ttl: Duration, capacity: usize) -> Self { - Self { - inner: Mutex::new(BidCacheInner { - entries: HashMap::new(), - insertion_order: std::collections::VecDeque::new(), - capacity, - ttl, - }), - } - } - - /// Register a request as in-flight. Call at auction start, before `run_auction`. - pub fn mark_pending(&self, request_id: &str, auction_deadline: Instant) { - let mut inner = self.inner.lock().expect("should lock bid_cache"); - inner.evict_expired(); - inner.evict_oldest_if_full(); - inner.entries.insert(request_id.to_string(), CacheEntry { - state: EntryState::Pending { auction_deadline }, - inserted_at: Instant::now(), - }); - inner.insertion_order.push_back(request_id.to_string()); - } - - /// Store completed auction results. Transitions entry from Pending → Complete. - pub fn put(&self, request_id: &str, bids: BidMap) { - let mut inner = self.inner.lock().expect("should lock bid_cache"); - if let Some(entry) = inner.entries.get_mut(request_id) { - entry.state = EntryState::Complete { bids }; - } - } - - /// Non-blocking lookup. Returns current state without sleeping. - pub fn try_get(&self, request_id: &str) -> CacheResult { - let inner = self.inner.lock().expect("should lock bid_cache"); - let now = Instant::now(); - match inner.entries.get(request_id) { - None => CacheResult::NotFound, - Some(entry) if now.duration_since(entry.inserted_at) >= inner.ttl => { - CacheResult::NotFound - } - Some(entry) => match &entry.state { - EntryState::Pending { .. } => CacheResult::Pending, - EntryState::Complete { bids } => CacheResult::Complete(bids.clone()), - }, - } - } - - /// Return the stored auction deadline for a pending entry (the `T₀ + auction_timeout_ms` - /// value minted when the page request arrived). Used by `/ts-bids` to enforce the correct - /// deadline rather than minting a fresh `Instant::now() + timeout`. - /// - /// Returns `None` if the entry is unknown, expired, or already complete. - pub fn get_auction_deadline(&self, request_id: &str) -> Option { - let inner = self.inner.lock().expect("should lock bid_cache"); - let now = Instant::now(); - inner.entries.get(request_id).and_then(|entry| { - if now.duration_since(entry.inserted_at) >= inner.ttl { - return None; - } - match entry.state { - EntryState::Pending { auction_deadline } => Some(auction_deadline), - EntryState::Complete { .. } => None, - } - }) - } - - /// Block until bids are available for `request_id` or `deadline` passes. - /// - /// Polls every 50ms. Returns `NotFound` immediately if `request_id` was never registered. - /// Returns `Empty` if deadline fires before auction completes. - pub fn wait_for(&self, request_id: &str, deadline: Instant) -> WaitResult { - loop { - match self.try_get(request_id) { - CacheResult::Complete(bids) => return WaitResult::Bids(bids), - CacheResult::NotFound => return WaitResult::NotFound, - CacheResult::Pending => { - if Instant::now() >= deadline { - return WaitResult::Empty; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - } - } - } + }; + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + })?; + Ok(()) + }) ``` -- [ ] **Step 3: Export from lib.rs** +- [ ] **Step 6: Run tests** - ```rust - pub mod bid_cache; - ``` + Run: `cargo test -p trusted-server-core html_processor` + Expected: all tests pass -- [ ] **Step 4: Run tests** +- [ ] **Step 7: Run full suite** - Run: `cargo test -p trusted-server-core bid_cache` - Expected: all tests pass + Run: `cargo test --workspace` + Expected: clean -- [ ] **Step 5: Commit** +- [ ] **Step 8: Commit** ```bash - git add crates/trusted-server-core/src/bid_cache.rs \ - crates/trusted-server-core/src/lib.rs - git commit -m "Add BidCache with 30s TTL, pending/complete states, and blocking wait_for" + git add crates/trusted-server-core/src/html_processor.rs \ + crates/trusted-server-core/src/integrations/registry.rs + git commit -m "Inject __ts_ad_slots at head-open and __ts_bids before via shared auction state" ``` --- -## Task 9: `handle_publisher_request` async restructuring +## Task 8: `handle_publisher_request` async restructuring **Files:** - Modify: `crates/trusted-server-core/src/publisher.rs` - Modify: `crates/trusted-server-adapter-fastly/src/main.rs` -> **Key constraint from spec §4.3:** Page rendering is never held for the auction. The auction and origin fetch run concurrently via Fastly's `send_async()` model — origin is dispatched first (non-blocking), then the auction runs its own `send_async` calls, so both overlap on the network. Bid results go to `bid_cache` only — they are NOT injected into the HTML. `Cache-Control: private, no-store` is set whenever slots matched (not just when bids arrived). +> **Key constraint from spec §4.3 and §3:** No `bid_cache`. No `/ts-bids`. No `request_id`. Bids travel inline with the HTML response via body injection. The `Arc>>` is the coordination mechanism within a single request's lifetime — it is written before HTML processing and read by the lol_html `` handler. + +> **Eligibility gating (spec §4.3):** Auctions fire only for real GET requests from non-bot, non-prefetch clients with TCF Purpose 1 consent and at least one matching slot. All other requests proceed with no auction and no `__ts_bids` injection. + +> **Cache-Control (spec §4.7):** Set `Cache-Control: private, max-age=0` (not `no-store`) to preserve BFCache eligibility. Strip `Surrogate-Control` and `Fastly-Surrogate-Control`. - [ ] **Step 1: Update function signature** Change `handle_publisher_request` in `publisher.rs`: + > **Existing context:** The existing `publisher.rs` function body already computes `consent_context`, `ec_id`, `request_info`, `origin_host`, and `backend_name` before the origin fetch. Steps below insert new logic between those existing computations and the origin fetch — they do not replace them. + ```rust pub async fn handle_publisher_request( settings: &Settings, @@ -1275,31 +1043,48 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL services: &RuntimeServices, orchestrator: &crate::auction::orchestrator::AuctionOrchestrator, slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, - bid_cache: &crate::bid_cache::BidCache, mut req: Request, ) -> Result> ``` - Add imports: + Add imports at top of file: ```rust + use std::sync::{Arc, RwLock}; + use fastly::http::header; use crate::auction::orchestrator::AuctionOrchestrator; use crate::auction::types::{AuctionContext, AuctionRequest, PublisherInfo, UserInfo, SiteInfo}; - use crate::bid_cache::{BidCache, BidMap}; use crate::creative_opportunities::{CreativeOpportunitiesFile, match_slots}; use crate::price_bucket::price_bucket; ``` -- [ ] **Step 2: Mint `request_id`, match URL, check consent** + > **`send_async` return type:** `req.send_async()` returns `fastly::handle::PendingRequestHandle` (re-exported as `fastly::PendingRequest` in recent versions). Confirm the exact type from the `fastly` crate version in `Cargo.toml`; `.wait()` is the blocking resolve method on whichever type is returned. - At the top of the function body, before the origin fetch: +- [ ] **Step 2: Apply auction-eligibility gates** - ```rust - // Mint per-request UUID — included in head injection and /ts-bids lookup key. - let request_id = uuid::Uuid::new_v4().to_string(); + At the top of the function body, before origin fetch: + ```rust let request_path = req.get_path().to_string(); - let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() { + let request_method = req.get_method().clone(); + + // Gate 1: Only GET triggers auctions. HEAD skips everything. + let is_get = request_method == fastly::http::Method::GET; + + // Gate 2: Skip prefetch hints (Sec-Purpose: prefetch or Purpose: prefetch). + let is_prefetch = req.get_header_str("sec-purpose") + .map_or(false, |v| v.contains("prefetch")) + || req.get_header_str("purpose") + .map_or(false, |v| v.contains("prefetch")); + + // Gate 3: Skip well-known crawler UAs (protects SSP QPS budget). + let user_agent = req.get_header_str("user-agent").unwrap_or(""); + let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] + .iter() + .any(|bot| user_agent.contains(bot)); + + // Gate 4: Slot match. + let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { match_slots(&slots_file.slots, &request_path) .into_iter() .cloned() @@ -1308,11 +1093,17 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL Vec::new() }; + // Gate 5: TCF Purpose 1 consent. let consent_allows_auction = consent_context .tcf .as_ref() .map_or(false, |tcf| tcf.has_purpose_consent(1)); - let should_run_auction = !matched_slots.is_empty() && consent_allows_auction; + + let should_run_auction = is_get + && !is_prefetch + && !is_bot + && !matched_slots.is_empty() + && consent_allows_auction; let auction_timeout_ms = settings .creative_opportunities @@ -1321,33 +1112,24 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL .unwrap_or(settings.auction.timeout_ms); ``` -- [ ] **Step 3: Register pending in bid_cache, fire origin + auction concurrently** +- [ ] **Step 3: Create shared bid state, fire origin + auction concurrently** ```rust - // Mint T₀ auction deadline. Stored in bid_cache so /ts-bids uses the same deadline, - // not a freshly-minted one when the browser's fetch arrives. - let auction_deadline = std::time::Instant::now() - + std::time::Duration::from_millis(u64::from(auction_timeout_ms)); - - // Register request as in-flight so /ts-bids can long-poll for it. - if should_run_auction { - bid_cache.mark_pending(&request_id, auction_deadline); - } + // Shared state: auction task writes the ready-to-inject script; lol_html + // handler reads it. Both within the same request — no cross-request sharing. + let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); - // Fire origin request immediately — Fastly's send_async dispatches the HTTP request - // to the network without blocking. The origin fetch is in-flight from this point. - // The auction below also uses send_async internally, so both origin SSP requests - // overlap on the network. This is Fastly's concurrency model — no join! needed. + // Fire origin immediately — both origin and auction SSP calls overlap on the network. let pending_origin = req .send_async(&backend_name) .change_context(TrustedServerError::Proxy { message: "Failed to dispatch async origin request".to_string(), })?; - // Run auction (internal send_async calls overlap with origin fetch on the network). + // Run auction. Internal SSP calls use send_async and overlap with origin fetch. let auction_result = if should_run_auction { let co_config = settings.creative_opportunities.as_ref() .expect("should be present when should_run_auction is true"); @@ -1377,17 +1159,20 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL None }; - // Write auction results to bid_cache — /ts-bids will serve them. + // Write auction result to shared state before HTML processing begins. + // The lol_html handler reads this synchronously — it is always populated here. + // `build_bid_map` returns `serde_json::Map`. if should_run_auction { let co_config = settings.creative_opportunities.as_ref() .expect("should be present"); - // Bind empty map to a local to avoid &Default::default() referencing a temporary. - let empty_bids = std::collections::HashMap::new(); + let empty_bids: std::collections::HashMap = + std::collections::HashMap::new(); let winning_bids = auction_result.as_ref() .map(|r| &r.winning_bids) .unwrap_or(&empty_bids); let bid_map = build_bid_map(winning_bids, co_config.price_granularity); - bid_cache.put(&request_id, bid_map); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); } // Await origin response (may already be buffered since we started it before the auction). @@ -1403,10 +1188,9 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL After acquiring `response`: ```rust - // Build head injection script: __ts_ad_slots + __ts_request_id (never bids). let ad_slots_script = if let Some(co_config) = &settings.creative_opportunities { if !matched_slots.is_empty() { - Some(build_head_globals_script(&matched_slots, &request_id, co_config)) + Some(build_ad_slots_script(&matched_slots, co_config)) } else { None } @@ -1414,33 +1198,91 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL None }; - // When slots matched: prevent browser/CDN caching of the per-user assembled HTML. - // Spec §4.4: set regardless of whether bids arrived — the request_id is now in the page. + // Set cache headers when slots matched. private, max-age=0 (not no-store) preserves + // BFCache eligibility — browser back/forward cache restores the already-rendered ad + // without firing a new GAM call, which is the desired behavior. if ad_slots_script.is_some() { - response.set_header(header::CACHE_CONTROL, "private, no-store"); + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); } - // Spec §4.3/§4.7: Force chunked encoding on every origin response so that - // reaches the browser immediately as chunks arrive — regardless of whether origin - // sent a buffered response (WordPress, Drupal) or a streaming one (NextJS 16). - // Removing Content-Length is required; sending both headers is invalid HTTP/1.1. + // Force chunked encoding so reaches the browser immediately as chunks arrive. + // Sending both Content-Length and Transfer-Encoding is invalid HTTP/1.1. response.remove_header(header::CONTENT_LENGTH); response.set_header("transfer-encoding", "chunked"); ``` -- [ ] **Step 5: Add `pub(crate)` helper functions** +- [ ] **Step 5: Thread shared state into `OwnedProcessResponseParams`** + + Update `OwnedProcessResponseParams`: + + ```rust + pub struct OwnedProcessResponseParams { + // existing fields... + pub(crate) ad_slots_script: Option, + pub(crate) ad_bids_state: Arc>>, + } + ``` + + Pass both through to `create_html_stream_processor` and into `HtmlProcessorConfig`. + +- [ ] **Step 6: Add `pub(crate)` helper functions** + + > **`BidMap` type:** Use `serde_json::Map` directly — no separate module needed. + + Add helpers in this order (each function is used by the one below it, so define leaf functions first): ```rust + /// HTML-escape a JSON string for safe inline `"#) + } + /// Build the `"# - ) - } - - /// Build the `BidMap` stored in `bid_cache` and returned by `/ts-bids`. - /// - /// Keyed by slot ID. Values contain `hb_pb`, `hb_bidder`, `hb_adid`, `burl`. - pub(crate) fn build_bid_map( - winning_bids: &std::collections::HashMap, - price_granularity: crate::price_bucket::PriceGranularity, - ) -> crate::bid_cache::BidMap { - winning_bids - .iter() - .filter_map(|(slot_id, bid)| { - let cpm = bid.price?; - let entry: std::collections::HashMap = [ - ("hb_pb".to_string(), serde_json::Value::String(price_bucket(cpm, price_granularity))), - ("hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone())), - ("hb_adid".to_string(), serde_json::Value::String( - bid.ad_id.as_deref().unwrap_or("").to_string() - )), - ("burl".to_string(), bid.burl.as_deref() - .map(serde_json::Value::from) - .unwrap_or(serde_json::Value::Null)), - ].into_iter().collect(); - Some((slot_id.clone(), entry.into_iter() - .map(|(k, v)| (k, v)) - .collect::>() - .into())) - }) - .collect() - } - - /// HTML-escape a JSON string for safe inline `"#) } fn build_auction_request( @@ -1535,39 +1332,25 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL } ``` -- [ ] **Step 6: Thread `ad_slots_script` into `OwnedProcessResponseParams`** - - Update `OwnedProcessResponseParams`: - - ```rust - pub struct OwnedProcessResponseParams { - // existing fields... - pub(crate) ad_slots_script: Option, - } - ``` - - Pass `ad_slots_script` through to `create_html_stream_processor` and into `HtmlProcessorConfig`. + > **Type note:** All helper signatures use `serde_json::Map` directly. Do not create a `BidMap` type alias or `bid_types.rs` module. - [ ] **Step 7: Update `main.rs` call site** In `crates/trusted-server-adapter-fastly/src/main.rs`: ```rust - // At startup — load creative-opportunities.toml and initialize bid_cache. + // At startup (top of main() / request handler setup, before the request dispatch loop). + // include_str! embeds the file at compile time — no runtime file I/O. const CREATIVE_OPPORTUNITIES_TOML: &str = include_str!("../../../creative-opportunities.toml"); - let slots_file: creative_opportunities::CreativeOpportunitiesFile = + let slots_file: trusted_server_core::creative_opportunities::CreativeOpportunitiesFile = toml::from_str(CREATIVE_OPPORTUNITIES_TOML) .expect("should parse creative-opportunities.toml"); - - // BidCache: 30s TTL, capacity 1000 entries (each entry is a few KB). - let bid_cache = crate::bid_cache::BidCache::new( - std::time::Duration::from_secs(30), - 1000, - ); ``` + `slots_file` is a local in the startup/handler scope and passed by reference into `handle_publisher_request` on each request — no `Arc` needed since it's immutable and the handler borrows it. + Update the call to `handle_publisher_request`: ```rust @@ -1575,15 +1358,16 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL settings, integration_registry, &publisher_services, - orchestrator, // existing - &slots_file, // new - &bid_cache, // new + orchestrator, // existing + &slots_file, // new req, ).await { // existing match arms unchanged } ``` + There is **no `/ts-bids` route** to add. The body injection is complete within `handle_publisher_request`. + - [ ] **Step 8: Compile check** Run: `cargo check --workspace` @@ -1599,164 +1383,43 @@ The `BidCache` stores auction results keyed by `request_id` with a 30-second TTL ```bash git add crates/trusted-server-core/src/publisher.rs \ crates/trusted-server-adapter-fastly/src/main.rs - git commit -m "Convert handle_publisher_request to async; auction writes to bid_cache; inject head globals only" + git commit -m "Convert handle_publisher_request to async; body-inject __ts_bids; eligibility gates; max-age=0" ``` --- -## Task 10: `/ts-bids` endpoint - -**Files:** - -- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` - -The `/ts-bids` endpoint is the client's fetch target for bid results. It long-polls until the auction completes or the deadline fires, then returns JSON. Bid results were already stored in `bid_cache` by Task 9. - -- [ ] **Step 1: Write failing test (integration-style)** - - In `main.rs` test module (or a new `tests/ts_bids.rs`): - - ```rust - #[test] - fn ts_bids_response_structure() { - use crate::bid_cache::{BidCache, WaitResult}; - use std::time::{Duration, Instant}; - - let cache = BidCache::new(Duration::from_secs(30), 100); - let rid = "test-rid-abc"; - let deadline = Instant::now() + Duration::from_secs(5); - cache.mark_pending(rid, deadline); - let mut bids = std::collections::HashMap::new(); - bids.insert("atf".to_string(), serde_json::json!({ - "hb_pb": "1.00", "hb_bidder": "kargo", "hb_adid": "abc", "burl": null, - })); - cache.put(rid, bids); - - match cache.wait_for(rid, deadline) { - WaitResult::Bids(b) => { - assert!(b.contains_key("atf"), "should contain atf slot bids"); - } - other => panic!("expected Bids, got {:?}", other), - } - } - ``` - - Run: `cargo test -p trusted-server-adapter-fastly ts_bids` - Expected: compile error (no handler yet, or pass since it's testing bid_cache directly) - -- [ ] **Step 2: Add `/ts-bids` route handler in `main.rs`** - - In the request routing section, before the publisher fallback, add: - - ```rust - if req.get_path() == "/ts-bids" && req.get_method() == fastly::http::Method::GET { - return handle_ts_bids_request(req, &bid_cache, settings); - } - ``` - - Add the handler function: - - ```rust - fn handle_ts_bids_request( - req: fastly::Request, - bid_cache: &crate::bid_cache::BidCache, - settings: &Settings, - ) -> fastly::Response { - // Parse `rid` query param. - let rid = req.get_query_parameter("rid").map(String::from); - let rid = match rid { - Some(r) if !r.is_empty() => r, - _ => { - return fastly::Response::from_status(fastly::http::StatusCode::BAD_REQUEST) - .with_body_text_plain("missing rid parameter"); - } - }; - - // Use the stored T₀ auction deadline from bid_cache — not a freshly-minted - // Instant::now() + timeout, which would extend the window past the original A_deadline. - // Spec §4.4: "/ts-bids blocks until auction completion or A_deadline" where A_deadline - // = T₀ + auction_timeout_ms (minted at page request receipt, stored in bid_cache entry). - let deadline = bid_cache.get_auction_deadline(&rid) - .unwrap_or_else(|| { - // Fallback: rid is unknown or already complete. wait_for returns immediately. - std::time::Instant::now() - }); - - let result = bid_cache.wait_for(&rid, deadline); - - match result { - crate::bid_cache::WaitResult::Bids(bids) => { - let body = serde_json::to_string(&bids) - .unwrap_or_else(|_| "{}".to_string()); - fastly::Response::from_status(fastly::http::StatusCode::OK) - .with_header(fastly::http::header::CONTENT_TYPE, "application/json") - .with_header(fastly::http::header::CACHE_CONTROL, "private, no-store") - .with_body(body) - } - crate::bid_cache::WaitResult::Empty => { - fastly::Response::from_status(fastly::http::StatusCode::OK) - .with_header(fastly::http::header::CONTENT_TYPE, "application/json") - .with_header(fastly::http::header::CACHE_CONTROL, "private, no-store") - .with_body("{}") - } - crate::bid_cache::WaitResult::NotFound => { - fastly::Response::from_status(fastly::http::StatusCode::NOT_FOUND) - .with_header(fastly::http::header::CACHE_CONTROL, "private, no-store") - .with_body_text_plain("unknown request id") - } - } - } - ``` - -- [ ] **Step 3: Compile check** - - Run: `cargo check --workspace` - Expected: clean - -- [ ] **Step 4: Run tests** - - Run: `cargo test --workspace` - Expected: all pass - -- [ ] **Step 5: Commit** - - ```bash - git add crates/trusted-server-adapter-fastly/src/main.rs - git commit -m "Add /ts-bids endpoint with long-poll semantics; serves bid_cache results by request_id" - ``` - ---- - -## Task 11: GPT head injector — emit `__tsAdInit` with `/ts-bids` fetch +## Task 9: GPT head injector — emit `__tsAdInit` with synchronous bid read **Files:** - Modify: `crates/trusted-server-core/src/integrations/gpt.rs` -> **Critical:** The `__tsAdInit` function MUST fetch `/ts-bids?rid=` — it must NOT read from `window.__ts_bids` (which is never set). The `window.__ts_request_id` global (injected at head-open by Task 9) supplies the RID. +> **Critical:** `__tsAdInit` reads `window.__ts_bids` **synchronously** — no fetch, no Promise. `window.__ts_bids` is already on the page (injected before ``) when `__tsAdInit` runs (it executes post-DCL, after `` is received). Both `nurl` and `burl` fire client-side from `slotRenderEnded`; neither is fired server-side. - [ ] **Step 1: Write failing test** ```rust #[test] - fn head_inserts_includes_ts_ad_init_with_ts_bids_fetch() { + fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { let config = test_config(); let integration = GptIntegration::new(config); let ctx = make_test_context(); let inserts = integration.head_inserts(&ctx); let combined = inserts.join(""); assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); - assert!(combined.contains("/ts-bids"), "should fetch from /ts-bids endpoint"); - assert!(combined.contains("__ts_request_id"), "should use __ts_request_id for rid"); - assert!(combined.contains("bidsPromise"), "should use bidsPromise pattern"); + assert!(combined.contains("window.__ts_bids"), "should read window.__ts_bids synchronously"); + assert!(combined.contains("ts_initial"), "should set ts_initial sentinel"); assert!(combined.contains("slotRenderEnded"), "should register slotRenderEnded"); - assert!(combined.contains("sendBeacon"), "should fire burl via sendBeacon"); - assert!(!combined.contains("__ts_bids"), "must NOT read window.__ts_bids — bids come from /ts-bids fetch"); + assert!(combined.contains("sendBeacon"), "should fire nurl and burl via sendBeacon"); + assert!(combined.contains("nurl"), "should fire nurl on confirmed render"); + assert!(!combined.contains("/ts-bids"), "must NOT fetch /ts-bids — bids are inline on the page"); + assert!(!combined.contains("bidsPromise"), "must NOT use bidsPromise — bids are synchronous"); + assert!(!combined.contains("__ts_request_id"), "must NOT reference request_id — no longer used"); } ``` Run: `cargo test -p trusted-server-core integrations::gpt` - Expected: FAIL — `__tsAdInit` not defined / assertion on `/ts-bids` string fails if old version present + Expected: FAIL - [ ] **Step 2: Replace `head_inserts()` in gpt.rs** @@ -1771,42 +1434,39 @@ The `/ts-bids` endpoint is the client's fetch target for bid results. It long-po "" .to_string(), - // __tsAdInit: fetches /ts-bids for bid targeting, then drives GPT. - // window.__ts_ad_slots and window.__ts_request_id are injected at head-open by TS. - // bidsPromise resolves concurrently with page rendering — never blocks FCP. + // __tsAdInit: reads window.__ts_bids synchronously (injected before ). + // No fetch, no Promise. Executes post-DCL when has already arrived. + // Both nurl and burl fire client-side from slotRenderEnded — never server-side. + // Note: window.__tsjs_installGptShim above is an EXISTING function in the + // tsjs-core bundle that stubs googletag.cmd before the real GPT loads. concat!( "" @@ -1825,18 +1485,18 @@ The `/ts-bids` endpoint is the client's fetch target for bid results. It long-po ```bash git add crates/trusted-server-core/src/integrations/gpt.rs - git commit -m "Emit __tsAdInit with /ts-bids fetch pattern from GPT head injector" + git commit -m "Emit __tsAdInit with synchronous window.__ts_bids read; nurl+burl from slotRenderEnded" ``` --- -## Task 12: `gpt/index.ts` — TypeScript `__tsAdInit` with `/ts-bids` fetch +## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` with slim-Prebid lazy loader **Files:** - Modify: `crates/js/lib/src/integrations/gpt/index.ts` -The TypeScript version mirrors the Rust inline string from Task 11. It uses the `bidsPromise` pattern — fetching `/ts-bids` concurrently with GPT slot definition. +The TypeScript version mirrors the Rust inline string from Task 9 and adds the lazy slim-Prebid loader. Slim-Prebid loads post-`window.load` and handles two things: refresh auctions (via existing GPT refresh triggers) and userID module warm-up to enrich the EC graph for the next request. - [ ] **Step 1: Write failing tests** @@ -1848,16 +1508,16 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the describe('installTsAdInit', () => { beforeEach(() => { delete (window as any).__ts_ad_slots - delete (window as any).__ts_request_id + delete (window as any).__ts_bids delete (window as any).__tsAdInit }) - it('fetches /ts-bids with request_id and applies bid targeting before refresh', async () => { + it('reads window.__ts_bids synchronously and applies bid targeting before refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), - getTargeting: vi.fn().mockReturnValue([]), + getTargeting: vi.fn().mockReturnValue(['abc']), } const mockPubads = { enableSingleRequest: vi.fn(), @@ -1879,71 +1539,94 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the targeting: { pos: 'atf' }, }, ] - ;(window as any).__ts_request_id = 'test-rid-123' - - const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ - ok: true, - json: async () => ({ - atf: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'abc', - burl: 'https://ssp/bill', - }, - }), - } as Response) + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } + + const fetchSpy = vi.spyOn(global, 'fetch') const { installTsAdInit } = await import('./index') installTsAdInit() - await (window as any).__tsAdInit() + ;(window as any).__tsAdInit() - expect(fetchSpy).toHaveBeenCalledWith( - expect.stringContaining('/ts-bids?rid=test-rid-123'), - expect.objectContaining({ credentials: 'omit' }) - ) + expect(fetchSpy).not.toHaveBeenCalled() expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1') expect(mockPubads.refresh).toHaveBeenCalled() fetchSpy.mockRestore() }) - it('calls refresh with empty bids when fetch fails', async () => { + it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) + let capturedListener: ((e: any) => void) | undefined + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['abc']), + } const mockPubads = { enableSingleRequest: vi.fn(), - addEventListener: vi.fn(), refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn + }), } ;(window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue({ - addService: vi.fn().mockReturnThis(), - setTargeting: vi.fn().mockReturnThis(), - }), + defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), } - ;(window as any).__ts_ad_slots = [] - ;(window as any).__ts_request_id = 'rid-fail' - - vi.spyOn(global, 'fetch').mockRejectedValue(new Error('network error')) + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } const { installTsAdInit } = await import('./index') installTsAdInit() - await (window as any).__tsAdInit() + ;(window as any).__tsAdInit() - expect(mockPubads.refresh).toHaveBeenCalled() + expect(capturedListener).toBeDefined() + capturedListener!({ isEmpty: false, slot: mockSlot }) + + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win') + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') + beaconSpy.mockRestore() }) - it('fires burl via sendBeacon on slotRenderEnded when our bid won', async () => { + it('does not fire nurl/burl when bid did not win GAM line item', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) let capturedListener: ((e: any) => void) | undefined - const mockSlot = { + const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), - getTargeting: vi.fn().mockReturnValue(['abc']), + getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), } const mockPubads = { enableSingleRequest: vi.fn(), @@ -1954,7 +1637,7 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the } ;(window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue(mockSlot), + defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), } @@ -1967,43 +1650,58 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the targeting: {}, }, ] - ;(window as any).__ts_request_id = 'rid-burl-test' - - vi.spyOn(global, 'fetch').mockResolvedValue({ - ok: true, - json: async () => ({ - atf: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'abc', - burl: 'https://ssp/bill', - }, - }), - } as Response) + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } const { installTsAdInit } = await import('./index') installTsAdInit() - await (window as any).__tsAdInit() + ;(window as any).__tsAdInit() + capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }) - // Trigger slotRenderEnded — slot has our winning hb_adid - expect(capturedListener).toBeDefined() - capturedListener!({ - isEmpty: false, - slot: mockSlot, - }) - - expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') + expect(beaconSpy).not.toHaveBeenCalled() beaconSpy.mockRestore() }) + + it('calls refresh even when __ts_bids is empty (graceful fallback)', () => { + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [] + ;(window as any).__ts_bids = {} + + const { installTsAdInit } = require('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect(mockPubads.refresh).toHaveBeenCalled() + }) }) ``` Run: `cd crates/js/lib && npx vitest run` - Expected: FAIL — `installTsAdInit` not exported or fetches wrong endpoint + Expected: FAIL — `installTsAdInit` not defined or assertions fail -- [ ] **Step 2: Add `installTsAdInit` to `index.ts`** +- [ ] **Step 2: Implement `installTsAdInit` in `index.ts`** - Add to `crates/js/lib/src/integrations/gpt/index.ts`: + Replace the old `/ts-bids` fetch implementation with: ```typescript interface TsAdSlot { @@ -2018,38 +1716,30 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the hb_pb?: string hb_bidder?: string hb_adid?: string + nurl?: string burl?: string } type TsWindow = Window & { __ts_ad_slots?: TsAdSlot[] - __ts_request_id?: string + __ts_bids?: Record __tsAdInit?: () => void } /** * Install `window.__tsAdInit`. * - * Reads `window.__ts_ad_slots` and `window.__ts_request_id` (both injected by - * the edge at `` open). Fetches bid results from `/ts-bids?rid=` - * concurrently with GPT slot definition. Applies targeting and calls `refresh()` - * after the fetch resolves. Registers `slotRenderEnded` to fire `burl` via - * `sendBeacon` when our specific Prebid bid wins the GAM line item match. + * Reads `window.__ts_ad_slots` (injected at head-open) and `window.__ts_bids` + * (injected before ) synchronously — no fetch, no Promise. Applies bid + * targeting to GPT slots, sets the `ts_initial` sentinel, registers + * `slotRenderEnded` to fire both nurl and burl via sendBeacon when our + * specific Prebid bid wins the GAM line item match, then calls refresh(). */ export function installTsAdInit(): void { const w = window as TsWindow w.__tsAdInit = function () { const slots = w.__ts_ad_slots ?? [] - const rid = w.__ts_request_id - - const bidsPromise: Promise> = rid - ? fetch(`/ts-bids?rid=${encodeURIComponent(rid)}`, { - credentials: 'omit', - }) - .then((r) => (r.ok ? r.json() : {})) - .catch(() => ({})) - : Promise.resolve({}) - + const bids = w.__ts_bids ?? {} const g = (window as GptWindow).googletag if (!g) return @@ -2066,6 +1756,11 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v) ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + gptSlot.setTargeting('ts_initial', '1') return { id: slot.id, gptSlot } }) .filter(Boolean) as Array<{ @@ -2076,153 +1771,86 @@ The TypeScript version mirrors the Rust inline string from Task 11. It uses the g.pubads().enableSingleRequest() g.enableServices() - bidsPromise.then((bids) => { - gptSlots.forEach(({ id, gptSlot }) => { - const bid = bids[id] ?? {} - ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, bid[key]!) - }) - }) - - g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? '' - const bid = bids[slotId] ?? {} - if ( - !event.isEmpty && - bid.burl && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid - ) { - navigator.sendBeacon(bid.burl) - } - }) - - g.pubads().refresh() + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + const ourBidWon = + !event.isEmpty && + bid.hb_adid && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl) + if (bid.burl) navigator.sendBeacon(bid.burl) + } }) + + g.pubads().refresh() }) } } ``` - Call `installTsAdInit()` from the integration's initialization path. +- [ ] **Step 3: Add lazy slim-Prebid loader (post-`window.load`)** -- [ ] **Step 3: Run JS tests** + After `installTsAdInit`, add: - Run: `cd crates/js/lib && npx vitest run` - Expected: new tests pass - -- [ ] **Step 4: Build JS bundle** - - Run: `cd crates/js/lib && node build-all.mjs` - Expected: clean build - -- [ ] **Step 5: Commit** - - ```bash - git add crates/js/lib/src/integrations/gpt/ - git commit -m "Add installTsAdInit with /ts-bids fetch pattern and slotRenderEnded burl firing" - ``` - ---- - -## Task 13: `nurl` fire-and-forget - -**Files:** - -- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` -- Modify: `crates/trusted-server-core/src/publisher.rs` - -- [ ] **Step 1: Write failing test** - - ```rust - #[test] - fn prebid_config_fire_nurl_defaults_to_true() { - let config = PrebidConfig::default(); - assert!(config.fire_nurl_at_edge, "should fire nurl at edge by default"); + ```typescript + /** + * Register the slim-Prebid lazy loader. Fires after window.load — off the + * critical path. slim-Prebid handles refresh auctions and userID module + * warm-up (ID5, sharedID, LiveRamp ATS, Lockr). It skips initial-render slots + * (ts_initial=1) and registers as the GPT refresh handler for scroll/sticky auctions. + * + * Phase 1: no-op unless window.__tsjs_slim_prebid_url is set (it won't be until + * the slim-Prebid bundle build target ships in a later phase). + */ + export function installSlimPrebidLoader(): void { + const url = (window as any).__tsjs_slim_prebid_url as string | undefined + if (!url) return + window.addEventListener('load', () => { + const script = document.createElement('script') + script.src = url + script.defer = true + document.head.appendChild(script) + }) } ``` - Run: `cargo test -p trusted-server-core integrations::prebid` - Expected: FAIL - -- [ ] **Step 2: Add `fire_nurl_at_edge` to `PrebidConfig`** + Call `installTsAdInit()` from the integration's existing initialization path — wherever the module's init function runs at page load (look for the existing `init()` or module-level call that sets up the GPT integration). Add: - ```rust - #[serde(default = "default_fire_nurl_at_edge")] - pub fire_nurl_at_edge: bool, - ``` - - ```rust - fn default_fire_nurl_at_edge() -> bool { true } - ``` - -- [ ] **Step 3: Fire nurls in publisher.rs after bid_cache.put()** - - After the `bid_cache.put(...)` call (Task 9 Step 3), add: - - ```rust - if let Some(ref result) = auction_result { - fire_winning_nurls(result, settings); - } + ```typescript + // In the integration's init / module entry point: + installTsAdInit() ``` - Add helper: - - ```rust - fn fire_winning_nurls( - result: &crate::auction::orchestrator::OrchestrationResult, - settings: &Settings, - ) { - use crate::backend::BackendConfig; - - let fire_nurl = settings - .integrations - .get_typed::("prebid") - .map(|c| c.fire_nurl_at_edge) - .unwrap_or(true); + `window.__tsAdInit()` itself is called by `__tsAdInit` being invoked from the `"); @@ -2289,7 +1914,7 @@ Tests use `pub(crate)` helpers from Task 9 directly. } #[test] - fn bid_map_uses_price_bucket_and_ad_id() { + fn bid_map_includes_nurl_and_burl() { let mut winning_bids = HashMap::new(); winning_bids.insert("atf_sidebar_ad".to_string(), Bid { slot_id: "atf_sidebar_ad".to_string(), @@ -2298,46 +1923,60 @@ Tests use `pub(crate)` helpers from Task 9 directly. creative: None, adomain: None, bidder: "kargo".to_string(), - width: 300, height: 250, + width: 300, + height: 250, + nurl: Some("https://ssp/win".to_string()), + burl: Some("https://ssp/bill".to_string()), + ad_id: Some("abc123".to_string()), + metadata: Default::default(), + }); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); + assert_eq!(entry.get("hb_pb").and_then(|v| v.as_str()), Some("2.50")); + assert_eq!(entry.get("hb_bidder").and_then(|v| v.as_str()), Some("kargo")); + assert_eq!(entry.get("hb_adid").and_then(|v| v.as_str()), Some("abc123")); + assert_eq!(entry.get("nurl").and_then(|v| v.as_str()), Some("https://ssp/win")); + assert_eq!(entry.get("burl").and_then(|v| v.as_str()), Some("https://ssp/bill")); + } + + #[test] + fn bid_map_excludes_slot_when_price_is_none() { + let mut winning_bids = HashMap::new(); + winning_bids.insert("no-price-slot".to_string(), Bid { + slot_id: "no-price-slot".to_string(), + price: None, + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, nurl: None, - burl: Some("https://ssp.example/billing?id=abc123".to_string()), - ad_id: Some("prebid-uuid-abc123".to_string()), - metadata: HashMap::new(), + burl: None, + ad_id: None, + metadata: Default::default(), }); - let bid_map = build_bid_map(&winning_bids, PriceGranularity::Dense); - let slot_bids = bid_map.get("atf_sidebar_ad").expect("should have slot bids"); - assert_eq!( - slot_bids.get("hb_pb").and_then(|v| v.as_str()), - Some("2.53"), - "should bucket 2.53 as 2.53 (dense)" - ); - assert_eq!( - slot_bids.get("hb_bidder").and_then(|v| v.as_str()), - Some("kargo"), - "should include bidder" - ); - assert_eq!( - slot_bids.get("hb_adid").and_then(|v| v.as_str()), - Some("prebid-uuid-abc123"), - "should use ad_id not creative markup" - ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + assert!(map.is_empty(), "slot with no price should be excluded from bid map"); } #[test] - fn html_escape_neutralizes_xss_in_json() { - let malicious = r#"{"zone":""), "should escape "); - assert!(escaped.contains("\\u003c"), "should unicode-escape <"); - assert!(escaped.contains("\\u003e"), "should unicode-escape >"); + fn bids_script_is_xss_safe() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let script = build_bids_script(&map); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in bids script"); + assert!(!inner.contains('>'), "no unescaped > in bids script"); } #[test] - fn url_matching_end_to_end() { - let file = CreativeOpportunitiesFile { slots: vec![make_slot()] }; - assert_eq!(match_slots(&file.slots, "/2024/01/my-article").len(), 1, "should match article"); - assert_eq!(match_slots(&file.slots, "/about").len(), 0, "should not match /about"); - assert_eq!(match_slots(&file.slots, "/").len(), 0, "should not match root"); + fn html_escape_encodes_special_chars() { + assert_eq!(html_escape_for_script("`. + /// Injected at `` open. `None` when no slots matched. + pub ad_slots_script: Option, + /// Shared auction result — written by auction task before HTML processing begins. + /// Handler reads this in `el.on_end_tag()` on the body element. + /// `None` means no auction ran; inject empty `__ts_bids = {}` as fallback. + pub ad_bids_state: std::sync::Arc>>, } impl HtmlProcessorConfig { @@ -151,6 +158,8 @@ impl HtmlProcessorConfig { request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), integrations: integrations.clone(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), } } } @@ -230,6 +239,8 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let injected_tsjs = Rc::new(Cell::new(false)); let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); + let ad_slots_script = config.ad_slots_script.clone(); + let ad_bids_state = config.ad_bids_state.clone(); let mut element_content_handlers = vec![ // Inject unified tsjs bundle once at the start of @@ -238,9 +249,14 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integrations = integration_registry.clone(); let patterns = patterns.clone(); let document_state = document_state.clone(); + let ad_slots_script = ad_slots_script.clone(); move |el| { if !injected_tsjs.get() { let mut snippet = String::new(); + // Inject ad slots script first so it appears before tsjs bundle. + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } let ctx = IntegrationHtmlContext { request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, @@ -265,6 +281,30 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), + // Inject __ts_bids before via end_tag_handlers. + element!("body", { + let state = ad_bids_state.clone(); + move |el| { + let state = state.clone(); + if let Some(handlers) = el.end_tag_handlers() { + let handler: EndTagHandler<'static> = + Box::new(move |end_tag: &mut EndTag<'_>| { + let script_guard = state.read().expect("should read bid state"); + let bids_script = match &*script_guard { + Some(s) => s.clone(), + None => { + r#""# + .to_string() + } + }; + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + }); + handlers.push(handler); + } + Ok(()) + } + }), // Replace URLs in href attributes element!("[href]", { let patterns = patterns.clone(); @@ -540,6 +580,8 @@ mod tests { request_host: "test.example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), } } @@ -1185,4 +1227,85 @@ mod tests { "should contain post-processor mutation" ); } + + #[test] + fn injects_ad_slots_at_head_open() { + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: Some( + r#""#.to_string(), + ), + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk( + b"Tcontent", + true, + ) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("window.__ts_ad_slots"), + "should inject ad slots at head-open" + ); + assert!( + !html.contains("__ts_request_id"), + "must NOT inject request_id" + ); + } + + #[test] + fn injects_ts_bids_before_body_close() { + let bids_script = + r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("window.__ts_bids"), + "should inject bids before " + ); + let bids_pos = html + .find("window.__ts_bids") + .expect("bids should be in output"); + let body_close_pos = html.find("").expect(" should be in output"); + assert!(bids_pos < body_close_pos, "bids must appear before "); + } + + #[test] + fn injects_empty_ts_bids_when_state_is_none() { + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("__ts_bids=JSON.parse(\"{}\")"), + "should inject empty bids on None state" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 8b55493be..ffad78921 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -853,6 +853,14 @@ impl IntegrationRegistry { .collect() } + #[cfg(test)] + #[must_use] + pub fn empty_for_tests() -> Self { + Self { + inner: Arc::new(IntegrationRegistryInner::default()), + } + } + #[cfg(test)] #[must_use] pub fn from_rewriters( From 8b9500cf7c5d83d4d7bf97910ddb414651ec704d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 5 May 2026 19:47:16 +0530 Subject: [PATCH 016/195] Convert handle_publisher_request to async; body-inject __ts_bids; eligibility gates; max-age=0 - Make handle_publisher_request async; add orchestrator and slots_file params - Dispatch origin request with send_async before running auction in parallel - Gate auction on GET, no prefetch, no bot, matched slots, TCF purpose-1 consent - Run server-side auction and write bucketed bids to ad_bids_state Arc - Compute ad_slots_script after response headers; set Cache-Control: private, max-age=0 - Fix Stream arm to thread actual ad_slots_script and ad_bids_state through - Add build_auction_request, build_bid_map, build_bids_script, build_ad_slots_script helpers - Update route_tests.rs to pass empty slots_file to route_request --- .../src/route_tests.rs | 6 + crates/trusted-server-core/src/publisher.rs | 249 +++++++++++++++++- 2 files changed, 243 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 0fd0113f8..06336a9b1 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -184,6 +184,8 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { let orchestrator = build_orchestrator(&settings).expect("should build auction orchestrator"); let integration_registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); + let slots_file = + trusted_server_core::creative_opportunities::CreativeOpportunitiesFile::default(); let discovery_req = Request::get("https://test.com/.well-known/trusted-server.json"); let discovery_services = test_runtime_services(&discovery_req); @@ -192,6 +194,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &discovery_services, + &slots_file, discovery_req, )) .expect("should route discovery request"); @@ -208,6 +211,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &admin_services, + &slots_file, admin_req, )) .expect("should route admin request"); @@ -224,6 +228,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &auction_services, + &slots_file, auction_req, )) .expect("should return an error response for auction requests"); @@ -240,6 +245,7 @@ fn configured_missing_consent_store_only_breaks_consent_routes() { &orchestrator, &integration_registry, &publisher_services, + &slots_file, publisher_req, )) .expect("should return an error response for publisher fallback"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 5bcef6941..4037bdf8a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -12,11 +12,14 @@ //! content-rewriting concern. use std::io::Write; +use std::sync::{Arc, RwLock}; use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; +use crate::auction::orchestrator::AuctionOrchestrator; +use crate::auction::types::{AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo}; use crate::backend::BackendConfig; use crate::consent::{allows_ec_creation, build_consent_context, ConsentPipelineInput}; use crate::constants::{COOKIE_TS_EC, HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; @@ -26,6 +29,7 @@ use crate::error::TrustedServerError; use crate::http_util::{serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; use crate::platform::RuntimeServices; +use crate::price_bucket::price_bucket; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; @@ -182,6 +186,8 @@ struct ProcessResponseParams<'a> { settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + ad_slots_script: Option<&'a str>, + ad_bids_state: &'a Arc>>, } /// Process response body through the streaming pipeline. @@ -224,6 +230,8 @@ fn process_response_streaming( params.request_scheme, params.settings, params.integration_registry, + params.ad_slots_script.map(str::to_string), + params.ad_bids_state.clone(), )?; StreamingPipeline::new(config, processor).process(body, output)?; } else if is_rsc_flight { @@ -252,18 +260,21 @@ fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, - settings: &Settings, + _settings: &Settings, integration_registry: &IntegrationRegistry, + ad_slots_script: Option, + ad_bids_state: Arc>>, ) -> Result> { use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; - let config = HtmlProcessorConfig::from_settings( - settings, - integration_registry, - origin_host, - request_host, - request_scheme, - ); + let config = HtmlProcessorConfig { + origin_host: origin_host.to_string(), + request_host: request_host.to_string(), + request_scheme: request_scheme.to_string(), + integrations: integration_registry.clone(), + ad_slots_script, + ad_bids_state, + }; Ok(create_html_processor(config)) } @@ -392,6 +403,8 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + pub(crate) ad_slots_script: Option, + pub(crate) ad_bids_state: Arc>>, } /// Stream the publisher response body through the processing pipeline. @@ -420,6 +433,8 @@ pub fn stream_publisher_body( settings, content_type: ¶ms.content_type, integration_registry, + ad_slots_script: params.ad_slots_script.as_deref(), + ad_bids_state: ¶ms.ad_bids_state, }; process_response_streaming(body, output, &borrowed) } @@ -441,10 +456,12 @@ pub fn stream_publisher_body( /// /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. -pub fn handle_publisher_request( +pub async fn handle_publisher_request( settings: &Settings, integration_registry: &IntegrationRegistry, services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, mut req: Request, ) -> Result> { log::debug!("Proxying request to publisher_origin"); @@ -520,14 +537,105 @@ pub fn handle_publisher_request( backend_name, settings.publisher.origin_url ); + + let request_path = req.get_path().to_string(); + let is_get = req.get_method() == fastly::http::Method::GET; + + let is_prefetch = req.get_header_str("sec-purpose") + .map_or(false, |v| v.contains("prefetch")) + || req.get_header_str("purpose") + .map_or(false, |v| v.contains("prefetch")); + + let user_agent = req.get_header_str("user-agent").unwrap_or(""); + let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] + .iter() + .any(|bot| user_agent.contains(bot)); + + let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { + crate::creative_opportunities::match_slots(&slots_file.slots, &request_path) + .into_iter() + .cloned() + .collect() + } else { + Vec::new() + }; + + let consent_allows_auction = consent_context + .tcf + .as_ref() + .map_or(false, |tcf| tcf.has_purpose_consent(1)); + + let should_run_auction = is_get + && !is_prefetch + && !is_bot + && !matched_slots.is_empty() + && consent_allows_auction; + + let auction_timeout_ms = settings + .creative_opportunities + .as_ref() + .and_then(|co| co.auction_timeout_ms) + .unwrap_or(settings.auction.timeout_ms); + + let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); + // Only advertise encodings the rewrite pipeline can decode and re-encode. restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); - let mut response = req - .send(&backend_name) + let pending_origin = req + .send_async(&backend_name) .change_context(TrustedServerError::Proxy { - message: "Failed to proxy request to origin".to_string(), + message: "Failed to dispatch async origin request".to_string(), + })?; + + let auction_result = if should_run_auction { + let co_config = settings.creative_opportunities.as_ref() + .expect("should be present when should_run_auction is true"); + let auction_request = build_auction_request( + &matched_slots, + &ec_id, + &consent_context, + &request_info, + co_config, + ); + let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); + let auction_context = AuctionContext { + settings, + request: &placeholder_req, + client_info: services.client_info(), + timeout_ms: auction_timeout_ms, + provider_responses: None, + services, + }; + match orchestrator.run_auction(&auction_request, &auction_context, services).await { + Ok(result) => Some(result), + Err(e) => { + log::warn!("server-side auction failed, proceeding without bids: {e:?}"); + None + } + } + } else { + None + }; + + if should_run_auction { + let co_config = settings.creative_opportunities.as_ref() + .expect("should be present"); + let empty: std::collections::HashMap = + std::collections::HashMap::new(); + let winning_bids = auction_result.as_ref() + .map(|r| &r.winning_bids) + .unwrap_or(&empty); + let bid_map = build_bid_map(winning_bids, co_config.price_granularity); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); + } + + let mut response = pending_origin + .wait() + .change_context(TrustedServerError::Proxy { + message: "Failed to await origin response".to_string(), })?; log::debug!("Response headers:"); @@ -535,6 +643,22 @@ pub fn handle_publisher_request( log::debug!(" {}: {:?}", name, value); } + let ad_slots_script = if let Some(co_config) = &settings.creative_opportunities { + if !matched_slots.is_empty() { + Some(build_ad_slots_script(&matched_slots, co_config)) + } else { + None + } + } else { + None + }; + + if ad_slots_script.is_some() { + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); + } + // Set EC ID / cookie headers BEFORE body processing. // These are body-independent (computed from request cookies + consent). apply_ec_headers( @@ -623,6 +747,8 @@ pub fn handle_publisher_request( request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), content_type, + ad_slots_script: ad_slots_script.clone(), + ad_bids_state: ad_bids_state.clone(), }, }) } @@ -642,6 +768,8 @@ pub fn handle_publisher_request( settings, content_type: &content_type, integration_registry, + ad_slots_script: ad_slots_script.as_deref(), + ad_bids_state: &ad_bids_state, }; let mut output = Vec::new(); process_response_streaming(body, &mut output, ¶ms)?; @@ -654,6 +782,93 @@ pub fn handle_publisher_request( } } +/// Build an [`AuctionRequest`] from matched creative opportunity slots. +pub(crate) fn build_auction_request( + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + ec_id: &str, + consent_context: &crate::consent::ConsentContext, + request_info: &crate::http_util::RequestInfo, + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, +) -> AuctionRequest { + let slots = matched_slots + .iter() + .map(|s| s.to_ad_slot(&co_config.gam_network_id)) + .collect(); + AuctionRequest { + id: format!("ts-{}", ec_id), + slots, + publisher: PublisherInfo { + domain: request_info.host.clone(), + page_url: None, + }, + user: UserInfo { + id: ec_id.to_string(), + fresh_id: ec_id.to_string(), + consent: Some(consent_context.clone()), + }, + device: None, + site: Some(SiteInfo { + domain: request_info.host.clone(), + page: String::new(), + }), + context: std::collections::HashMap::new(), + } +} + +/// Build a price-bucketed bid map from winning bids. +/// +/// Returns a map of slot ID → bucketed CPM string. +pub(crate) fn build_bid_map( + winning_bids: &std::collections::HashMap, + granularity: crate::price_bucket::PriceGranularity, +) -> std::collections::HashMap { + winning_bids + .iter() + .filter_map(|(slot_id, bid)| { + bid.price.map(|cpm| { + let bucket = price_bucket(cpm, granularity); + (slot_id.clone(), bucket) + }) + }) + .collect() +} + +/// Build the `__ts_bids` inline script content from a bucketed bid map. +pub(crate) fn build_bids_script(bid_map: &std::collections::HashMap) -> String { + let entries: Vec = bid_map + .iter() + .map(|(slot_id, bucket)| format!("\"{}\":\"{}\"", slot_id, bucket)) + .collect(); + format!("window.__ts_bids={{{}}};", entries.join(",")) +} + +/// Build the `__ts_ad_slots` inline script content from matched slots. +pub(crate) fn build_ad_slots_script( + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, +) -> String { + let entries: Vec = matched_slots + .iter() + .map(|slot| { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| format!("[{},{}]", f.width, f.height)) + .collect(); + format!( + "{{\"id\":\"{}\",\"div\":\"{}\",\"path\":\"{}\",\"sizes\":[{}]}}", + slot.id, + div_id, + gam_path, + formats.join(",") + ) + }) + .collect(); + format!("window.__ts_ad_slots=[{}];", entries.join(",")) +} + /// Whether the content type requires processing (URL rewriting, HTML injection). /// /// Text-based and JavaScript/JSON responses are processable; binary types @@ -1366,6 +1581,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); @@ -1407,6 +1624,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); @@ -1439,6 +1658,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let bogus_body = Body::from(b"not gzip".to_vec()); @@ -1538,6 +1759,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -1588,6 +1811,8 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(RwLock::new(None)), }; let mut output = Vec::new(); From 9cdbb36fd7bb62212258a433c2764f07eb8f7e54 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 5 May 2026 20:05:59 +0530 Subject: [PATCH 017/195] Emit __tsAdInit with synchronous window.__ts_bids read; nurl+burl from slotRenderEnded --- .../src/integrations/gpt.rs | 89 +++++++++++++++++-- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 40bcf7f2c..796d633e1 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -438,13 +438,42 @@ impl IntegrationHeadInjector for GptIntegration { } fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - // Set the enable flag and best-effort call the activation function - // registered by the GPT shim module. The bundle also auto-installs - // when it sees the pre-set flag, so this works regardless of whether - // the inline bootstrap runs before or after the TSJS bundle. vec![ - "" + "" .to_string(), + concat!( + "" + ).to_string(), ] } } @@ -1020,7 +1049,7 @@ mod tests { let inserts = integration.head_inserts(&ctx); - assert_eq!(inserts.len(), 1, "should emit exactly one head insert"); + assert_eq!(inserts.len(), 2, "should emit exactly two head inserts"); assert_eq!( inserts[0], "", @@ -1028,6 +1057,54 @@ mod tests { ); } + #[test] + fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let inserts = integration.head_inserts(&ctx); + let combined = inserts.join(""); + assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); + assert!( + combined.contains("window.__ts_bids"), + "should read window.__ts_bids synchronously" + ); + assert!( + combined.contains("ts_initial"), + "should set ts_initial sentinel" + ); + assert!( + combined.contains("slotRenderEnded"), + "should register slotRenderEnded" + ); + assert!( + combined.contains("sendBeacon"), + "should fire nurl and burl via sendBeacon" + ); + assert!( + combined.contains("nurl"), + "should fire nurl on confirmed render" + ); + assert!( + !combined.contains("/ts-bids"), + "must NOT fetch /ts-bids — bids are inline on the page" + ); + assert!( + !combined.contains("bidsPromise"), + "must NOT use bidsPromise — bids are synchronous" + ); + assert!( + !combined.contains("__ts_request_id"), + "must NOT reference request_id — no longer used" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); From 6b624e3c9262cdc06b163eec4b7e18d024acdb3e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 09:32:32 +0530 Subject: [PATCH 018/195] Fix bid map shape and ad slots property names; resolve clippy errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build_bid_map now returns serde_json::Map with full bid objects (hb_pb, hb_bidder, hb_adid, nurl, burl) instead of a plain CPM string map - build_bids_script / build_ad_slots_script now emit full "# - .to_string() - } + None => r#""# + .to_string(), }; end_tag.before(&bids_script, ContentType::Html); Ok(()) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4037bdf8a..c614e5ed4 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -19,7 +19,9 @@ use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; use crate::auction::orchestrator::AuctionOrchestrator; -use crate::auction::types::{AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo}; +use crate::auction::types::{ + AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo, +}; use crate::backend::BackendConfig; use crate::consent::{allows_ec_creation, build_consent_context, ConsentPipelineInput}; use crate::constants::{COOKIE_TS_EC, HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; @@ -456,6 +458,12 @@ pub fn stream_publisher_body( /// /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. +/// +/// # Panics +/// +/// Panics if `should_run_auction` is `true` but `settings.creative_opportunities` is `None`. +/// This is a logic invariant: `should_run_auction` is only set when creative opportunities +/// are configured, so this state is unreachable in practice. pub async fn handle_publisher_request( settings: &Settings, integration_registry: &IntegrationRegistry, @@ -541,10 +549,12 @@ pub async fn handle_publisher_request( let request_path = req.get_path().to_string(); let is_get = req.get_method() == fastly::http::Method::GET; - let is_prefetch = req.get_header_str("sec-purpose") - .map_or(false, |v| v.contains("prefetch")) - || req.get_header_str("purpose") - .map_or(false, |v| v.contains("prefetch")); + let is_prefetch = req + .get_header_str("sec-purpose") + .is_some_and(|v| v.contains("prefetch")) + || req + .get_header_str("purpose") + .is_some_and(|v| v.contains("prefetch")); let user_agent = req.get_header_str("user-agent").unwrap_or(""); let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] @@ -563,13 +573,10 @@ pub async fn handle_publisher_request( let consent_allows_auction = consent_context .tcf .as_ref() - .map_or(false, |tcf| tcf.has_purpose_consent(1)); + .is_some_and(|tcf| tcf.has_purpose_consent(1)); - let should_run_auction = is_get - && !is_prefetch - && !is_bot - && !matched_slots.is_empty() - && consent_allows_auction; + let should_run_auction = + is_get && !is_prefetch && !is_bot && !matched_slots.is_empty() && consent_allows_auction; let auction_timeout_ms = settings .creative_opportunities @@ -583,14 +590,16 @@ pub async fn handle_publisher_request( restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); - let pending_origin = req - .send_async(&backend_name) - .change_context(TrustedServerError::Proxy { - message: "Failed to dispatch async origin request".to_string(), - })?; + let pending_origin = + req.send_async(&backend_name) + .change_context(TrustedServerError::Proxy { + message: "Failed to dispatch async origin request".to_string(), + })?; let auction_result = if should_run_auction { - let co_config = settings.creative_opportunities.as_ref() + let co_config = settings + .creative_opportunities + .as_ref() .expect("should be present when should_run_auction is true"); let auction_request = build_auction_request( &matched_slots, @@ -608,7 +617,10 @@ pub async fn handle_publisher_request( provider_responses: None, services, }; - match orchestrator.run_auction(&auction_request, &auction_context, services).await { + match orchestrator + .run_auction(&auction_request, &auction_context, services) + .await + { Ok(result) => Some(result), Err(e) => { log::warn!("server-side auction failed, proceeding without bids: {e:?}"); @@ -620,11 +632,13 @@ pub async fn handle_publisher_request( }; if should_run_auction { - let co_config = settings.creative_opportunities.as_ref() + let co_config = settings + .creative_opportunities + .as_ref() .expect("should be present"); - let empty: std::collections::HashMap = - std::collections::HashMap::new(); - let winning_bids = auction_result.as_ref() + let empty: std::collections::HashMap = std::collections::HashMap::new(); + let winning_bids = auction_result + .as_ref() .map(|r| &r.winning_bids) .unwrap_or(&empty); let bid_map = build_bid_map(winning_bids, co_config.price_granularity); @@ -815,58 +829,103 @@ pub(crate) fn build_auction_request( } } +/// Escape a JSON string so it is safe to embed inside a JS double-quoted string literal. +/// +/// Backslashes are doubled first (so they survive the next pass), then +/// double-quotes are escaped so they do not terminate the JS string. +/// The result is always valid to write as `JSON.parse("…")`. +fn html_escape_for_script(s: &str) -> String { + s.replace('\\', "\\\\").replace('"', "\\\"") +} + /// Build a price-bucketed bid map from winning bids. /// -/// Returns a map of slot ID → bucketed CPM string. +/// Returns a JSON object map of slot ID → bid metadata including the bucketed +/// CPM (`hb_pb`), bidder (`hb_bidder`), and optional ad ID, nurl, and burl. pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, -) -> std::collections::HashMap { +) -> serde_json::Map { winning_bids .iter() .filter_map(|(slot_id, bid)| { bid.price.map(|cpm| { let bucket = price_bucket(cpm, granularity); - (slot_id.clone(), bucket) + let mut obj = serde_json::Map::new(); + obj.insert("hb_pb".to_string(), serde_json::Value::String(bucket)); + obj.insert( + "hb_bidder".to_string(), + serde_json::Value::String(bid.bidder.clone()), + ); + if let Some(ref ad_id) = bid.ad_id { + obj.insert( + "hb_adid".to_string(), + serde_json::Value::String(ad_id.clone()), + ); + } + if let Some(ref nurl) = bid.nurl { + obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + } + if let Some(ref burl) = bid.burl { + obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + } + (slot_id.clone(), serde_json::Value::Object(obj)) }) }) .collect() } -/// Build the `__ts_bids` inline script content from a bucketed bid map. -pub(crate) fn build_bids_script(bid_map: &std::collections::HashMap) -> String { - let entries: Vec = bid_map - .iter() - .map(|(slot_id, bucket)| format!("\"{}\":\"{}\"", slot_id, bucket)) - .collect(); - format!("window.__ts_bids={{{}}};", entries.join(",")) +/// Build the `__ts_bids` `` sequences inside the string. +pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + let json = serde_json::to_string(bid_map).unwrap_or_else(|_| "{}".to_string()); + let escaped = html_escape_for_script(&json); + format!( + "", + escaped + ) } -/// Build the `__ts_ad_slots` inline script content from matched slots. +/// Build the `__ts_ad_slots` `", + escaped + ) } /// Whether the content type requires processing (URL rewriting, HTML injection). From c212ec544138791419b8faed992627101a7a60dc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 11:47:51 +0530 Subject: [PATCH 019/195] Wire slots_file and orchestrator into adapter; parse creative-opportunities.toml at startup --- Cargo.lock | 8 +++++ .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/main.rs | 14 ++++++++- crates/trusted-server-core/build.rs | 11 +++---- .../src/creative_opportunities.rs | 30 +++++++++++++------ crates/trusted-server-core/src/lib.rs | 2 +- crates/trusted-server-core/src/settings.rs | 4 ++- 7 files changed, 53 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e06ac75e7..65d1d777c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,6 +1151,12 @@ dependencies = [ "wasip2", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -2707,6 +2713,7 @@ dependencies = [ "log-fastly", "serde", "serde_json", + "toml 1.0.7+spec-1.1.0", "trusted-server-core", "urlencoding", ] @@ -2731,6 +2738,7 @@ dependencies = [ "fastly", "flate2", "futures", + "glob", "hex", "hmac", "http", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index e483ea621..a730efcd6 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -20,6 +20,7 @@ log = { workspace = true } log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +toml = { workspace = true } trusted-server-core = { workspace = true } urlencoding = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 52c869d7f..74414220b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -39,6 +39,8 @@ use crate::error::to_error_response; use crate::logging::init_logger; use crate::platform::{build_runtime_services, open_kv_store, UnavailableKvStore}; +const CREATIVE_OPPORTUNITIES_TOML: &str = include_str!("../../../creative-opportunities.toml"); + /// Entry point for the Fastly Compute program. /// /// Uses an undecorated `main()` with `Request::from_client()` instead of @@ -80,6 +82,10 @@ fn main() { } }; + let slots_file: trusted_server_core::creative_opportunities::CreativeOpportunitiesFile = + toml::from_str(CREATIVE_OPPORTUNITIES_TOML) + .expect("should parse creative-opportunities.toml"); + let integration_registry = match IntegrationRegistry::new(&settings) { Ok(r) => r, Err(e) => { @@ -103,6 +109,7 @@ fn main() { &orchestrator, &integration_registry, &runtime_services, + &slots_file, req, )) { response.send_to_client(); @@ -114,6 +121,7 @@ async fn route_request( orchestrator: &AuctionOrchestrator, integration_registry: &IntegrationRegistry, runtime_services: &RuntimeServices, + slots_file: &trusted_server_core::creative_opportunities::CreativeOpportunitiesFile, mut req: Request, ) -> Option { // Strip client-spoofable forwarded headers at the edge. @@ -221,8 +229,12 @@ async fn route_request( settings, integration_registry, &publisher_services, + orchestrator, + slots_file, req, - ) { + ) + .await + { Ok(PublisherResponse::Stream { mut response, body, diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index 469c11048..b21cb6845 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -92,14 +92,15 @@ fn main() { let co_path = Path::new(CREATIVE_OPPORTUNITIES_PATH); if co_path.exists() { - let co_content = fs::read_to_string(co_path) - .expect("should read creative-opportunities.toml"); - let co_value: toml::Value = toml::from_str(&co_content) - .expect("creative-opportunities.toml: invalid TOML"); + let co_content = + fs::read_to_string(co_path).expect("should read creative-opportunities.toml"); + let co_value: toml::Value = + toml::from_str(&co_content).expect("creative-opportunities.toml: invalid TOML"); let slot_id_re = regex::Regex::new(r"^[A-Za-z0-9_\-]+$").expect("should compile regex"); if let Some(slots) = co_value.get("slot").and_then(|v| v.as_array()) { for slot in slots { - let id = slot.get("id") + let id = slot + .get("id") .and_then(|v| v.as_str()) .expect("creative-opportunities.toml: slot missing 'id' field"); if !slot_id_re.is_match(id) { diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 7bf3856c2..f051c340f 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -6,9 +6,10 @@ use std::collections::HashMap; -use glob::Pattern; use serde::{Deserialize, Serialize}; +use glob::Pattern; + use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::price_bucket::PriceGranularity; @@ -64,8 +65,9 @@ impl CreativeOpportunitySlot { /// Patterns that cannot be compiled even after normalisation are silently skipped. #[must_use] pub fn matches_path(&self, path: &str) -> bool { - self.page_patterns.iter().any(|pattern| { - match Pattern::new(pattern) { + self.page_patterns + .iter() + .any(|pattern| match Pattern::new(pattern) { Ok(p) => p.matches(path), Err(_) => { let normalised = pattern.replace("**", "*"); @@ -73,8 +75,7 @@ impl CreativeOpportunitySlot { .map(|p| p.matches(path)) .unwrap_or(false) } - } - }) + }) } /// Returns the GAM ad unit path for this slot. @@ -227,7 +228,10 @@ mod tests { #[test] fn glob_matches_article_path() { let slot = make_slot("atf", vec!["/20**"]); - assert!(slot.matches_path("/2024/01/my-article/"), "should match article path"); + assert!( + slot.matches_path("/2024/01/my-article/"), + "should match article path" + ); assert!(!slot.matches_path("/"), "should not match root"); } @@ -243,14 +247,20 @@ mod tests { assert!(validate_slot_id("atf_sidebar_ad").is_ok()); assert!(validate_slot_id("below-content-0").is_ok()); assert!(validate_slot_id("").is_err(), "empty id should fail"); - assert!(validate_slot_id("xss"); + assert!(!inner.contains('<'), "no unescaped < in script content"); + assert!(!inner.contains('>'), "no unescaped > in script content"); + } + + #[test] + fn bid_map_includes_nurl_and_burl() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ), + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); + let obj = entry.as_object().expect("should be object"); + assert_eq!( + obj.get("hb_pb").and_then(|v| v.as_str()), + Some("1.50"), + "should bucket price with dense granularity" + ); + assert_eq!( + obj.get("hb_bidder").and_then(|v| v.as_str()), + Some("kargo"), + "should include bidder" + ); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("abc123"), + "should include ad_id" + ); + assert_eq!( + obj.get("nurl").and_then(|v| v.as_str()), + Some("https://ssp/win"), + "should include nurl" + ); + assert_eq!( + obj.get("burl").and_then(|v| v.as_str()), + Some("https://ssp/bill"), + "should include burl" + ); + } + + #[test] + fn bid_map_excludes_slot_when_price_is_none() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "no-price-slot".to_string(), + Bid { + slot_id: "no-price-slot".to_string(), + price: None, + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + assert!( + map.is_empty(), + "slot with no price should be excluded from bid map" + ); + } + + #[test] + fn bids_script_is_xss_safe() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let script = build_bids_script(&map); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in bids script"); + assert!(!inner.contains('>'), "no unescaped > in bids script"); + } + + #[test] + fn html_escape_encodes_special_chars() { + assert_eq!( + html_escape_for_script("text\\with\\backslash"), + "text\\\\with\\\\backslash", + "should escape backslashes" + ); + assert_eq!( + html_escape_for_script("string\"with\"quotes"), + "string\\\"with\\\"quotes", + "should escape quotes" + ); + assert_eq!( + html_escape_for_script("simple"), + "simple", + "should not change simple text" + ); + assert_eq!( + html_escape_for_script("both\\\"mixed"), + "both\\\\\\\"mixed", + "should escape both backslashes and quotes" + ); + } + } } From b047add10a3f9138949b4ad19e783fca2e3b9a8d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 13:24:00 +0530 Subject: [PATCH 023/195] Enable server-side auction with APS provider and adserver_mock mediator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enable APS and adserver_mock in auction config; set providers and mediator - Increase auction_timeout_ms from 500ms to 3000ms — 500ms was too tight for HTTPS round-trips to mocktioneer, leaving the mediator zero budget - Fix mediation request: send numeric price instead of opaque encoded_price; mocktioneer requires a decoded price field and does not support encoded_price - Expand creative-opportunities slot page_patterns to include /news/** --- .../src/integrations/adserver_mock.rs | 45 +++++++------------ creative-opportunities.toml | 2 +- trusted-server.toml | 12 ++--- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 7ed2da595..8ec94a9c5 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -133,36 +133,21 @@ impl AdServerMockProvider { .bids .iter() .map(|bid| { - // Check if this is an APS bid with encoded price (inferred from amznbid in metadata) - let encoded_price = bid - .metadata - .get("amznbid") - .and_then(|v| v.as_str()) - .map(String::from); - - if encoded_price.is_some() { - // APS bid - send encoded price for mediation to decode - json!({ - "imp_id": bid.slot_id, - "encoded_price": encoded_price, - "adm": bid.creative, - "w": bid.width, - "h": bid.height, - "crid": format!("{}-creative", bid.bidder), - "adomain": bid.adomain, - }) - } else { - // Regular bid with decoded price - json!({ - "imp_id": bid.slot_id, - "price": bid.price, - "adm": bid.creative, - "w": bid.width, - "h": bid.height, - "crid": format!("{}-creative", bid.bidder), - "adomain": bid.adomain, - }) - } + // Mocktioneer mediator always requires a numeric `price` field. + // APS bids carry price as an opaque encoded string (`amznbid`) + // that cannot be decoded client-side; use `bid.price` when set + // (a real decoded value) or fall back to a mock floor price for + // test/demo purposes. + let price = bid.price.unwrap_or(1.50); + json!({ + "imp_id": bid.slot_id, + "price": price, + "adm": bid.creative, + "w": bid.width, + "h": bid.height, + "crid": format!("{}-creative", bid.bidder), + "adomain": bid.adomain, + }) }) .collect(); diff --git a/creative-opportunities.toml b/creative-opportunities.toml index b44e215b6..b79d23810 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -5,7 +5,7 @@ id = "atf_sidebar_ad" gam_unit_path = "/21765378893/publisher/atf-sidebar" div_id = "div-atf-sidebar" -page_patterns = ["/20**"] +page_patterns = ["/", "/20**", "/news/**"] formats = [{ width = 300, height = 250 }] floor_price = 0.50 diff --git a/trusted-server.toml b/trusted-server.toml index c2ecab335..8036b7ec4 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -161,16 +161,16 @@ rewrite_script = true [auction] enabled = true -providers = ["prebid"] -# mediator = "adserver_mock" # will use mediator when set +providers = ["prebid", "aps"] +mediator = "adserver_mock" timeout_ms = 2000 # Context keys the JS client is allowed to forward into auction requests. # Keys not in this list are silently dropped. An empty list blocks all keys. allowed_context_keys = ["permutive_segments"] [integrations.aps] -enabled = false -pub_id = "your-aps-publisher-id" +enabled = true +pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" timeout_ms = 1000 @@ -180,7 +180,7 @@ container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" [integrations.adserver_mock] -enabled = false +enabled = true endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" timeout_ms = 1000 @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "21765378893" -auction_timeout_ms = 500 +auction_timeout_ms = 3000 price_granularity = "dense" From 6a5df1060471818c8335178ce35ecd88978aa2ac Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 13:35:24 +0530 Subject: [PATCH 024/195] Fix adserver_mock test for numeric price; fix GPT JS formatting --- .../js/lib/src/integrations/gpt/index.test.ts | 150 +++++++++--------- crates/js/lib/src/integrations/gpt/index.ts | 14 +- .../src/integrations/adserver_mock.rs | 19 +-- 3 files changed, 91 insertions(+), 92 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 0a6993818..7e2783f2f 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -1,20 +1,20 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' +import { describe, it, expect, vi, beforeEach } from 'vitest'; describe('installTsAdInit', () => { beforeEach(() => { - vi.resetModules() - delete (window as any).__ts_ad_slots - delete (window as any).__ts_bids - delete (window as any).__tsAdInit + vi.resetModules(); + delete (window as any).__ts_ad_slots; + delete (window as any).__ts_bids; + delete (window as any).__tsAdInit; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { Object.defineProperty(navigator, 'sendBeacon', { value: vi.fn().mockReturnValue(true), writable: true, configurable: true, - }) + }); } - }) + }); it('reads window.__ts_bids synchronously and applies bid targeting before refresh', async () => { const mockSlot = { @@ -22,19 +22,19 @@ describe('installTsAdInit', () => { setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), getTargeting: vi.fn().mockReturnValue(['abc']), - } + }; const mockPubads = { enableSingleRequest: vi.fn(), addEventListener: vi.fn(), refresh: vi.fn(), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [ + }; + (window as any).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -42,8 +42,8 @@ describe('installTsAdInit', () => { formats: [[300, 250]], targeting: { pos: 'atf' }, }, - ] - ;(window as any).__ts_bids = { + ]; + (window as any).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -51,47 +51,47 @@ describe('installTsAdInit', () => { nurl: 'https://ssp/win', burl: 'https://ssp/bill', }, - } + }; - const fetchSpy = vi.spyOn(global, 'fetch') + const fetchSpy = vi.spyOn(global, 'fetch'); - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); - expect(fetchSpy).not.toHaveBeenCalled() - expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') - expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') - expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1') - expect(mockPubads.refresh).toHaveBeenCalled() + expect(fetchSpy).not.toHaveBeenCalled(); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockPubads.refresh).toHaveBeenCalled(); - fetchSpy.mockRestore() - }) + fetchSpy.mockRestore(); + }); it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { - const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) - let capturedListener: ((e: any) => void) | undefined + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: any) => void) | undefined; const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), getTargeting: vi.fn().mockReturnValue(['abc']), - } + }; const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), addEventListener: vi.fn((event: string, fn: (e: any) => void) => { - if (event === 'slotRenderEnded') capturedListener = fn + if (event === 'slotRenderEnded') capturedListener = fn; }), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [ + }; + (window as any).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -99,8 +99,8 @@ describe('installTsAdInit', () => { formats: [[300, 250]], targeting: {}, }, - ] - ;(window as any).__ts_bids = { + ]; + (window as any).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -108,44 +108,44 @@ describe('installTsAdInit', () => { nurl: 'https://ssp/win', burl: 'https://ssp/bill', }, - } + }; - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); - expect(capturedListener).toBeDefined() - capturedListener!({ isEmpty: false, slot: mockSlot }) + expect(capturedListener).toBeDefined(); + capturedListener!({ isEmpty: false, slot: mockSlot }); - expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win') - expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') - beaconSpy.mockRestore() - }) + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill'); + beaconSpy.mockRestore(); + }); it('does not fire nurl/burl when bid did not win GAM line item', async () => { - const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) - let capturedListener: ((e: any) => void) | undefined + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: any) => void) | undefined; const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('atf'), getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), - } + }; const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), addEventListener: vi.fn((event: string, fn: (e: any) => void) => { - if (event === 'slotRenderEnded') capturedListener = fn + if (event === 'slotRenderEnded') capturedListener = fn; }), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [ + }; + (window as any).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -153,8 +153,8 @@ describe('installTsAdInit', () => { formats: [[300, 250]], targeting: {}, }, - ] - ;(window as any).__ts_bids = { + ]; + (window as any).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -162,24 +162,24 @@ describe('installTsAdInit', () => { nurl: 'https://ssp/win', burl: 'https://ssp/bill', }, - } + }; - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() - capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }) + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); + capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }); - expect(beaconSpy).not.toHaveBeenCalled() - beaconSpy.mockRestore() - }) + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); it('calls refresh even when __ts_bids is empty (graceful fallback)', async () => { const mockPubads = { enableSingleRequest: vi.fn(), addEventListener: vi.fn(), refresh: vi.fn(), - } - ;(window as any).googletag = { + }; + (window as any).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue({ addService: vi.fn().mockReturnThis(), @@ -187,14 +187,14 @@ describe('installTsAdInit', () => { }), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), - } - ;(window as any).__ts_ad_slots = [] - ;(window as any).__ts_bids = {} + }; + (window as any).__ts_ad_slots = []; + (window as any).__ts_bids = {}; - const { installTsAdInit } = await import('./index') - installTsAdInit() - ;(window as any).__tsAdInit() + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as any).__tsAdInit(); - expect(mockPubads.refresh).toHaveBeenCalled() - }) -}) + expect(mockPubads.refresh).toHaveBeenCalled(); + }); +}); diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 1494d793f..95b6d4279 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -217,7 +217,11 @@ export function installTsAdInit(): void { g.cmd?.push(() => { slots .map((slot) => { - const gptSlot = g.defineSlot?.(slot.gam_unit_path, slot.formats as Array, slot.div_id); + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ); if (!gptSlot) return null; gptSlot.addService(g.pubads!()); Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); @@ -280,13 +284,13 @@ export function installSlimPrebidLoader(): void { // regardless of script order, the module also checks for a pre-set enable flag // immediately after registering the function. if (typeof window !== 'undefined') { - const win = window as Record + const win = window as Record; - win.__tsjs_installGptShim = installGptShim + win.__tsjs_installGptShim = installGptShim; if (win.__tsjs_gpt_enabled === true) { - installGptShim() + installGptShim(); } - installTsAdInit() + installTsAdInit(); } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 8ec94a9c5..3a42ec2a0 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -675,20 +675,15 @@ mod tests { let bid = &bidder_resp["bids"][0]; assert_eq!(bid["imp_id"], "slot-1"); - // Key assertions for APS-style encoded price bids: - // 1. Should NOT have "price" field (or it should be null) - assert!( - bid["price"].is_null(), - "APS bids should not have decoded price, got: {:?}", - bid["price"] - ); - // 2. Should have "encoded_price" field + // APS bids have no decoded price (bid.price == None), so the mock floor + // price (1.50) is used. Mocktioneer requires a numeric price field and + // does not accept an opaque encoded_price string. assert_eq!( - bid["encoded_price"].as_str(), - Some("encoded-price-value"), - "APS bids should have encoded_price from metadata" + bid["price"].as_f64(), + Some(1.50), + "APS bids with no decoded price should fall back to mock floor price 1.50" ); - // 3. adm should be null (not a string) + // adm should be null (not a string) assert!( bid["adm"].is_null(), "Creative-less bids should have null adm, got: {:?}", From e6c18ad5ec4de17713a840d2c44e0d2b532b5946 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 14:13:49 +0530 Subject: [PATCH 025/195] Replace explicit any in GPT integration with typed interfaces Define SlotRenderEndedEvent, SlotRenderEvent, and TestWindow types to eliminate all @typescript-eslint/no-explicit-any violations in gpt/index.ts and gpt/index.test.ts. Extend GptWindow with __tsjs_slim_prebid_url so installSlimPrebidLoader avoids the any cast. --- .../js/lib/src/integrations/gpt/index.test.ts | 61 ++++++++++++------- crates/js/lib/src/integrations/gpt/index.ts | 15 +++-- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 7e2783f2f..e908a201e 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -1,11 +1,26 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; +interface SlotRenderEvent { + isEmpty: boolean; + slot: { + getSlotElementId(): string; + getTargeting(key: string): string[]; + }; +} + +type TestWindow = Window & { + googletag?: unknown; + __ts_ad_slots?: unknown; + __ts_bids?: unknown; + __tsAdInit?: () => void; +}; + describe('installTsAdInit', () => { beforeEach(() => { vi.resetModules(); - delete (window as any).__ts_ad_slots; - delete (window as any).__ts_bids; - delete (window as any).__tsAdInit; + delete (window as TestWindow).__ts_ad_slots; + delete (window as TestWindow).__ts_bids; + delete (window as TestWindow).__tsAdInit; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { Object.defineProperty(navigator, 'sendBeacon', { @@ -28,13 +43,13 @@ describe('installTsAdInit', () => { addEventListener: vi.fn(), refresh: vi.fn(), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = [ + (window as TestWindow).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -43,7 +58,7 @@ describe('installTsAdInit', () => { targeting: { pos: 'atf' }, }, ]; - (window as any).__ts_bids = { + (window as TestWindow).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -57,7 +72,7 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); expect(fetchSpy).not.toHaveBeenCalled(); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); @@ -70,7 +85,7 @@ describe('installTsAdInit', () => { it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); - let capturedListener: ((e: any) => void) | undefined; + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -81,17 +96,17 @@ describe('installTsAdInit', () => { const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), - addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { if (event === 'slotRenderEnded') capturedListener = fn; }), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = [ + (window as TestWindow).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -100,7 +115,7 @@ describe('installTsAdInit', () => { targeting: {}, }, ]; - (window as any).__ts_bids = { + (window as TestWindow).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -112,7 +127,7 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); expect(capturedListener).toBeDefined(); capturedListener!({ isEmpty: false, slot: mockSlot }); @@ -124,7 +139,7 @@ describe('installTsAdInit', () => { it('does not fire nurl/burl when bid did not win GAM line item', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); - let capturedListener: ((e: any) => void) | undefined; + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), @@ -135,17 +150,17 @@ describe('installTsAdInit', () => { const mockPubads = { enableSingleRequest: vi.fn(), refresh: vi.fn(), - addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { if (event === 'slotRenderEnded') capturedListener = fn; }), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = [ + (window as TestWindow).__ts_ad_slots = [ { id: 'atf', gam_unit_path: '/123/atf', @@ -154,7 +169,7 @@ describe('installTsAdInit', () => { targeting: {}, }, ]; - (window as any).__ts_bids = { + (window as TestWindow).__ts_bids = { atf: { hb_pb: '1.00', hb_bidder: 'kargo', @@ -166,7 +181,7 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }); expect(beaconSpy).not.toHaveBeenCalled(); @@ -179,7 +194,7 @@ describe('installTsAdInit', () => { addEventListener: vi.fn(), refresh: vi.fn(), }; - (window as any).googletag = { + (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue({ addService: vi.fn().mockReturnThis(), @@ -188,12 +203,12 @@ describe('installTsAdInit', () => { pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as any).__ts_ad_slots = []; - (window as any).__ts_bids = {}; + (window as TestWindow).__ts_ad_slots = []; + (window as TestWindow).__ts_bids = {}; const { installTsAdInit } = await import('./index'); installTsAdInit(); - (window as any).__tsAdInit(); + (window as TestWindow).__tsAdInit!(); expect(mockPubads.refresh).toHaveBeenCalled(); }); diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 95b6d4279..ffb4a687f 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -32,13 +32,19 @@ interface GoogleTagSlot { getSlotElementId(): string; setTargeting(key: string, value: string | string[]): GoogleTagSlot; addService(service: GoogleTagPubAdsService): GoogleTagSlot; + getTargeting?(key: string): string[]; +} + +interface SlotRenderEndedEvent { + isEmpty: boolean; + slot: GoogleTagSlot; } interface GoogleTagPubAdsService { setTargeting(key: string, value: string | string[]): GoogleTagPubAdsService; getTargeting(key: string): string[]; enableSingleRequest(): void; - addEventListener(event: string, fn: (e: any) => void): void; + addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; refresh(): void; } @@ -57,6 +63,7 @@ interface GoogleTag { type GptWindow = Window & { googletag?: Partial; + __tsjs_slim_prebid_url?: string; }; // ------------------------------------------------------------------ @@ -237,7 +244,7 @@ export function installTsAdInit(): void { g.pubads!().enableSingleRequest(); g.enableServices?.(); - g.pubads!().addEventListener?.('slotRenderEnded', (event: any) => { + g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { const slotId: string = event.slot?.getSlotElementId?.() ?? ''; const bid = bids[slotId] ?? {}; const ourBidWon = @@ -265,7 +272,7 @@ export function installTsAdInit(): void { * the slim-Prebid bundle build target ships in a later phase). */ export function installSlimPrebidLoader(): void { - const url = (window as any).__tsjs_slim_prebid_url as string | undefined; + const url = (window as GptWindow).__tsjs_slim_prebid_url; if (!url) return; window.addEventListener('load', () => { const script = document.createElement('script'); @@ -284,7 +291,7 @@ export function installSlimPrebidLoader(): void { // regardless of script order, the module also checks for a pre-set enable flag // immediately after registering the function. if (typeof window !== 'undefined') { - const win = window as Record; + const win = window as unknown as Record; win.__tsjs_installGptShim = installGptShim; From 74bbc25b4b52ab1b5ea012894d109c67e606ceb2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 15:33:39 +0530 Subject: [PATCH 026/195] Update creative-opportunities config to real autoblog.com GAM values Set gam_network_id to 88059007 (autoblog production network). Update atf_sidebar_ad slot to /88059007/autoblog/news with div_id ad-atf_sidebar-0-_r_2_ (desktop ATF sidebar, 300x250); restrict page_patterns to article paths only (/20**, /news/**) since that div does not exist on the homepage. Add homepage_header_ad slot targeting /88059007/autoblog/homepage with ad-header-0-_R_jpalubtak5lb_ for 970x90/728x90/970x250 leaderboard formats. Reduce auction_timeout_ms from 3000 to 500 to cap TTFB at the spec-recommended ceiling. --- creative-opportunities.toml | 21 ++++++++++++++++++--- trusted-server.toml | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/creative-opportunities.toml b/creative-opportunities.toml index b79d23810..0261110a2 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -3,9 +3,9 @@ [[slot]] id = "atf_sidebar_ad" -gam_unit_path = "/21765378893/publisher/atf-sidebar" -div_id = "div-atf-sidebar" -page_patterns = ["/", "/20**", "/news/**"] +gam_unit_path = "/88059007/autoblog/news" +div_id = "ad-atf_sidebar-0-_r_2_" +page_patterns = ["/20**", "/news/**"] formats = [{ width = 300, height = 250 }] floor_price = 0.50 @@ -15,3 +15,18 @@ zone = "atfSidebar" [slot.providers.aps] slot_id = "aps-slot-atf-sidebar" + +[[slot]] +id = "homepage_header_ad" +gam_unit_path = "/88059007/autoblog/homepage" +div_id = "ad-header-0-_R_jpalubtak5lb_" +page_patterns = ["/"] +formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }, { width = 970, height = 250 }] +floor_price = 0.50 + +[slot.targeting] +pos = "atf" +zone = "header" + +[slot.providers.aps] +slot_id = "aps-slot-homepage-header" diff --git a/trusted-server.toml b/trusted-server.toml index 8036b7ec4..da00c3ed7 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -191,7 +191,7 @@ timeout_ms = 1000 permutive_segments = "permutive" [creative_opportunities] -gam_network_id = "21765378893" -auction_timeout_ms = 3000 +gam_network_id = "88059007" +auction_timeout_ms = 500 price_granularity = "dense" From 51aba8f1b48a5a2c18bf1fb3df5ed76fc66d837c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 17:49:49 +0530 Subject: [PATCH 027/195] Update auction timeout and APS slot ID bug --- .../src/integrations/aps.rs | 139 ++++++++++++++++-- trusted-server.toml | 2 +- 2 files changed, 128 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 79eca5a32..ba6c14bbd 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -286,24 +286,46 @@ impl IntegrationConfig for ApsConfig { /// Amazon APS auction provider. pub struct ApsAuctionProvider { config: ApsConfig, + // Maps APS slot ID → creative opportunity slot ID for the in-flight request. + // Written by request_bids before the async send; read by parse_response when the + // response arrives. Safe because Fastly Compute runs each request in an isolated + // single-threaded Wasm instance — the Mutex never contends in practice. + slot_id_map: std::sync::Mutex>, } impl ApsAuctionProvider { /// Create a new APS auction provider. #[must_use] pub fn new(config: ApsConfig) -> Self { - Self { config } + Self { + config, + slot_id_map: std::sync::Mutex::new(HashMap::new()), + } } /// Convert unified `AuctionRequest` to APS TAM bid request format. /// + /// Returns the serialisable `ApsBidRequest` and a map of APS slot ID → + /// creative-opportunity slot ID so the caller can remap bids in the response. /// Populates consent fields (GDPR, US Privacy, GPP) from the /// [`ConsentContext`](crate::consent::ConsentContext) attached to the request. - fn to_aps_request(&self, request: &AuctionRequest) -> ApsBidRequest { + fn to_aps_request(&self, request: &AuctionRequest) -> (ApsBidRequest, HashMap) { + let mut slot_id_map: HashMap = HashMap::new(); let slots: Vec = request .slots .iter() .map(|slot| { + // Use the APS-specific slot ID from [slot.providers.aps] if configured; + // fall back to the creative-opportunity slot ID otherwise. + let aps_slot_id = slot + .bidders + .get("aps") + .and_then(|p| p.get("slotID")) + .and_then(|v| v.as_str()) + .unwrap_or(&slot.id) + .to_string(); + slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()); + // Extract sizes from banner formats let sizes: Vec<[u32; 2]> = slot .formats @@ -313,7 +335,7 @@ impl ApsAuctionProvider { .collect(); ApsSlot { - slot_id: slot.id.clone(), + slot_id: aps_slot_id, sizes, slot_name: Some(slot.id.clone()), } @@ -337,7 +359,7 @@ impl ApsAuctionProvider { }) }); - ApsBidRequest { + let bid_request = ApsBidRequest { pub_id: self.config.pub_id.clone(), slots, page_url: request.publisher.page_url.clone(), @@ -347,7 +369,8 @@ impl ApsAuctionProvider { us_privacy, gpp, gpp_sid, - } + }; + (bid_request, slot_id_map) } /// Parse size string (e.g., "300x250") into width and height. @@ -433,9 +456,19 @@ impl ApsAuctionProvider { aps_response.contextual.slots.len() ); + let slot_map = self + .slot_id_map + .lock() + .expect("should lock APS slot id map"); for slot in aps_response.contextual.slots { match self.parse_aps_slot(&slot) { - Ok(bid) => { + Ok(mut bid) => { + // Remap APS slot ID (e.g. "aps-slot-atf-sidebar") back to the + // creative-opportunity slot ID (e.g. "atf_sidebar_ad") so the + // mediator and bid_map can match by creative slot ID. + if let Some(creative_id) = slot_map.get(&bid.slot_id) { + bid.slot_id = creative_id.clone(); + } let encoded_price = bid .metadata .get("amznbid") @@ -485,8 +518,13 @@ impl AuctionProvider for ApsAuctionProvider { self.config.pub_id ); - // Transform to APS format - let aps_request = self.to_aps_request(request); + // Transform to APS format; store the APS-slot-ID → creative-slot-ID map so + // parse_response can remap bids back to the creative opportunity slot ID. + let (aps_request, slot_id_map) = self.to_aps_request(request); + *self + .slot_id_map + .lock() + .expect("should lock APS slot id map") = slot_id_map; // Serialize to JSON let aps_json = @@ -703,7 +741,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let auction_request = create_test_auction_request(); - let aps_request = provider.to_aps_request(&auction_request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&auction_request); // Verify basic fields assert_eq!(aps_request.pub_id, "5128"); @@ -729,6 +767,83 @@ mod tests { assert_eq!(slot2.sizes[0], [300, 250]); } + #[test] + fn aps_slot_id_from_bidders_map_used_in_request_and_remapped_in_response() { + use serde_json::json; + + let config = ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 800, + }; + let provider = ApsAuctionProvider::new(config); + + let mut bidders = HashMap::new(); + bidders.insert( + "aps".to_string(), + json!({ "slotID": "aps-slot-atf-sidebar" }), + ); + let request = AuctionRequest { + id: "test".to_string(), + slots: vec![AdSlot { + id: "atf_sidebar_ad".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: None, + targeting: HashMap::new(), + bidders, + }], + publisher: PublisherInfo { + domain: "example.com".to_string(), + page_url: None, + }, + user: UserInfo { + id: "user-1".to_string(), + fresh_id: "fresh-1".to_string(), + consent: None, + }, + device: None, + site: None, + context: HashMap::new(), + }; + + let (aps_request, slot_id_map) = provider.to_aps_request(&request); + assert_eq!( + aps_request.slots[0].slot_id, "aps-slot-atf-sidebar", + "should send configured APS slot ID to APS" + ); + assert_eq!( + slot_id_map.get("aps-slot-atf-sidebar").map(String::as_str), + Some("atf_sidebar_ad"), + "should build reverse map from APS slot ID to creative slot ID" + ); + + *provider.slot_id_map.lock().expect("should lock") = slot_id_map; + + let aps_response = json!({ + "contextual": { + "slots": [{ + "slotID": "aps-slot-atf-sidebar", + "size": "300x250", + "fif": "1", + "amznbid": "1gtm3q", + "meta": ["slotID"] + }] + } + }); + + let response = provider.parse_aps_response(&aps_response, 100); + assert_eq!(response.bids.len(), 1, "should parse one bid"); + assert_eq!( + response.bids[0].slot_id, "atf_sidebar_ad", + "bid slot_id should be remapped to creative slot ID" + ); + } + #[test] fn test_aps_response_parsing_success() { let config = ApsConfig { @@ -957,7 +1072,7 @@ mod tests { ..Default::default() }); - let aps_request = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request); // Verify GDPR consent let gdpr = aps_request.gdpr.expect("should have gdpr"); @@ -986,7 +1101,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let request = create_test_auction_request(); // consent is None - let aps_request = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request); assert!(aps_request.gdpr.is_none()); assert!(aps_request.us_privacy.is_none()); @@ -1013,7 +1128,7 @@ mod tests { ..Default::default() }); - let aps_request = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request); let json = serde_json::to_value(&aps_request).expect("should serialize"); // GDPR fields present diff --git a/trusted-server.toml b/trusted-server.toml index da00c3ed7..43e090fea 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" -auction_timeout_ms = 500 +auction_timeout_ms = 1500 price_granularity = "dense" From 3d51fe487e68d08621b0c6a5ffa1364406f45ac1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 18:43:18 +0530 Subject: [PATCH 028/195] Call __tsAdInit after injecting __ts_bids into page The bids script set window.__ts_bids but never invoked the __tsAdInit function, leaving GPT slots undefined and server-side targeting (hb_pb, hb_bidder) never applied. Both the winning-bid path (build_bids_script) and the no-auction fallback (html_processor None branch) now guard-call the function after the assignment. --- crates/trusted-server-core/src/html_processor.rs | 2 +- crates/trusted-server-core/src/publisher.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 9ef6edb68..45e066609 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -301,7 +301,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let script_guard = state.read().expect("should read bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), - None => r#""# + None => r#""# .to_string(), }; end_tag.before(&bids_script, ContentType::Html); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 73e489dc6..193f702c3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -883,7 +883,7 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Mapwindow.__ts_bids=JSON.parse(\"{}\");", + "", escaped ) } From 4cf6d98c3adae70c1fdec3ca1c97f531136a16ef Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 18:50:33 +0530 Subject: [PATCH 029/195] Fix format error --- crates/trusted-server-core/src/html_processor.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 45e066609..a3608d9ec 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -296,8 +296,8 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { let state = state.clone(); if let Some(handlers) = el.end_tag_handlers() { - let handler: EndTagHandler<'static> = - Box::new(move |end_tag: &mut EndTag<'_>| { + let handler: EndTagHandler<'static> = Box::new( + move |end_tag: &mut EndTag<'_>| { let script_guard = state.read().expect("should read bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), @@ -306,7 +306,8 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso }; end_tag.before(&bids_script, ContentType::Html); Ok(()) - }); + }, + ); handlers.push(handler); } Ok(()) From e06af4b0fddee2f6e1ecffba436a7af4f333f247 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 19:36:41 +0530 Subject: [PATCH 030/195] Add PBS inline bidder params via creative-opportunities.toml Adds [slot.providers.pbs.bidders] support so PBS bidder params live in creative-opportunities.toml alongside APS params, without needing PBS stored requests configured server-side. PrebidAuctionProvider now sends imp.ext.prebid.storedrequest.id as a fallback for slots with no inline PBS params, and skips non-PBS provider keys (e.g. "aps") that belong to separate auction providers. PrebidImpExt gains an optional storedrequest field; empty bidder maps are omitted during serialisation. Wires mocktioneer and criteo (placeholder IDs) for both autoblog creative-opportunity slots. --- .../src/creative_opportunities.rs | 66 +++++++++- .../src/integrations/prebid.rs | 116 ++++++++++++++++-- crates/trusted-server-core/src/openrtb.rs | 14 ++- creative-opportunities.toml | 8 ++ 4 files changed, 191 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index f051c340f..a7fd99cb6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -99,7 +99,8 @@ impl CreativeOpportunitySlot { /// Converts this slot into an [`AdSlot`] ready for use in an auction request. /// - /// Provider-specific params (e.g., APS `slotID`) are wired into the `bidders` map. + /// Provider-specific params (e.g., APS `slotID`, PBS bidder params) are wired + /// into the `bidders` map keyed by provider/bidder name. #[must_use] pub fn to_ad_slot(&self, gam_network_id: &str) -> AdSlot { let _ = gam_network_id; @@ -110,6 +111,11 @@ impl CreativeOpportunitySlot { serde_json::json!({ "slotID": aps.slot_id }), ); } + if let Some(ref pbs) = self.providers.pbs { + for (bidder_name, params) in &pbs.bidders { + bidders.insert(bidder_name.clone(), params.clone()); + } + } AdSlot { id: self.id.clone(), formats: self @@ -155,6 +161,8 @@ impl CreativeOpportunityFormat { pub struct SlotProviders { /// Amazon Publisher Services (APS/TAM) slot parameters. pub aps: Option, + /// Prebid Server (PBS) slot parameters. + pub pbs: Option, } /// APS-specific parameters for a slot. @@ -164,6 +172,24 @@ pub struct ApsSlotParams { pub slot_id: String, } +/// PBS-specific parameters for a slot. +/// +/// Bidder params are sent inline to Prebid Server so bidder credentials +/// stay in `creative-opportunities.toml` rather than in PBS stored requests. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct PbsSlotParams { + /// Per-bidder params keyed by bidder name (must match PBS adapter name). + /// + /// Example in TOML: + /// ```toml + /// [slot.providers.pbs.bidders] + /// mocktioneer = { bid = 2.00 } + /// criteo = { networkId = 123456, pubid = "123456" } + /// ``` + #[serde(default)] + pub bidders: HashMap, +} + /// TOML file structure for creative opportunity slot definitions. #[derive(Debug, Clone, Deserialize, Default)] pub struct CreativeOpportunitiesFile { @@ -293,6 +319,44 @@ mod tests { ); } + #[test] + fn to_ad_slot_wires_pbs_bidder_params_into_bidders() { + let mut slot = make_slot("atf_sidebar_ad", vec!["/"]); + slot.providers.pbs = Some(PbsSlotParams { + bidders: [ + ( + "mocktioneer".to_string(), + serde_json::json!({ "bid": 2.00 }), + ), + ( + "criteo".to_string(), + serde_json::json!({ "networkId": 123456, "pubid": "123456" }), + ), + ] + .into_iter() + .collect(), + }); + let ad_slot = slot.to_ad_slot("88059007"); + let mock_params = ad_slot + .bidders + .get("mocktioneer") + .expect("should have mocktioneer bidder"); + assert_eq!( + mock_params.get("bid").and_then(|v| v.as_f64()), + Some(2.0), + "should wire mocktioneer bid param" + ); + let criteo_params = ad_slot + .bidders + .get("criteo") + .expect("should have criteo bidder"); + assert_eq!( + criteo_params.get("networkId").and_then(|v| v.as_i64()), + Some(112141), + "should wire criteo networkId param" + ); + } + #[test] fn to_ad_slot_sets_floor_price_and_formats() { let slot = make_slot("atf", vec!["/"]); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 62e112c77..46b87cc0e 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -26,8 +26,8 @@ use crate::integrations::{ }; use crate::openrtb::{ to_openrtb_i32, Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, - OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, - TrustedServerExt, User, UserExt, + ImpStoredRequest, OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, + RequestExt, Site, ToExt, TrustedServerExt, User, UserExt, }; use crate::platform::RuntimeServices; use crate::request_signing::{RequestSigner, SigningParams, SIGNING_VERSION}; @@ -529,22 +529,27 @@ impl PrebidAuctionProvider { // Build the bidder map for PBS. // The JS adapter sends "trustedServer" as the bidder (our orchestrator // adapter name). Replace it with the real PBS bidders from config. - // Pass through any other bidders with their params as-is. + // Only pass through keys that are known PBS bidders — skip provider-specific + // keys like "aps" which belong to their own separate auction provider. let mut bidder: HashMap = HashMap::new(); for (name, params) in &slot.bidders { if name == TRUSTED_SERVER_BIDDER { bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); - } else { + } else if self.config.bidders.iter().any(|b| b == name) { bidder.insert(name.clone(), params.clone()); } } - // Fallback to config bidders if none provided - if bidder.is_empty() { - for b in &self.config.bidders { - bidder.insert(b.clone(), Json::Object(serde_json::Map::new())); - } - } + // When no inline PBS bidder params exist (e.g. creative-opportunity slots + // whose PBS params live in stored requests), tell PBS to resolve bidder + // config from the stored request keyed by this slot ID. + let storedrequest = if bidder.is_empty() { + Some(ImpStoredRequest { + id: slot.id.clone(), + }) + } else { + None + }; // Apply zone-specific bid param overrides when configured. for (name, params) in &mut bidder { @@ -582,7 +587,10 @@ impl PrebidAuctionProvider { secure: Some(true), // require HTTPS creatives tagid: Some(slot.id.clone()), ext: ImpExt { - prebid: PrebidImpExt { bidder }, + prebid: PrebidImpExt { + bidder, + storedrequest, + }, } .to_ext(), ..Default::default() @@ -3044,4 +3052,90 @@ fixed_bottom = {placementId = "_s2sBottom"} assert_eq!(statuses[0]["bidder"], "kargo"); assert_eq!(statuses[1]["status"], "timeout"); } + + // ======================================================================== + // PBS stored request tests + // ======================================================================== + + #[test] + fn to_openrtb_uses_stored_request_when_slot_has_no_pbs_bidder_params() { + // Slot only has "aps" provider — not a PBS bidder + let slot = make_slot( + "atf_sidebar_ad", + HashMap::from([("aps".to_string(), json!({"slotID": "aps-slot-atf-sidebar"}))]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should not send inline bidder params when using stored request" + ); + assert_eq!( + prebid["storedrequest"]["id"], "atf_sidebar_ad", + "should use slot id as stored request id" + ); + } + + #[test] + fn to_openrtb_uses_stored_request_when_slot_has_empty_bidders() { + let slot = make_slot("homepage_header_ad", HashMap::new()); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert_eq!( + prebid["storedrequest"]["id"], "homepage_header_ad", + "should use slot id as stored request id for slot with no bidder map" + ); + } + + #[test] + fn to_openrtb_uses_inline_bidder_params_not_stored_request_for_trusted_server_slots() { + let mut config = base_config(); + config.bidders = vec!["kargo".to_string()]; + + let slot = make_ts_slot( + "in_content_ad", + &json!({ "kargo": { "placementId": "client_123" } }), + None, + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("storedrequest").is_none(), + "should not use stored request when inline bidder params are present" + ); + assert_eq!( + prebid["bidder"]["kargo"]["placementId"], "client_123", + "should use inline bidder params from trustedServer expansion" + ); + } + + #[test] + fn to_openrtb_skips_aps_key_from_slot_bidders_in_pbs_request() { + let slot = make_slot( + "atf_sidebar_ad", + HashMap::from([("aps".to_string(), json!({"slotID": "aps-slot-atf-sidebar"}))]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should not forward aps key into PBS imp.ext.prebid.bidder" + ); + } } diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 3c9be932e..eca5e70f5 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -162,9 +162,21 @@ pub struct ImpExt { impl ToExt for ImpExt {} -#[derive(Debug, Serialize)] +#[derive(Debug, Default, Serialize)] pub struct PrebidImpExt { + #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub bidder: std::collections::HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub storedrequest: Option, +} + +/// PBS imp-level stored request reference. +/// +/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, +/// populating bidder params that are not sent inline. +#[derive(Debug, Serialize)] +pub struct ImpStoredRequest { + pub id: String, } #[derive(Debug, Serialize)] diff --git a/creative-opportunities.toml b/creative-opportunities.toml index 0261110a2..3cd27f2b1 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -16,6 +16,10 @@ zone = "atfSidebar" [slot.providers.aps] slot_id = "aps-slot-atf-sidebar" +[slot.providers.pbs.bidders] +mocktioneer = { bid = 2.00 } +criteo = { networkId = 123456, pubid = "123456" } + [[slot]] id = "homepage_header_ad" gam_unit_path = "/88059007/autoblog/homepage" @@ -30,3 +34,7 @@ zone = "header" [slot.providers.aps] slot_id = "aps-slot-homepage-header" + +[slot.providers.pbs.bidders] +mocktioneer = { bid = 2.00 } +criteo = { networkId = 123456, pubid = "123456" } From 5cbf05f1f9f908bbd200a2de52cdec119396a34f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 19:41:03 +0530 Subject: [PATCH 031/195] Fix clippy errors --- crates/trusted-server-core/src/creative_opportunities.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index a7fd99cb6..fa3449fd4 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -342,7 +342,7 @@ mod tests { .get("mocktioneer") .expect("should have mocktioneer bidder"); assert_eq!( - mock_params.get("bid").and_then(|v| v.as_f64()), + mock_params.get("bid").and_then(serde_json::Value::as_f64), Some(2.0), "should wire mocktioneer bid param" ); @@ -351,7 +351,7 @@ mod tests { .get("criteo") .expect("should have criteo bidder"); assert_eq!( - criteo_params.get("networkId").and_then(|v| v.as_i64()), + criteo_params.get("networkId").and_then(serde_json::Value::as_i64), Some(112141), "should wire criteo networkId param" ); From 60011f08b25f8e062366e15c863623767476acd6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 19:46:07 +0530 Subject: [PATCH 032/195] Fix test assertion --- crates/trusted-server-core/src/creative_opportunities.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index fa3449fd4..7a4a10df5 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -351,8 +351,10 @@ mod tests { .get("criteo") .expect("should have criteo bidder"); assert_eq!( - criteo_params.get("networkId").and_then(serde_json::Value::as_i64), - Some(112141), + criteo_params + .get("networkId") + .and_then(serde_json::Value::as_i64), + Some(123456), "should wire criteo networkId param" ); } From cf5091fabfaa76e08aeb34c5905943ae54dd38de Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 20:27:56 +0530 Subject: [PATCH 033/195] Fix double __ts_bids injection --- .../trusted-server-core/src/html_processor.rs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index a3608d9ec..86a8abe79 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -20,6 +20,7 @@ use std::cell::Cell; use std::io; use std::rc::Rc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use lol_html::{ @@ -246,6 +247,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso }); let injected_tsjs = Rc::new(Cell::new(false)); + let injected_bids = Arc::new(AtomicBool::new(false)); let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); let ad_slots_script = config.ad_slots_script.clone(); @@ -291,13 +293,20 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } }), // Inject __ts_bids before via end_tag_handlers. + // Guard with AtomicBool so the script is only injected once even if + // the origin HTML contains multiple elements (e.g. template fragments). element!("body", { let state = ad_bids_state.clone(); + let injected_bids = injected_bids.clone(); move |el| { let state = state.clone(); + let injected_bids = injected_bids.clone(); if let Some(handlers) = el.end_tag_handlers() { let handler: EndTagHandler<'static> = Box::new( move |end_tag: &mut EndTag<'_>| { + if injected_bids.swap(true, Ordering::SeqCst) { + return Ok(()); + } let script_guard = state.read().expect("should read bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), @@ -1295,6 +1304,32 @@ mod tests { assert!(bids_pos < body_close_pos, "bids must appear before "); } + #[test] + fn injects_ts_bids_only_once_with_multiple_body_elements() { + let bids_script = + r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + // Malformed HTML with two elements (common in CMS template pages) + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert_eq!( + html.matches("window.__ts_bids").count(), + 1, + "should inject __ts_bids exactly once even with multiple elements" + ); + } + #[test] fn injects_empty_ts_bids_when_state_is_none() { let state = std::sync::Arc::new(std::sync::RwLock::new(None)); From eccfd4538547ddb71b2761669fa7e053d88b4cb0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 6 May 2026 21:10:45 +0530 Subject: [PATCH 034/195] Fix max-age cookie issue -> no-store --- crates/trusted-server-core/src/publisher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 193f702c3..c7744ed2e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -668,7 +668,7 @@ pub async fn handle_publisher_request( }; if ad_slots_script.is_some() { - response.set_header(header::CACHE_CONTROL, "private, max-age=0"); + response.set_header(header::CACHE_CONTROL, "private, no-store"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); } From 5bb12d08257da12d3bfa43c85394ee4f4b6198e0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 13:02:30 +0530 Subject: [PATCH 035/195] Add /__ts/page-bids endpoint for pushState/replaceState --- .../js/lib/src/integrations/gpt/index.test.ts | 16 +- crates/js/lib/src/integrations/gpt/index.ts | 159 ++++++++++++++---- .../trusted-server-adapter-fastly/src/main.rs | 14 +- crates/trusted-server-core/src/publisher.rs | 150 ++++++++++++++++- 4 files changed, 301 insertions(+), 38 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index e908a201e..4d501ae34 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -13,6 +13,9 @@ type TestWindow = Window & { __ts_ad_slots?: unknown; __ts_bids?: unknown; __tsAdInit?: () => void; + __tsPrevGptSlots?: unknown; + __tsServicesEnabled?: boolean; + __tsSpaHookInstalled?: boolean; }; describe('installTsAdInit', () => { @@ -21,6 +24,9 @@ describe('installTsAdInit', () => { delete (window as TestWindow).__ts_ad_slots; delete (window as TestWindow).__ts_bids; delete (window as TestWindow).__tsAdInit; + delete (window as TestWindow).__tsPrevGptSlots; + delete (window as TestWindow).__tsSpaHookInstalled; + (window as TestWindow).__tsServicesEnabled = false; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { Object.defineProperty(navigator, 'sendBeacon', { @@ -203,7 +209,15 @@ describe('installTsAdInit', () => { pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), }; - (window as TestWindow).__ts_ad_slots = []; + (window as TestWindow).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ]; (window as TestWindow).__ts_bids = {}; const { installTsAdInit } = await import('./index'); diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index ffb4a687f..06bc7143a 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -45,7 +45,7 @@ interface GoogleTagPubAdsService { getTargeting(key: string): string[]; enableSingleRequest(): void; addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; - refresh(): void; + refresh(slots?: GoogleTagSlot[]): void; } interface GoogleTag { @@ -56,6 +56,7 @@ interface GoogleTag { size: Array, elementId: string ): GoogleTagSlot | null; + destroySlots(slots?: GoogleTagSlot[]): boolean; enableServices(): void; display(elementId: string): void; _loaded_?: boolean; @@ -202,6 +203,8 @@ type TsWindow = Window & { __ts_ad_slots?: TsAdSlot[]; __ts_bids?: Record; __tsAdInit?: () => void; + __tsPrevGptSlots?: GoogleTagSlot[]; + __tsServicesEnabled?: boolean; }; /** @@ -212,6 +215,9 @@ type TsWindow = Window & { * targeting to GPT slots, sets the `ts_initial` sentinel, registers * `slotRenderEnded` to fire both nurl and burl via sendBeacon when our * specific Prebid bid wins the GAM line item match, then calls refresh(). + * + * Idempotent: destroys previously created TS-managed slots before redefining them, + * so it is safe to call again after SPA navigation updates `__ts_ad_slots`/`__ts_bids`. */ export function installTsAdInit(): void { const w = window as TsWindow; @@ -222,46 +228,128 @@ export function installTsAdInit(): void { if (!g) return; g.cmd?.push(() => { - slots - .map((slot) => { - const gptSlot = g.defineSlot?.( - slot.gam_unit_path, - slot.formats as Array, - slot.div_id - ); - if (!gptSlot) return null; - gptSlot.addService(g.pubads!()); - Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); - const bid = bids[slot.id] ?? {}; - (['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, bid[key]!); - }); - gptSlot.setTargeting('ts_initial', '1'); - return { id: slot.id, gptSlot }; - }) - .filter(Boolean); - - g.pubads!().enableSingleRequest(); - g.enableServices?.(); - - g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? ''; - const bid = bids[slotId] ?? {}; - const ourBidWon = - !event.isEmpty && - bid.hb_adid && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; - if (ourBidWon) { - if (bid.nurl) navigator.sendBeacon(bid.nurl); - if (bid.burl) navigator.sendBeacon(bid.burl); - } + // Destroy previously defined TS slots before redefining for the new page. + if (w.__tsPrevGptSlots && w.__tsPrevGptSlots.length > 0) { + g.destroySlots?.(w.__tsPrevGptSlots); + w.__tsPrevGptSlots = []; + } + + const newSlots: GoogleTagSlot[] = []; + + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ); + if (!gptSlot) return; + gptSlot.addService(g.pubads!()); + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); + const bid = bids[slot.id] ?? {}; + (['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!); + }); + gptSlot.setTargeting('ts_initial', '1'); + newSlots.push(gptSlot); }); - g.pubads!().refresh(); + w.__tsPrevGptSlots = newSlots; + + // enableSingleRequest and enableServices must only be called once per page load. + if (!w.__tsServicesEnabled) { + g.pubads!().enableSingleRequest(); + g.enableServices?.(); + w.__tsServicesEnabled = true; + + g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? ''; + const bid = (w.__ts_bids ?? {})[slotId] ?? {}; + const ourBidWon = + !event.isEmpty && + bid.hb_adid && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl); + if (bid.burl) navigator.sendBeacon(bid.burl); + } + }); + } + + if (newSlots.length > 0) { + g.pubads!().refresh(newSlots); + } }); }; } +interface PageBidsResponse { + slots: TsAdSlot[]; + bids: Record; +} + +/** + * Install SPA navigation hook. + * + * Patches `history.pushState` and `history.replaceState`, and listens to + * `popstate`, so that after each client-side route change the trusted server + * fetches fresh slots + bids from `/__ts/page-bids?path=`, updates + * `window.__ts_ad_slots` / `window.__ts_bids`, and calls `window.__tsAdInit()`. + * + * Idempotent: guarded by `window.__tsSpaHookInstalled` so multiple calls are safe. + */ +export function installSpaAuctionHook(): void { + if (typeof window === 'undefined') return; + const win = window as TsWindow & { __tsSpaHookInstalled?: boolean }; + if (win.__tsSpaHookInstalled) return; + win.__tsSpaHookInstalled = true; + + let inflight: AbortController | null = null; + + async function onNavigate(path: string): Promise { + inflight?.abort(); + const controller = new AbortController(); + inflight = controller; + + try { + const res = await fetch(`/__ts/page-bids?path=${encodeURIComponent(path)}`, { + credentials: 'include', + signal: controller.signal, + }); + if (!res.ok) return; + const data = (await res.json()) as PageBidsResponse; + win.__ts_ad_slots = data.slots; + win.__ts_bids = data.bids; + win.__tsAdInit?.(); + } catch (err) { + if (err instanceof DOMException && err.name === 'AbortError') return; + log.warn('SPA auction hook: fetch failed', err); + } + } + + function patchHistoryMethod(method: 'pushState' | 'replaceState'): void { + const original = history[method].bind(history); + history[method] = function ( + state: unknown, + unused: string, + url?: string | URL | null + ): void { + const prevPath = location.pathname; + original(state, unused, url); + const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; + if (newPath !== prevPath) { + void onNavigate(newPath); + } + }; + } + + patchHistoryMethod('pushState'); + patchHistoryMethod('replaceState'); + + window.addEventListener('popstate', () => { + void onNavigate(location.pathname); + }); +} + /** * Register the slim-Prebid lazy loader. Fires after window.load — off the * critical path. slim-Prebid handles refresh auctions and userID module @@ -300,4 +388,5 @@ if (typeof window !== 'undefined') { } installTsAdInit(); + installSpaAuctionHook(); } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 74414220b..55af1468e 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -19,7 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, PublisherResponse, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, + PublisherResponse, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -194,6 +195,17 @@ async fn route_request( } } + // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path + (Method::GET, "/__ts/page-bids") => { + match runtime_services_for_consent_route(settings, runtime_services) { + Ok(publisher_services) => { + handle_page_bids(settings, orchestrator, &publisher_services, slots_file, req) + .await + } + Err(e) => Err(e), + } + } + // tsjs endpoints (Method::GET, "/first-party/proxy") => { handle_first_party_proxy(settings, runtime_services, req).await diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c7744ed2e..ec4f4a227 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -668,7 +668,7 @@ pub async fn handle_publisher_request( }; if ad_slots_script.is_some() { - response.set_header(header::CACHE_CONTROL, "private, no-store"); + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); } @@ -990,6 +990,154 @@ fn apply_ec_headers( } } +/// Handle `GET /__ts/page-bids?path=` — server-side auction for SPA navigation. +/// +/// Matches creative opportunity slots for the given path, runs a server-side +/// auction (APS + PBS), and returns the slot definitions and winning bids as JSON. +/// Called by the client-side SPA navigation hook after `pushState` / `popstate`. +/// +/// # Errors +/// +/// Returns [`TrustedServerError`] if cookie parsing or EC ID generation fails. +pub async fn handle_page_bids( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + req: Request, +) -> Result> { + let Some(co_config) = &settings.creative_opportunities else { + return Ok(Response::from_status(StatusCode::NOT_FOUND) + .with_body_text_plain("Creative opportunities not configured")); + }; + + let path_param = req + .get_url() + .query_pairs() + .find(|(k, _)| k == "path") + .map(|(_, v)| v.into_owned()) + .unwrap_or_else(|| "/".to_string()); + + let matched_slots: Vec<_> = + crate::creative_opportunities::match_slots(&slots_file.slots, &path_param) + .into_iter() + .cloned() + .collect(); + + let request_info = crate::http_util::RequestInfo::from_request(&req, &services.client_info); + let cookie_jar = handle_request_cookies(&req)?; + let ec_id = get_or_generate_ec_id(settings, services, &req)?; + let geo = services + .geo() + .lookup(services.client_info.client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let consent_context = build_consent_context(&ConsentPipelineInput { + jar: cookie_jar.as_ref(), + req: &req, + config: &settings.consent, + geo: geo.as_ref(), + ec_id: Some(ec_id.as_str()), + kv_store: settings + .consent + .consent_store + .as_deref() + .map(|_| services.kv_store()), + }); + + let consent_allows_auction = consent_context + .tcf + .as_ref() + .is_some_and(|tcf| tcf.has_purpose_consent(1)); + + let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { + let mut auction_request = build_auction_request( + &matched_slots, + &ec_id, + &consent_context, + &request_info, + co_config, + ); + let page_url = format!( + "{}://{}{}", + request_info.scheme, request_info.host, path_param + ); + auction_request.publisher.page_url = Some(page_url.clone()); + if let Some(ref mut site) = auction_request.site { + site.page = page_url; + } + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); + let auction_context = AuctionContext { + settings, + request: &placeholder_req, + client_info: services.client_info(), + timeout_ms, + provider_responses: None, + services, + }; + match orchestrator + .run_auction(&auction_request, &auction_context, services) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; + + let bid_map = build_bid_map(&winning_bids, co_config.price_granularity); + + let slots_json: Vec = matched_slots + .iter() + .map(|slot| { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| serde_json::json!([f.width, f.height])) + .collect(); + let targeting: serde_json::Map = slot + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + serde_json::json!({ + "id": slot.id, + "gam_unit_path": gam_path, + "div_id": div_id, + "formats": formats, + "targeting": targeting, + }) + }) + .collect(); + + let body = serde_json::json!({ + "slots": slots_json, + "bids": bid_map, + }); + + let json_str = serde_json::to_string(&body).change_context(TrustedServerError::Proxy { + message: "Failed to serialize page-bids response".to_string(), + })?; + + let mut response = Response::from_status(StatusCode::OK); + response.set_header(header::CONTENT_TYPE, "application/json"); + response.set_header(header::CACHE_CONTROL, "private, no-store"); + response.set_body(json_str); + + Ok(response) +} + #[cfg(test)] mod tests { use super::*; From 982fa3edbf8ed881797c6dff4aacd51d2878d68b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 13:04:19 +0530 Subject: [PATCH 036/195] Fix format ts --- crates/js/lib/src/integrations/gpt/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 06bc7143a..bf9fc99de 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -328,11 +328,7 @@ export function installSpaAuctionHook(): void { function patchHistoryMethod(method: 'pushState' | 'replaceState'): void { const original = history[method].bind(history); - history[method] = function ( - state: unknown, - unused: string, - url?: string | URL | null - ): void { + history[method] = function (state: unknown, unused: string, url?: string | URL | null): void { const prevPath = location.pathname; original(state, unused, url); const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; From 77d3c4a2e92f7d098c901322637463657bdb01ee Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 15:46:49 +0530 Subject: [PATCH 037/195] =?UTF-8?q?=5F=5FtsDivToSlotId=20now=20replaced=20?= =?UTF-8?q?per=20navigation=20(not=20merged)=20=E2=80=94=20stale=20div=5Fi?= =?UTF-8?q?d=20entries=20from=20destroyed=20slots=20no=20longer=20persist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../js/lib/src/integrations/gpt/index.test.ts | 138 ++++++++++++++++-- crates/js/lib/src/integrations/gpt/index.ts | 17 ++- 2 files changed, 138 insertions(+), 17 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 4d501ae34..87455591e 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -16,6 +16,7 @@ type TestWindow = Window & { __tsPrevGptSlots?: unknown; __tsServicesEnabled?: boolean; __tsSpaHookInstalled?: boolean; + __tsDivToSlotId?: Record; }; describe('installTsAdInit', () => { @@ -26,6 +27,7 @@ describe('installTsAdInit', () => { delete (window as TestWindow).__tsAdInit; delete (window as TestWindow).__tsPrevGptSlots; delete (window as TestWindow).__tsSpaHookInstalled; + delete (window as TestWindow).__tsDivToSlotId; (window as TestWindow).__tsServicesEnabled = false; // jsdom does not implement navigator.sendBeacon; polyfill it for tests if (!('sendBeacon' in navigator)) { @@ -41,7 +43,7 @@ describe('installTsAdInit', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('atf'), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue(['abc']), }; const mockPubads = { @@ -57,15 +59,15 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: { pos: 'atf' }, }, ]; (window as TestWindow).__ts_bids = { - atf: { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc', @@ -96,7 +98,7 @@ describe('installTsAdInit', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('atf'), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue(['abc']), }; const mockPubads = { @@ -114,15 +116,15 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: {}, }, ]; (window as TestWindow).__ts_bids = { - atf: { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc', @@ -143,6 +145,64 @@ describe('installTsAdInit', () => { beaconSpy.mockRestore(); }); + it('fires beacons for APS bid (no hb_adid) when ad renders in our slot', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).__ts_ad_slots = [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ]; + (window as TestWindow).__ts_bids = { + atf_sidebar_ad: { + hb_pb: '1.50', + hb_bidder: 'aps', + nurl: 'https://aps/win', + burl: 'https://aps/bill', + }, + }; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).__tsAdInit!(); + + expect(capturedListener).toBeDefined(); + capturedListener!({ isEmpty: false, slot: mockSlot }); + + expect(beaconSpy).toHaveBeenCalledWith('https://aps/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://aps/bill'); + + beaconSpy.mockClear(); + capturedListener!({ isEmpty: true, slot: mockSlot }); + expect(beaconSpy).not.toHaveBeenCalled(); + + beaconSpy.mockRestore(); + }); + it('does not fire nurl/burl when bid did not win GAM line item', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); let capturedListener: ((e: SlotRenderEvent) => void) | undefined; @@ -150,7 +210,7 @@ describe('installTsAdInit', () => { const mockSlotNoMatch = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('atf'), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), }; const mockPubads = { @@ -168,15 +228,15 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: {}, }, ]; (window as TestWindow).__ts_bids = { - atf: { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc', @@ -194,6 +254,56 @@ describe('installTsAdInit', () => { beaconSpy.mockRestore(); }); + it('does not fire beacons for slotRenderEnded on slots not owned by TS', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue(['abc']), + }; + const arenaSlot = { + getSlotElementId: () => 'arena-owned-div', + getTargeting: () => [], + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).__ts_ad_slots = [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ]; + (window as TestWindow).__ts_bids = { + atf_sidebar_ad: { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'abc' }, + }; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).__tsAdInit!(); + + capturedListener!({ isEmpty: false, slot: arenaSlot }); + + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + it('calls refresh even when __ts_bids is empty (graceful fallback)', async () => { const mockPubads = { enableSingleRequest: vi.fn(), @@ -211,9 +321,9 @@ describe('installTsAdInit', () => { }; (window as TestWindow).__ts_ad_slots = [ { - id: 'atf', + id: 'atf_sidebar_ad', gam_unit_path: '/123/atf', - div_id: 'atf', + div_id: 'div-atf-sidebar', formats: [[300, 250]], targeting: {}, }, diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index bf9fc99de..fee79c1b6 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -205,6 +205,7 @@ type TsWindow = Window & { __tsAdInit?: () => void; __tsPrevGptSlots?: GoogleTagSlot[]; __tsServicesEnabled?: boolean; + __tsDivToSlotId?: Record; }; /** @@ -235,6 +236,7 @@ export function installTsAdInit(): void { } const newSlots: GoogleTagSlot[] = []; + const divToSlotId: Record = {}; slots.forEach((slot) => { const gptSlot = g.defineSlot?.( @@ -250,10 +252,13 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, bid[key]!); }); gptSlot.setTargeting('ts_initial', '1'); + divToSlotId[slot.div_id] = slot.id; newSlots.push(gptSlot); }); w.__tsPrevGptSlots = newSlots; + // Replace (not merge) so destroyed slots from previous navigation don't linger. + w.__tsDivToSlotId = divToSlotId; // enableSingleRequest and enableServices must only be called once per page load. if (!w.__tsServicesEnabled) { @@ -262,12 +267,18 @@ export function installTsAdInit(): void { w.__tsServicesEnabled = true; g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { - const slotId: string = event.slot?.getSlotElementId?.() ?? ''; + const divId: string = event.slot?.getSlotElementId?.() ?? ''; + const slotId = (w.__tsDivToSlotId ?? {})[divId]; + if (!slotId) return; const bid = (w.__ts_bids ?? {})[slotId] ?? {}; + // Prebid: compare hb_adid targeting to verify the specific creative won. + // APS: no hb_adid equivalent — fires if bidder exists and slot is non-empty. + // Known limitation: APS path may over-fire if a non-APS line item wins. const ourBidWon = !event.isEmpty && - bid.hb_adid && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder); if (ourBidWon) { if (bid.nurl) navigator.sendBeacon(bid.nurl); if (bid.burl) navigator.sendBeacon(bid.burl); From 38c8bf17701dea1a76ba1344620f726a0a18711b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 17:20:49 +0530 Subject: [PATCH 038/195] Update timeout for mocktioneer --- trusted-server.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trusted-server.toml b/trusted-server.toml index 43e090fea..d17e86479 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -172,7 +172,7 @@ allowed_context_keys = ["permutive_segments"] enabled = true pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" -timeout_ms = 1000 +timeout_ms = 400 [integrations.google_tag_manager] enabled = false @@ -182,7 +182,7 @@ container_id = "GTM-XXXXXX" [integrations.adserver_mock] enabled = true endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" -timeout_ms = 1000 +timeout_ms = 400 # Map auction-request context keys to mediation URL query parameters. # Each key is a context key from the JS client; the value becomes the @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" -auction_timeout_ms = 1500 +auction_timeout_ms = 500 price_granularity = "dense" From e32bfa556e99d1fb225876c286e2fb7164317c9f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 7 May 2026 17:28:59 +0530 Subject: [PATCH 039/195] Revert with updated tiomeout --- trusted-server.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trusted-server.toml b/trusted-server.toml index d17e86479..43e090fea 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -172,7 +172,7 @@ allowed_context_keys = ["permutive_segments"] enabled = true pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" -timeout_ms = 400 +timeout_ms = 1000 [integrations.google_tag_manager] enabled = false @@ -182,7 +182,7 @@ container_id = "GTM-XXXXXX" [integrations.adserver_mock] enabled = true endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" -timeout_ms = 400 +timeout_ms = 1000 # Map auction-request context keys to mediation URL query parameters. # Each key is a context key from the JS client; the value becomes the @@ -192,6 +192,6 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" -auction_timeout_ms = 500 +auction_timeout_ms = 1500 price_granularity = "dense" From b1e74c986ec44f78053d4ef2dbadfc34697bb824 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 9 May 2026 14:45:34 +0530 Subject: [PATCH 040/195] Wip: Align with the spec --- .../trusted-server-adapter-fastly/src/main.rs | 18 +- .../src/auction/orchestrator.rs | 338 ++++++++++++++++++ .../src/creative_opportunities.rs | 16 +- .../trusted-server-core/src/html_processor.rs | 47 ++- crates/trusted-server-core/src/publisher.rs | 329 +++++++++++++++-- trusted-server.toml | 6 + 6 files changed, 710 insertions(+), 44 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 55af1468e..895299f54 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -19,7 +19,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body_async, PublisherResponse, }; use trusted_server_core::request_signing::{ @@ -250,18 +250,26 @@ async fn route_request( Ok(PublisherResponse::Stream { mut response, body, - params, + mut params, }) => { // Streaming path: finalize headers, then stream body to client. + // TTFB happens at stream_to_client() — SSP bids are already + // in-flight in Fastly's native layer (dispatched before origin wait). finalize_response(settings, geo_info.as_ref(), &mut response); let mut streaming_body = response.stream_to_client(); - if let Err(e) = stream_publisher_body( + // stream_publisher_body_async falls back to the sync path + // when no auction was dispatched (dispatched_auction is None). + let stream_result = stream_publisher_body_async( body, &mut streaming_body, - ¶ms, + &mut *params, settings, integration_registry, - ) { + orchestrator, + &publisher_services, + ) + .await; + if let Err(e) = stream_result { // Headers already committed. Log and abort — client // sees a truncated response. Standard proxy behavior. log::error!("Streaming processing failed: {e:?}"); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 0a52b07c8..953ed6a04 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -13,6 +13,23 @@ use super::config::AuctionConfig; use super::provider::AuctionProvider; use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +/// In-flight auction requests dispatched to SSP backends. +/// +/// Created by [`AuctionOrchestrator::dispatch_auction`] and consumed by +/// [`AuctionOrchestrator::collect_dispatched_auction`]. Carrying this handle +/// across `pending_origin.wait()` lets origin response and SSP HTTP requests +/// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than +/// TTFB ≈ auction timeout. +pub struct DispatchedAuction { + pending_requests: Vec, + backend_to_provider: HashMap)>, + auction_start: Instant, + timeout_ms: u32, + floor_prices: HashMap, + /// Carried so the mediator call in collect can pass it as the auction request. + request: AuctionRequest, +} + /// Compute the remaining time budget from a deadline. /// /// Returns the number of milliseconds left before `timeout_ms` is exceeded, @@ -584,6 +601,327 @@ impl AuctionOrchestrator { }) } + /// Dispatch SSP bid requests without blocking WASM. + /// + /// Calls each enabled provider's [`AuctionProvider::request_bids`] (which + /// internally calls Fastly's `send_async`), then returns immediately with a + /// [`DispatchedAuction`] token. The Fastly host begins the SSP round-trips + /// while WASM continues to `pending_origin.wait()`. + /// + /// Returns `None` when no providers are configured or all providers are + /// disabled / over budget. The caller should fall back to the synchronous + /// `run_auction` path. + #[must_use] + pub fn dispatch_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Option { + let provider_names = self.config.provider_names(); + if provider_names.is_empty() { + return None; + } + + let auction_start = Instant::now(); + let mut backend_to_provider: HashMap)> = + HashMap::new(); + let mut pending_requests: Vec = Vec::new(); + + for provider_name in provider_names { + let provider = match self.providers.get(provider_name) { + Some(p) => p, + None => { + log::warn!("Provider '{}' not registered, skipping", provider_name); + continue; + } + }; + + if !provider.is_enabled() { + log::debug!("Provider '{}' is disabled, skipping", provider.provider_name()); + continue; + } + + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let effective_timeout = remaining_ms.min(provider.timeout_ms()); + + if effective_timeout == 0 { + log::warn!( + "Auction timeout ({}ms) exhausted before launching '{}' — skipping", + context.timeout_ms, + provider.provider_name() + ); + continue; + } + + let backend_name = match provider.backend_name(effective_timeout) { + Some(name) => name, + None => { + log::warn!("Provider '{}' has no backend_name, skipping", provider.provider_name()); + continue; + } + }; + + let provider_context = AuctionContext { + settings: context.settings, + request: context.request, + client_info: context.client_info, + timeout_ms: effective_timeout, + provider_responses: context.provider_responses, + services: context.services, + }; + + let start_time = Instant::now(); + match provider.request_bids(request, &provider_context) { + Ok(pending) => { + log::info!( + "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", + provider.provider_name(), + backend_name, + effective_timeout + ); + backend_to_provider.insert( + backend_name.clone(), + (provider.provider_name().to_string(), start_time, Arc::clone(provider)), + ); + pending_requests + .push(PlatformPendingRequest::new(pending).with_backend_name(backend_name)); + } + Err(e) => { + log::warn!( + "Provider '{}' failed to dispatch request: {:?}", + provider.provider_name(), + e + ); + } + } + } + + if pending_requests.is_empty() { + return None; + } + + log::info!( + "Dispatched {} SSP requests (timeout: {}ms); Fastly host will race them against origin", + pending_requests.len(), + context.timeout_ms + ); + + Some(DispatchedAuction { + pending_requests, + backend_to_provider, + auction_start, + timeout_ms: context.timeout_ms, + floor_prices: self.floor_prices_by_slot(request), + request: request.clone(), + }) + } + + /// Collect bid responses from a previously-dispatched auction. + /// + /// Runs the select-loop phase (equivalent to Phase 2 of + /// `run_providers_parallel`) and, if the orchestrator has a mediator + /// configured, forwards collected bids to it. The overall auction deadline + /// is enforced from `dispatched.auction_start`. + /// + /// On any error or partial failure the method returns the best available + /// result rather than propagating — the caller should still inject the + /// winning bids even if some providers timed out. + pub async fn collect_dispatched_auction( + &self, + dispatched: DispatchedAuction, + services: &RuntimeServices, + context: &AuctionContext<'_>, + ) -> OrchestrationResult { + let DispatchedAuction { + pending_requests, + mut backend_to_provider, + auction_start, + timeout_ms, + floor_prices, + request, + } = dispatched; + + let deadline = Duration::from_millis(u64::from(timeout_ms)); + + log::info!( + "Collecting {} in-flight SSP responses (timeout: {}ms remaining: {}ms)", + pending_requests.len(), + timeout_ms, + remaining_budget_ms(auction_start, timeout_ms), + ); + + let mut responses: Vec = Vec::new(); + let mut remaining = pending_requests; + + while !remaining.is_empty() { + let select_result = match services + .http_client() + .select(remaining) + .await + .change_context(TrustedServerError::Auction { + message: "HTTP select failed".to_string(), + }) { + Ok(r) => r, + Err(e) => { + log::warn!("select() failed during auction collection: {:?}", e); + break; + } + }; + remaining = select_result.remaining; + + match select_result.ready { + Ok(platform_response) => { + let backend_name = platform_response.backend_name.clone().unwrap_or_default(); + if let Some((provider_name, start_time, provider)) = + backend_to_provider.remove(&backend_name) + { + let response_time_ms = start_time.elapsed().as_millis() as u64; + match platform_response_to_fastly(platform_response) { + Ok(response) => match provider.parse_response(response, response_time_ms) { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); + } + Err(e) => { + log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); + responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + } + }, + Err(e) => { + log::warn!("Provider '{}' unsupported body: {:?}", provider_name, e); + responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + } + } + } else { + log::warn!("Received response from unknown backend '{}', ignoring", backend_name); + } + } + Err(e) => { + log::warn!("A provider request failed during collection: {:?}", e); + } + } + + if auction_start.elapsed() >= deadline && !remaining.is_empty() { + log::warn!( + "Auction timeout ({}ms) reached, dropping {} remaining request(s)", + timeout_ms, + remaining.len() + ); + break; + } + } + + let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { + match self.providers.get(mediator_name.as_str()) { + Some(mediator) => { + let remaining_ms = remaining_budget_ms(auction_start, timeout_ms); + if remaining_ms == 0 { + log::warn!("Auction timeout exhausted during bidding — skipping mediator"); + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }; + } + let placeholder = fastly::Request::get("https://placeholder.invalid/"); + let mediator_context = AuctionContext { + settings: context.settings, + request: &placeholder, + client_info: context.client_info, + timeout_ms: remaining_ms, + provider_responses: Some(&responses), + services: context.services, + }; + match mediator.request_bids(&request, &mediator_context) { + Ok(pending) => { + let platform_resp = services + .http_client() + .wait(PlatformPendingRequest::new(pending)) + .await; + match platform_resp.change_context(TrustedServerError::Auction { + message: format!("Mediator {} request failed", mediator.provider_name()), + }) { + Ok(platform_resp) => { + match platform_response_to_fastly(platform_resp).change_context( + TrustedServerError::Auction { + message: format!("Mediator {} unsupported body", mediator.provider_name()), + }, + ) { + Ok(response) => { + let response_time_ms = + remaining_ms as u64 - remaining_budget_ms(auction_start, timeout_ms) as u64; + match mediator.parse_response(response, response_time_ms) { + Ok(mediator_resp) => { + let winning = mediator_resp + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + mediator.provider_name(), + bid.slot_id + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + let winning = self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_resp), winning) + } + Err(e) => { + log::warn!("Mediator '{}' parse failed: {:?}", mediator.provider_name(), e); + let winning = self.select_winning_bids(&responses, &floor_prices); + (None, winning) + } + } + } + Err(e) => { + log::warn!("Mediator body error: {:?}", e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + Err(e) => { + log::warn!("Mediator request failed: {:?}", e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + Err(e) => { + log::warn!("Mediator '{}' failed to dispatch: {:?}", mediator.provider_name(), e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + None => { + log::warn!("Mediator '{}' not registered", mediator_name); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } else { + (None, self.select_winning_bids(&responses, &floor_prices)) + }; + + OrchestrationResult { + provider_responses: responses, + mediator_response, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + } + } + /// Check if orchestrator is enabled. #[must_use] pub fn is_enabled(&self) -> bool { diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 7a4a10df5..12957d4b8 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -18,7 +18,21 @@ use crate::price_bucket::PriceGranularity; pub struct CreativeOpportunitiesConfig { /// GAM network ID used to build default unit paths. pub gam_network_id: String, - /// Auction timeout in milliseconds. + /// Maximum time in milliseconds to wait for the server-side auction before + /// closing the response body. + /// + /// The auction runs concurrently with HTML body streaming. Body content + /// above `` has already been delivered and painted before the hold + /// begins, so **FCP is not affected**. What this timeout bounds is the slip + /// on `DOMContentLoaded` and `window.load`: third-party scripts that hook + /// those events fire later by at most this duration. + /// + /// The worst case is a cache-hit page where the origin drains in <50 ms + /// but the auction takes the full timeout — the browser sits idle waiting + /// for ``. 500 ms is the recommended default and the hard upper + /// bound on DCL slip the publisher is willing to accept. + /// + /// When absent, falls back to `[auction].timeout_ms` from global config. #[serde(default)] pub auction_timeout_ms: Option, /// Price granularity for header-bidding price bucketing. diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 86a8abe79..26978cef5 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -292,13 +292,21 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), - // Inject __ts_bids before via end_tag_handlers. + // Inject __ts_bids before via end_tag_handlers — only when + // slots matched this URL. When no slots matched, skip injection entirely + // so the publisher's existing client-side Prebid/GPT flow is unmodified + // (dual-mode rollout: calling __tsAdInit with empty slots would invoke + // enableSingleRequest/enableServices and conflict with the publisher's GPT init). // Guard with AtomicBool so the script is only injected once even if // the origin HTML contains multiple elements (e.g. template fragments). element!("body", { let state = ad_bids_state.clone(); let injected_bids = injected_bids.clone(); + let has_slots = ad_slots_script.is_some(); move |el| { + if !has_slots { + return Ok(()); + } let state = state.clone(); let injected_bids = injected_bids.clone(); if let Some(handlers) = el.end_tag_handlers() { @@ -1285,7 +1293,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: None, + ad_slots_script: Some("".to_string()), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1314,7 +1322,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: None, + ad_slots_script: Some("".to_string()), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1331,14 +1339,16 @@ mod tests { } #[test] - fn injects_empty_ts_bids_when_state_is_none() { + fn injects_empty_ts_bids_when_slots_matched_but_auction_returned_nothing() { + // Slots matched (ad_slots_script is Some) but auction task never wrote a result + // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::RwLock::new(None)); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: None, + ad_slots_script: Some("".to_string()), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1348,7 +1358,32 @@ mod tests { let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( html.contains("__ts_bids=JSON.parse(\"{}\")"), - "should inject empty bids on None state" + "should inject empty bids fallback when auction produced nothing" + ); + } + + #[test] + fn does_not_inject_ts_bids_when_no_slots_matched() { + // No slots matched this URL — ad_slots_script is None. __ts_bids must be + // omitted entirely so the publisher's existing client-side GPT flow is + // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + !html.contains("__ts_bids"), + "should NOT inject __ts_bids when no slots matched" ); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ec4f4a227..4a39c9623 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -18,7 +18,7 @@ use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; -use crate::auction::orchestrator::AuctionOrchestrator; +use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo, }; @@ -31,7 +31,7 @@ use crate::error::TrustedServerError; use crate::http_util::{serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; use crate::platform::RuntimeServices; -use crate::price_bucket::price_bucket; +use crate::price_bucket::{price_bucket, PriceGranularity}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; @@ -301,8 +301,9 @@ pub enum PublisherResponse { response: Response, /// Origin body to be piped through the streaming pipeline. body: Body, - /// Parameters for `process_response_streaming`. - params: OwnedProcessResponseParams, + /// Parameters for `process_response_streaming`. Boxed to keep this + /// variant's on-stack size comparable to the other variants. + params: Box, }, /// Non-processable 2xx response (images, fonts, video). The adapter must /// reattach the body via `response.set_body(body)` before returning. @@ -407,6 +408,12 @@ pub struct OwnedProcessResponseParams { pub(crate) content_type: String, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: Arc>>, + /// In-flight SSP bids dispatched before `pending_origin.wait()`. + /// The streaming phase collects these and writes bids to `ad_bids_state` + /// before processing the last body chunk, so `` injection sees live bids. + pub(crate) dispatched_auction: Option, + /// Price granularity used to bucket bids when building `__ts_bids`. + pub(crate) price_granularity: PriceGranularity, } /// Stream the publisher response body through the processing pipeline. @@ -441,6 +448,261 @@ pub fn stream_publisher_body( process_response_streaming(body, output, &borrowed) } +/// Stream publisher body with a "last-chunk hold" for live bid injection. +/// +/// Drives the origin body through the HTML pipeline one chunk at a time, using a +/// one-behind buffer so the last raw origin chunk is held back. When the origin +/// body is exhausted (`read` returns `Ok(0)`): +/// +/// 1. [`collect_dispatched_auction`](AuctionOrchestrator::collect_dispatched_auction) +/// is awaited with the remaining deadline. +/// 2. Winning bids are written to `ad_bids_state`. +/// 3. The held last chunk is fed through the pipeline — `lol_html` fires its +/// `` handler with bids now in state. +/// +/// For non-HTML content types the auction is collected before any body bytes +/// are written (no `` to inject). If `params.dispatched_auction` is +/// `None` the function falls back to the synchronous +/// [`stream_publisher_body`] path. +/// +/// # Errors +/// +/// Returns an error if processing fails mid-stream. Headers are already +/// committed at that point; the caller logs and drops the `StreamingBody`. +pub async fn stream_publisher_body_async( + body: Body, + output: &mut W, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result<(), Report> { + let Some(dispatched) = params.dispatched_auction.take() else { + // No auction — use the existing sync pipeline unchanged. + return stream_publisher_body(body, output, params, settings, integration_registry); + }; + + let is_html = params.content_type.contains("text/html"); + + if !is_html { + // Non-HTML: collect auction first, then stream. There is no + // to hold, so delaying the entire body until collection is acceptable. + let placeholder = Request::get("https://placeholder.invalid/"); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &make_collect_context(settings, services, &placeholder)) + .await; + write_bids_to_state(&result.winning_bids, params.price_granularity, ¶ms.ad_bids_state); + return stream_publisher_body(body, output, params, settings, integration_registry); + } + + // HTML: build the processor once and drive it chunk by chunk. + // One-behind buffer: stream chunk N-1 immediately; hold chunk N until origin + // EOF, then await auction and process chunk N (which contains ). + let mut processor = create_html_stream_processor( + ¶ms.origin_host, + ¶ms.request_host, + ¶ms.request_scheme, + settings, + integration_registry, + params.ad_slots_script.as_deref().map(str::to_string), + params.ad_bids_state.clone(), + )?; + + let compression = Compression::from_content_encoding(¶ms.content_encoding); + stream_html_with_auction_hold( + body, + output, + &mut processor, + compression, + AuctionCollectCtx { + dispatched, + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator, + services, + settings, + }, + ) + .await +} + +/// Build a minimal [`AuctionContext`] for the mediator call in collection. +/// +/// The `request` field is a short-lived placeholder (providers use it only for +/// header extraction; the placeholder is functionally equivalent to the original +/// since `req` was already consumed by `send_async` before dispatch). +fn make_collect_context<'a>( + settings: &'a Settings, + services: &'a RuntimeServices, + placeholder: &'a Request, +) -> AuctionContext<'a> { + AuctionContext { + settings, + request: placeholder, + client_info: services.client_info(), + timeout_ms: 0, + provider_responses: None, + services, + } +} + +/// Write winning bids from an auction result into the shared `ad_bids_state` lock. +pub(crate) fn write_bids_to_state( + winning_bids: &std::collections::HashMap, + price_granularity: PriceGranularity, + ad_bids_state: &Arc>>, +) { + let bid_map = build_bid_map(winning_bids, price_granularity); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); +} + +/// Bundles the auction-collection dependencies passed through the streaming helpers. +struct AuctionCollectCtx<'a> { + dispatched: DispatchedAuction, + price_granularity: PriceGranularity, + ad_bids_state: &'a Arc>>, + orchestrator: &'a AuctionOrchestrator, + services: &'a RuntimeServices, + settings: &'a Settings, +} + +/// Run the one-behind chunk loop for HTML bodies, collecting the auction before +/// the last chunk so `lol_html`'s `` handler sees live bids. +async fn stream_html_with_auction_hold( + body: Body, + output: &mut W, + processor: &mut P, + compression: Compression, + ctx: AuctionCollectCtx<'_>, +) -> Result<(), Report> { + use brotli::enc::writer::CompressorWriter; + use brotli::enc::BrotliEncoderParams; + use brotli::Decompressor; + use flate2::read::{GzDecoder, ZlibDecoder}; + use flate2::write::{GzEncoder, ZlibEncoder}; + + match compression { + Compression::None => one_behind_loop(body, output, processor, ctx).await, + Compression::Gzip => { + let decoder = GzDecoder::new(body); + let mut encoder = GzEncoder::new(&mut *output, flate2::Compression::default()); + one_behind_loop(decoder, &mut encoder, processor, ctx).await?; + encoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize gzip encoder".to_string(), + })?; + Ok(()) + } + Compression::Deflate => { + let decoder = ZlibDecoder::new(body); + let mut encoder = ZlibEncoder::new(&mut *output, flate2::Compression::default()); + one_behind_loop(decoder, &mut encoder, processor, ctx).await?; + encoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize deflate encoder".to_string(), + })?; + Ok(()) + } + Compression::Brotli => { + let decoder = Decompressor::new(body, 4096); + let params = BrotliEncoderParams { + quality: 4, + lgwin: 22, + ..Default::default() + }; + let mut encoder = CompressorWriter::with_params(&mut *output, 4096, ¶ms); + one_behind_loop(decoder, &mut encoder, processor, ctx).await?; + let _ = encoder.into_inner(); + Ok(()) + } + } +} + +/// Core one-behind chunk loop. +/// +/// Reads from `reader`, writing processed output to `writer` for every chunk +/// except the current one (which is held pending). On EOF, the auction is +/// collected, bids written, and the held chunk processed last. +async fn one_behind_loop( + mut reader: R, + writer: &mut W, + processor: &mut P, + ctx: AuctionCollectCtx<'_>, +) -> Result<(), Report> { + let AuctionCollectCtx { dispatched, price_granularity, ad_bids_state, orchestrator, services, settings } = ctx; + const CHUNK_SIZE: usize = 8192; + let mut buffer = vec![0u8; CHUNK_SIZE]; + let mut pending: Vec = Vec::new(); + + loop { + match reader.read(&mut buffer) { + Ok(0) => { + // Origin exhausted — pending holds the last chunk. + // Collect the auction before feeding it to lol_html so that + // the handler sees populated ad_bids_state. + let placeholder = Request::get("https://placeholder.invalid/"); + let collect_ctx = make_collect_context(settings, services, &placeholder); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &collect_ctx) + .await; + write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); + + // Process the held last chunk (not is_last — finalization is separate). + if !pending.is_empty() { + let out = processor.process_chunk(&pending, false).change_context( + TrustedServerError::Proxy { + message: "Failed to process last chunk".to_string(), + }, + )?; + if !out.is_empty() { + writer.write_all(&out).change_context(TrustedServerError::Proxy { + message: "Failed to write last chunk".to_string(), + })?; + } + } + // Signal EOF to lol_html (fires end() which flushes remaining state). + let final_out = processor.process_chunk(&[], true).change_context( + TrustedServerError::Proxy { + message: "Failed to finalize processor".to_string(), + }, + )?; + if !final_out.is_empty() { + writer.write_all(&final_out).change_context(TrustedServerError::Proxy { + message: "Failed to write finalized output".to_string(), + })?; + } + break; + } + Ok(n) => { + // Stream the previously held chunk (it is not the last). + if !pending.is_empty() { + let out = processor.process_chunk(&pending, false).change_context( + TrustedServerError::Proxy { + message: "Failed to process chunk".to_string(), + }, + )?; + if !out.is_empty() { + writer.write_all(&out).change_context(TrustedServerError::Proxy { + message: "Failed to write chunk".to_string(), + })?; + } + } + pending = buffer[..n].to_vec(); + } + Err(e) => { + return Err(Report::new(TrustedServerError::Proxy { + message: format!("Failed to read origin body: {e}"), + })); + } + } + } + + writer.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush output".to_string(), + })?; + Ok(()) +} + /// Proxies requests to the publisher's origin server. /// /// Returns a [`PublisherResponse`] indicating how the response should be sent: @@ -590,13 +852,22 @@ pub async fn handle_publisher_request( restrict_accept_encoding(&mut req); req.set_header("host", &origin_host); + // Dispatch origin request first. let pending_origin = req.send_async(&backend_name) .change_context(TrustedServerError::Proxy { message: "Failed to dispatch async origin request".to_string(), })?; - let auction_result = if should_run_auction { + // Dispatch SSP bid requests BEFORE awaiting origin — all HTTP is now in-flight + // in Fastly's native layer. WASM yields only for origin (fast, cache-hit path), + // so TTFB ≈ origin latency instead of TTFB ≈ auction timeout. + let price_granularity = settings + .creative_opportunities + .as_ref() + .map(|co| co.price_granularity) + .unwrap_or_default(); + let dispatched_auction = if should_run_auction { let co_config = settings .creative_opportunities .as_ref() @@ -617,35 +888,12 @@ pub async fn handle_publisher_request( provider_responses: None, services, }; - match orchestrator - .run_auction(&auction_request, &auction_context, services) - .await - { - Ok(result) => Some(result), - Err(e) => { - log::warn!("server-side auction failed, proceeding without bids: {e:?}"); - None - } - } + orchestrator.dispatch_auction(&auction_request, &auction_context) } else { None }; - if should_run_auction { - let co_config = settings - .creative_opportunities - .as_ref() - .expect("should be present"); - let empty: std::collections::HashMap = std::collections::HashMap::new(); - let winning_bids = auction_result - .as_ref() - .map(|r| &r.winning_bids) - .unwrap_or(&empty); - let bid_map = build_bid_map(winning_bids, co_config.price_granularity); - let bids_script = build_bids_script(&bid_map); - *ad_bids_state.write().expect("should write bid state") = Some(bids_script); - } - + // Now yield for origin — SSP requests are already racing in Fastly's native layer. let mut response = pending_origin .wait() .change_context(TrustedServerError::Proxy { @@ -754,7 +1002,7 @@ pub async fn handle_publisher_request( Ok(PublisherResponse::Stream { response, body, - params: OwnedProcessResponseParams { + params: Box::new(OwnedProcessResponseParams { content_encoding, origin_host, origin_url: settings.publisher.origin_url.clone(), @@ -763,7 +1011,9 @@ pub async fn handle_publisher_request( content_type, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), - }, + dispatched_auction, + price_granularity, + }), }) } ResponseRoute::BufferedProcessed => { @@ -1790,6 +2040,9 @@ mod tests { content_type: "text/css".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); @@ -1833,6 +2086,9 @@ mod tests { content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); @@ -1867,6 +2123,9 @@ mod tests { content_type: "text/html".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let bogus_body = Body::from(b"not gzip".to_vec()); @@ -1968,6 +2227,9 @@ mod tests { content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -2020,6 +2282,9 @@ mod tests { content_type: "text/html".to_string(), ad_slots_script: None, ad_bids_state: Arc::new(RwLock::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; let mut output = Vec::new(); diff --git a/trusted-server.toml b/trusted-server.toml index 43e090fea..b1b5a0b03 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -192,6 +192,12 @@ permutive_segments = "permutive" [creative_opportunities] gam_network_id = "88059007" +# FCP is not affected by this value — body content above has already +# streamed and painted before the hold begins. What this caps is the slip on +# DOMContentLoaded and window.load. Worst case: a cache-hit page where origin +# drains in <50 ms but the auction runs to the limit. 500 ms is the recommended +# default; raise only if your SSPs need more headroom and your analytics confirm +# the DCL slip is acceptable. auction_timeout_ms = 1500 price_granularity = "dense" From a2d08e78ea7c4bc3fd73c4259d3cb4b66f37a153 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 16:13:47 +0530 Subject: [PATCH 041/195] Fix clippy explicit-auto-deref in stream_publisher_body_async call --- crates/trusted-server-adapter-fastly/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 895299f54..94f095193 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -262,7 +262,7 @@ async fn route_request( let stream_result = stream_publisher_body_async( body, &mut streaming_body, - &mut *params, + &mut params, settings, integration_registry, orchestrator, From b03af6b1f94b4bb34b8a59db8b28b69a747287ca Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 16:34:20 +0530 Subject: [PATCH 042/195] =?UTF-8?q?Fix=20Cache-Control=20headers=20applied?= =?UTF-8?q?=20only=20when=20slots=20matched=20=E2=80=94=20apply=20to=20all?= =?UTF-8?q?=20HTML=20responses=20per=20spec=20=C2=A74.7=20+=20=C2=A78?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/trusted-server-core/src/publisher.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4a39c9623..dc93af768 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -915,7 +915,13 @@ pub async fn handle_publisher_request( None }; - if ad_slots_script.is_some() { + // §4.7: assembled HTML responses must never be shared-cached — per-user bid data + // travels inline. Apply regardless of slot match or auction outcome (§8). + let origin_content_type = response + .get_header(header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .unwrap_or_default(); + if origin_content_type.contains("text/html") { response.set_header(header::CACHE_CONTROL, "private, max-age=0"); response.remove_header("surrogate-control"); response.remove_header("fastly-surrogate-control"); From 349dcdcb2689d039085e1267f0011f8e94f00b2a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 16:45:54 +0530 Subject: [PATCH 043/195] cargo fmt --- .../src/auction/orchestrator.rs | 111 +++++++++++++----- crates/trusted-server-core/src/publisher.rs | 50 +++++--- 2 files changed, 114 insertions(+), 47 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 953ed6a04..820031050 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -637,7 +637,10 @@ impl AuctionOrchestrator { }; if !provider.is_enabled() { - log::debug!("Provider '{}' is disabled, skipping", provider.provider_name()); + log::debug!( + "Provider '{}' is disabled, skipping", + provider.provider_name() + ); continue; } @@ -656,7 +659,10 @@ impl AuctionOrchestrator { let backend_name = match provider.backend_name(effective_timeout) { Some(name) => name, None => { - log::warn!("Provider '{}' has no backend_name, skipping", provider.provider_name()); + log::warn!( + "Provider '{}' has no backend_name, skipping", + provider.provider_name() + ); continue; } }; @@ -681,7 +687,11 @@ impl AuctionOrchestrator { ); backend_to_provider.insert( backend_name.clone(), - (provider.provider_name().to_string(), start_time, Arc::clone(provider)), + ( + provider.provider_name().to_string(), + start_time, + Arc::clone(provider), + ), ); pending_requests .push(PlatformPendingRequest::new(pending).with_backend_name(backend_name)); @@ -777,28 +787,45 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; match platform_response_to_fastly(platform_response) { - Ok(response) => match provider.parse_response(response, response_time_ms) { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids ({}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); - responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + Ok(response) => { + match provider.parse_response(response, response_time_ms) { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); + } + Err(e) => { + log::warn!( + "Provider '{}' parse failed: {:?}", + provider_name, + e + ); + responses.push(AuctionResponse::error( + &provider_name, + response_time_ms, + )); + } } - }, + } Err(e) => { - log::warn!("Provider '{}' unsupported body: {:?}", provider_name, e); - responses.push(AuctionResponse::error(&provider_name, response_time_ms)); + log::warn!( + "Provider '{}' unsupported body: {:?}", + provider_name, + e + ); + responses + .push(AuctionResponse::error(&provider_name, response_time_ms)); } } } else { - log::warn!("Received response from unknown backend '{}', ignoring", backend_name); + log::warn!( + "Received response from unknown backend '{}', ignoring", + backend_name + ); } } Err(e) => { @@ -847,18 +874,27 @@ impl AuctionOrchestrator { .wait(PlatformPendingRequest::new(pending)) .await; match platform_resp.change_context(TrustedServerError::Auction { - message: format!("Mediator {} request failed", mediator.provider_name()), + message: format!( + "Mediator {} request failed", + mediator.provider_name() + ), }) { Ok(platform_resp) => { match platform_response_to_fastly(platform_resp).change_context( TrustedServerError::Auction { - message: format!("Mediator {} unsupported body", mediator.provider_name()), + message: format!( + "Mediator {} unsupported body", + mediator.provider_name() + ), }, ) { Ok(response) => { - let response_time_ms = - remaining_ms as u64 - remaining_budget_ms(auction_start, timeout_ms) as u64; - match mediator.parse_response(response, response_time_ms) { + let response_time_ms = remaining_ms as u64 + - remaining_budget_ms(auction_start, timeout_ms) + as u64; + match mediator + .parse_response(response, response_time_ms) + { Ok(mediator_resp) => { let winning = mediator_resp .bids @@ -876,19 +912,30 @@ impl AuctionOrchestrator { } }) .collect(); - let winning = self.apply_floor_prices(winning, &floor_prices); + let winning = self + .apply_floor_prices(winning, &floor_prices); (Some(mediator_resp), winning) } Err(e) => { - log::warn!("Mediator '{}' parse failed: {:?}", mediator.provider_name(), e); - let winning = self.select_winning_bids(&responses, &floor_prices); + log::warn!( + "Mediator '{}' parse failed: {:?}", + mediator.provider_name(), + e + ); + let winning = self.select_winning_bids( + &responses, + &floor_prices, + ); (None, winning) } } } Err(e) => { log::warn!("Mediator body error: {:?}", e); - (None, self.select_winning_bids(&responses, &floor_prices)) + ( + None, + self.select_winning_bids(&responses, &floor_prices), + ) } } } @@ -899,7 +946,11 @@ impl AuctionOrchestrator { } } Err(e) => { - log::warn!("Mediator '{}' failed to dispatch: {:?}", mediator.provider_name(), e); + log::warn!( + "Mediator '{}' failed to dispatch: {:?}", + mediator.provider_name(), + e + ); (None, self.select_winning_bids(&responses, &floor_prices)) } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index dc93af768..3a10e85f1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -490,9 +490,17 @@ pub async fn stream_publisher_body_async( // to hold, so delaying the entire body until collection is acceptable. let placeholder = Request::get("https://placeholder.invalid/"); let result = orchestrator - .collect_dispatched_auction(dispatched, services, &make_collect_context(settings, services, &placeholder)) + .collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + ) .await; - write_bids_to_state(&result.winning_bids, params.price_granularity, ¶ms.ad_bids_state); + write_bids_to_state( + &result.winning_bids, + params.price_granularity, + ¶ms.ad_bids_state, + ); return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -629,7 +637,14 @@ async fn one_behind_loop( processor: &mut P, ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { - let AuctionCollectCtx { dispatched, price_granularity, ad_bids_state, orchestrator, services, settings } = ctx; + let AuctionCollectCtx { + dispatched, + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + } = ctx; const CHUNK_SIZE: usize = 8192; let mut buffer = vec![0u8; CHUNK_SIZE]; let mut pending: Vec = Vec::new(); @@ -655,9 +670,11 @@ async fn one_behind_loop( }, )?; if !out.is_empty() { - writer.write_all(&out).change_context(TrustedServerError::Proxy { - message: "Failed to write last chunk".to_string(), - })?; + writer + .write_all(&out) + .change_context(TrustedServerError::Proxy { + message: "Failed to write last chunk".to_string(), + })?; } } // Signal EOF to lol_html (fires end() which flushes remaining state). @@ -667,9 +684,11 @@ async fn one_behind_loop( }, )?; if !final_out.is_empty() { - writer.write_all(&final_out).change_context(TrustedServerError::Proxy { - message: "Failed to write finalized output".to_string(), - })?; + writer + .write_all(&final_out) + .change_context(TrustedServerError::Proxy { + message: "Failed to write finalized output".to_string(), + })?; } break; } @@ -682,9 +701,11 @@ async fn one_behind_loop( }, )?; if !out.is_empty() { - writer.write_all(&out).change_context(TrustedServerError::Proxy { - message: "Failed to write chunk".to_string(), - })?; + writer + .write_all(&out) + .change_context(TrustedServerError::Proxy { + message: "Failed to write chunk".to_string(), + })?; } } pending = buffer[..n].to_vec(); @@ -2048,7 +2069,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); @@ -2094,7 +2114,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); @@ -2131,7 +2150,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let bogus_body = Body::from(b"not gzip".to_vec()); @@ -2235,7 +2253,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -2290,7 +2307,6 @@ mod tests { ad_bids_state: Arc::new(RwLock::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - }; let mut output = Vec::new(); From 78885f9c1f53007be16c1c18e480487c6e6a9fc3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 19:54:52 +0530 Subject: [PATCH 044/195] =?UTF-8?q?Fix=20auction=20consent=20gate=20blocki?= =?UTF-8?q?ng=20non-GDPR=20regions=20=E2=80=94=20only=20require=20TCF=20Pu?= =?UTF-8?q?rpose=201=20when=20gdpr=5Fapplies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/trusted-server-core/src/publisher.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 3a10e85f1..81637ae53 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -853,10 +853,13 @@ pub async fn handle_publisher_request( Vec::new() }; - let consent_allows_auction = consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Non-GDPR regions (US, etc.) have no TCF string — auction is freely allowed. + // GDPR regions require TCF Purpose 1 (storage/access) before firing. + let consent_allows_auction = !consent_context.gdpr_applies + || consent_context + .tcf + .as_ref() + .is_some_and(|tcf| tcf.has_purpose_consent(1)); let should_run_auction = is_get && !is_prefetch && !is_bot && !matched_slots.is_empty() && consent_allows_auction; @@ -1324,10 +1327,11 @@ pub async fn handle_page_bids( .map(|_| services.kv_store()), }); - let consent_allows_auction = consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + let consent_allows_auction = !consent_context.gdpr_applies + || consent_context + .tcf + .as_ref() + .is_some_and(|tcf| tcf.has_purpose_consent(1)); let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { let mut auction_request = build_auction_request( From 3783e68104258e71b8325820ac1c34026b928fc0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 10 May 2026 20:52:05 +0530 Subject: [PATCH 045/195] =?UTF-8?q?Fix=20SSP=20requests=20using=20placehol?= =?UTF-8?q?der=20headers=20=E2=80=94=20pass=20real=20request=20to=20dispat?= =?UTF-8?q?ch=5Fauction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dispatch_auction was building AuctionContext with a placeholder Request (GET https://placeholder.invalid/) that carried no headers. Prebid's request_bids copies User-Agent, x-forwarded-for, Referer, Accept-Language, and cookies from context.request before sending to Prebid Server, so SSPs received stripped requests and returned empty bids. Fix: dispatch SSP requests before req.send_async(), using the original request directly as AuctionContext.request. DispatchedAuction holds no lifetime reference to Request, so the borrow ends at return and req can be modified (restrict_accept_encoding, Host header) and sent to origin immediately after. --- crates/trusted-server-core/src/publisher.rs | 36 +++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 81637ae53..f5caa5710 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -872,25 +872,16 @@ pub async fn handle_publisher_request( let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); - // Only advertise encodings the rewrite pipeline can decode and re-encode. - restrict_accept_encoding(&mut req); - req.set_header("host", &origin_host); - - // Dispatch origin request first. - let pending_origin = - req.send_async(&backend_name) - .change_context(TrustedServerError::Proxy { - message: "Failed to dispatch async origin request".to_string(), - })?; - - // Dispatch SSP bid requests BEFORE awaiting origin — all HTTP is now in-flight - // in Fastly's native layer. WASM yields only for origin (fast, cache-hit path), - // so TTFB ≈ origin latency instead of TTFB ≈ auction timeout. let price_granularity = settings .creative_opportunities .as_ref() .map(|co| co.price_granularity) .unwrap_or_default(); + + // Dispatch SSP bid requests while req still has the original client headers + // (User-Agent, x-forwarded-for, cookies, etc.). The borrow ends when + // dispatch_auction returns — DispatchedAuction holds no lifetime — so req + // can be mutated and sent to origin immediately after. let dispatched_auction = if should_run_auction { let co_config = settings .creative_opportunities @@ -903,10 +894,9 @@ pub async fn handle_publisher_request( &request_info, co_config, ); - let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); let auction_context = AuctionContext { settings, - request: &placeholder_req, + request: &req, client_info: services.client_info(), timeout_ms: auction_timeout_ms, provider_responses: None, @@ -917,7 +907,19 @@ pub async fn handle_publisher_request( None }; - // Now yield for origin — SSP requests are already racing in Fastly's native layer. + // Only advertise encodings the rewrite pipeline can decode and re-encode. + restrict_accept_encoding(&mut req); + req.set_header("host", &origin_host); + + // Dispatch origin — SSP requests are already racing in Fastly's native layer. + // TTFB ≈ origin latency instead of TTFB ≈ auction timeout. + let pending_origin = + req.send_async(&backend_name) + .change_context(TrustedServerError::Proxy { + message: "Failed to dispatch async origin request".to_string(), + })?; + + // Now yield for origin. let mut response = pending_origin .wait() .change_context(TrustedServerError::Proxy { From 0c9465206f4e6b3ad865277dae63378090831a79 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 12:36:25 +0530 Subject: [PATCH 046/195] Fix async auction collect abandoning SSP bids when origin is slow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In collect_dispatched_auction, the select loop checked `auction_start.elapsed() >= deadline` after each SSP response and broke early if the 1500ms budget had elapsed. When origin TTFB + body download exceeded the auction budget, the check fired after collecting the first SSP response, abandoning the second SSP's already-buffered response. This left responses with only one (possibly errored) SSP, causing remaining_ms == 0 which skipped the mediator, and select_winning_bids on the partial set returned zero bids. The deadline break is wrong in this context: SSP HTTP connections are already bounded by the backend first_byte_timeout set at dispatch time (1000ms per provider). By the time collect is called at origin EOF, all SSPs have either responded or been errored by Fastly's host. The select() calls drain instantly — no WASM-level deadline enforcement is needed or safe. Also add info-level log statements at dispatch, collect, and write_bids_to_state to make the auction pipeline observable without requiring a dashboard. --- .../src/auction/orchestrator.rs | 10 --------- crates/trusted-server-core/src/publisher.rs | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 820031050..867bdf3a7 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -751,8 +751,6 @@ impl AuctionOrchestrator { request, } = dispatched; - let deadline = Duration::from_millis(u64::from(timeout_ms)); - log::info!( "Collecting {} in-flight SSP responses (timeout: {}ms remaining: {}ms)", pending_requests.len(), @@ -833,14 +831,6 @@ impl AuctionOrchestrator { } } - if auction_start.elapsed() >= deadline && !remaining.is_empty() { - log::warn!( - "Auction timeout ({}ms) reached, dropping {} remaining request(s)", - timeout_ms, - remaining.len() - ); - break; - } } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index f5caa5710..5284e20f3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -561,6 +561,15 @@ pub(crate) fn write_bids_to_state( price_granularity: PriceGranularity, ad_bids_state: &Arc>>, ) { + log::info!( + "write_bids_to_state: {} winning bid(s): [{}]", + winning_bids.len(), + winning_bids + .keys() + .cloned() + .collect::>() + .join(", ") + ); let bid_map = build_bid_map(winning_bids, price_granularity); let bids_script = build_bids_script(&bid_map); *ad_bids_state.write().expect("should write bid state") = Some(bids_script); @@ -655,11 +664,16 @@ async fn one_behind_loop( // Origin exhausted — pending holds the last chunk. // Collect the auction before feeding it to lol_html so that // the handler sees populated ad_bids_state. + log::info!("one_behind_loop: EOF — collecting dispatched auction"); let placeholder = Request::get("https://placeholder.invalid/"); let collect_ctx = make_collect_context(settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) .await; + log::info!( + "one_behind_loop: collect complete — {} winning bid(s)", + result.winning_bids.len() + ); write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); // Process the held last chunk (not is_last — finalization is separate). @@ -906,6 +920,14 @@ pub async fn handle_publisher_request( } else { None }; + log::info!( + "dispatch_auction: {}", + if dispatched_auction.is_some() { + "Some — auction running async" + } else { + "None — falling back to sync or skipped" + } + ); // Only advertise encodings the rewrite pipeline can decode and re-encode. restrict_accept_encoding(&mut req); From f172f4477c60ae0fc0e0d68b0d9f969652dc092f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 13:15:23 +0530 Subject: [PATCH 047/195] Cargo fmt --- crates/trusted-server-core/src/auction/orchestrator.rs | 1 - crates/trusted-server-core/src/publisher.rs | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 867bdf3a7..e9d8fa198 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -830,7 +830,6 @@ impl AuctionOrchestrator { log::warn!("A provider request failed during collection: {:?}", e); } } - } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 5284e20f3..3a9eb0895 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -564,11 +564,7 @@ pub(crate) fn write_bids_to_state( log::info!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), - winning_bids - .keys() - .cloned() - .collect::>() - .join(", ") + winning_bids.keys().cloned().collect::>().join(", ") ); let bid_map = build_bid_map(winning_bids, price_granularity); let bids_script = build_bids_script(&bid_map); From 14cd493ee907fbf2cce5030062bfed4ccac41043 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 13:32:28 +0530 Subject: [PATCH 048/195] Fix mediator always skipped when origin body exceeds SSP auction budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In collect_dispatched_auction, the mediator was skipped when remaining_budget_ms(auction_start, timeout_ms) == 0. In the async-dispatch path, auction_start is set before pending_origin.wait(), so elapsed time includes the full origin TTFB and body download. For heavy SSR pages (autoblog), this exceeds the 1500ms SSP budget, making remaining_ms == 0 at every collection and causing the mediator to be permanently skipped. The mediator (adserver_mock) is the primary bid source — SSPs alone return no bids. Skipping it means window.__ts_bids == {} on every full page load, while handle_page_bids (which uses the sequential run_auction path) works correctly because it measures remaining time from after SSP collection. Fix: give the mediator its own configured timeout (mediator.timeout_ms()) instead of the exhausted SSP budget. This mirrors how run_parallel_mediation works: the mediator's deadline is independent of SSP round-trip time. Side effect: mediator backend name is now stable (always t1000 for adserver_mock) rather than varying per request with remaining_ms. --- .../src/auction/orchestrator.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index e9d8fa198..892ff9ebb 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -835,24 +835,25 @@ impl AuctionOrchestrator { let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { match self.providers.get(mediator_name.as_str()) { Some(mediator) => { - let remaining_ms = remaining_budget_ms(auction_start, timeout_ms); - if remaining_ms == 0 { - log::warn!("Auction timeout exhausted during bidding — skipping mediator"); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }; - } + // Use the mediator's own configured timeout, not the remaining SSP + // budget. In the async-dispatch path, SSPs race against origin, so + // auction_start.elapsed() can exceed the SSP budget by the time the + // origin body finishes streaming. Skipping the mediator in that case + // would discard all bids — the mediator is the primary bid source. + let mediator_timeout = mediator.timeout_ms(); + let mediator_start = Instant::now(); + log::info!( + "Running mediator '{}' with {}ms budget (SSP budget remaining: {}ms)", + mediator.provider_name(), + mediator_timeout, + remaining_budget_ms(auction_start, timeout_ms), + ); let placeholder = fastly::Request::get("https://placeholder.invalid/"); let mediator_context = AuctionContext { settings: context.settings, request: &placeholder, client_info: context.client_info, - timeout_ms: remaining_ms, + timeout_ms: mediator_timeout, provider_responses: Some(&responses), services: context.services, }; @@ -878,9 +879,8 @@ impl AuctionOrchestrator { }, ) { Ok(response) => { - let response_time_ms = remaining_ms as u64 - - remaining_budget_ms(auction_start, timeout_ms) - as u64; + let response_time_ms = + mediator_start.elapsed().as_millis() as u64; match mediator .parse_response(response, response_time_ms) { From 6210ebbf1560558813c7d30865b032a5a7962649 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 14:04:28 +0530 Subject: [PATCH 049/195] Cargo fmt --- crates/trusted-server-core/src/publisher.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6fd9b0d6a..30abc5326 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1332,7 +1332,8 @@ pub async fn handle_page_bids( .collect(); let http_req = compat::from_fastly_headers_ref(&req); - let request_info = crate::http_util::RequestInfo::from_request(&http_req, &services.client_info); + let request_info = + crate::http_util::RequestInfo::from_request(&http_req, &services.client_info); let cookie_jar = handle_request_cookies(&http_req)?; let ec_id = get_or_generate_ec_id_from_http_request(settings, services, &http_req)?; let geo = services From 3cdf9952f54fcfe4c4c7cc87f8064aa1af8aa721 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 14:44:58 +0530 Subject: [PATCH 050/195] Adding debug info for auction --- crates/trusted-server-core/src/publisher.rs | 44 +++++++++++++++------ crates/trusted-server-core/src/settings.rs | 6 +++ trusted-server.toml | 6 ++- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 30abc5326..ba594d808 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -677,6 +677,29 @@ async fn one_behind_loop( ); write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); + if settings.debug.auction_html_comment { + let ssp_count = result.provider_responses.len(); + let mediator_info = match &result.mediator_response { + Some(r) => format!("ok({}_bids)", r.bids.len()), + None => "none".to_string(), + }; + let debug_comment = format!( + "", + result.winning_bids.len() + ); + let mut state = ad_bids_state + .write() + .expect("should write bid state for debug"); + match &mut *state { + Some(script) => { + *script = format!("{debug_comment}\n{script}"); + } + None => { + *state = Some(debug_comment); + } + } + } + // Process the held last chunk (not is_last — finalization is separate). if !pending.is_empty() { let out = processor.process_chunk(&pending, false).change_context( @@ -909,6 +932,7 @@ pub async fn handle_publisher_request( &ec_id, &consent_context, &request_info, + &request_path, co_config, ); let auction_context = AuctionContext { @@ -1109,18 +1133,23 @@ pub(crate) fn build_auction_request( ec_id: &str, consent_context: &crate::consent::ConsentContext, request_info: &crate::http_util::RequestInfo, + request_path: &str, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, ) -> AuctionRequest { let slots = matched_slots .iter() .map(|s| s.to_ad_slot(&co_config.gam_network_id)) .collect(); + let page_url = format!( + "{}://{}{}", + request_info.scheme, request_info.host, request_path + ); AuctionRequest { id: format!("ts-{}", ec_id), slots, publisher: PublisherInfo { domain: request_info.host.clone(), - page_url: None, + page_url: Some(page_url.clone()), }, user: UserInfo { id: ec_id.to_string(), @@ -1130,7 +1159,7 @@ pub(crate) fn build_auction_request( device: None, site: Some(SiteInfo { domain: request_info.host.clone(), - page: String::new(), + page: page_url, }), context: std::collections::HashMap::new(), } @@ -1363,21 +1392,14 @@ pub async fn handle_page_bids( .is_some_and(|tcf| tcf.has_purpose_consent(1)); let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { - let mut auction_request = build_auction_request( + let auction_request = build_auction_request( &matched_slots, &ec_id, &consent_context, &request_info, + &path_param, co_config, ); - let page_url = format!( - "{}://{}{}", - request_info.scheme, request_info.host, path_param - ); - auction_request.publisher.page_url = Some(page_url.clone()); - if let Some(ref mut site) = auction_request.site { - site.page = page_url; - } let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index e09c50e2d..386f0d54b 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -410,6 +410,12 @@ pub struct DebugConfig { /// Fastly-observed TLS details that browser JS cannot normally read. #[serde(default)] pub ja4_endpoint_enabled: bool, + + /// Inject a `` HTML comment before `` showing + /// auction pipeline stats (SSP count, mediator status, winning bid count). + /// Never enable in production — visible in page source. + #[serde(default)] + pub auction_html_comment: bool, } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] diff --git a/trusted-server.toml b/trusted-server.toml index 60876389f..a71abfdd7 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -208,7 +208,11 @@ endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" timeout_ms = 1000 # Debug configuration (all flags default to false — do not enable in production) -# [debug] +# TODO: remove [debug] block before merging to main +[debug] +# Inject before . +# Visible in page source. Disable after investigation. +auction_html_comment = true # Enable the JA4/TLS fingerprint debug endpoint at GET /_ts/debug/ja4. # Returns a plain-text response with the following fields (Fastly-observed values): # ja4 — JA4 TLS client fingerprint From e35c593f3b26f64ee148ed39a698b19c23b594db Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 15:30:15 +0530 Subject: [PATCH 051/195] Fix auction bids missing on Next.js buffered HTML path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `classify_response_route` returns `BufferedProcessed` when HTML has post-processors registered (e.g. the Next.js integration registers one via `with_html_post_processor`). Unlike the `Stream` path, which drives `one_behind_loop` to collect the dispatched auction at origin EOF, the `BufferedProcessed` branch previously discarded `dispatched_auction` entirely — so `ad_bids_state` stayed `None` and lol_html injected the fallback `window.__ts_bids = {}` instead of real bids. Fix: collect the in-flight dispatched auction in the `BufferedProcessed` branch before calling `process_response_streaming`, using the same `collect_dispatched_auction` + `write_bids_to_state` pattern that the stream path uses. The `debug.auction_html_comment` injection is mirrored here as well so the comment appears in both code paths when enabled. --- crates/trusted-server-core/src/publisher.rs | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ba594d808..0178d56d1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1103,6 +1103,49 @@ pub async fn handle_publisher_request( content_type, content_encoding, request_host, origin_host ); + // Collect any in-flight auction before processing buffered HTML. + // BufferedProcessed is taken when HTML has post-processors (e.g. Next.js rewriters). + // Unlike the Stream path, the body is fully buffered first — collect auction + // now so bids are available when the handler fires. + if let Some(dispatched) = dispatched_auction { + let placeholder = fastly::Request::get("https://placeholder.invalid/"); + let result = orchestrator + .collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + ) + .await; + log::info!( + "BufferedProcessed: auction collected — {} winning bid(s)", + result.winning_bids.len() + ); + write_bids_to_state(&result.winning_bids, price_granularity, &ad_bids_state); + + if settings.debug.auction_html_comment { + let ssp_count = result.provider_responses.len(); + let mediator_info = match &result.mediator_response { + Some(r) => format!("ok({}_bids)", r.bids.len()), + None => "none".to_string(), + }; + let debug_comment = format!( + "", + result.winning_bids.len() + ); + let mut state = ad_bids_state + .write() + .expect("should write bid state for debug"); + match &mut *state { + Some(script) => { + *script = format!("{debug_comment}\n{script}"); + } + None => { + *state = Some(debug_comment); + } + } + } + } + let body = response.take_body(); let params = ProcessResponseParams { content_encoding: &content_encoding, From 3dac7760f6361d022e831137b37314084d3687b9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 15:57:54 +0530 Subject: [PATCH 052/195] Add path label and auction time to debug HTML comment --- crates/trusted-server-core/src/publisher.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 0178d56d1..21399ef74 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -684,8 +684,9 @@ async fn one_behind_loop( None => "none".to_string(), }; let debug_comment = format!( - "", - result.winning_bids.len() + "", + result.winning_bids.len(), + result.total_time_ms, ); let mut state = ad_bids_state .write() @@ -1129,8 +1130,9 @@ pub async fn handle_publisher_request( None => "none".to_string(), }; let debug_comment = format!( - "", - result.winning_bids.len() + "", + result.winning_bids.len(), + result.total_time_ms, ); let mut state = ad_bids_state .write() From 65c0ad3090427a84e3655d81c01ff13c7079472f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 11 May 2026 19:53:15 +0530 Subject: [PATCH 053/195] Fix XSS in script injection and cap mediator at A_deadline html_escape_for_script now unicode-escapes <, >, & and U+2028/2029 in addition to \ and ". These characters allow a crafted bid value to break out of the ` injection breaking out of the script context +/// - U+2028, U+2029 — line/paragraph separators that are valid JSON but terminate +/// a JS string literal in some parsers +/// +/// All substitutions use `\uXXXX` form, which is valid inside both JSON strings +/// and JS string literals. The result is always safe to write as `JSON.parse("…")`. fn html_escape_for_script(s: &str) -> String { - s.replace('\\', "\\\\").replace('"', "\\\"") + s.replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('<', "\\u003C") + .replace('>', "\\u003E") + .replace('&', "\\u0026") + .replace('\u{2028}', "\\u2028") + .replace('\u{2029}', "\\u2029") } /// Build a price-bucketed bid map from winning bids. @@ -2633,6 +2645,26 @@ mod tests { "both\\\\\\\"mixed", "should escape both backslashes and quotes" ); + assert_eq!( + html_escape_for_script(""), + "\\u003Cscript\\u003Ealert(1)\\u003C/script\\u003E", + "should unicode-escape angle brackets to prevent script injection" + ); + assert_eq!( + html_escape_for_script("a&b"), + "a\\u0026b", + "should unicode-escape ampersand" + ); + assert_eq!( + html_escape_for_script("line\u{2028}sep"), + "line\\u2028sep", + "should unicode-escape U+2028 line separator" + ); + assert_eq!( + html_escape_for_script("para\u{2029}sep"), + "para\\u2029sep", + "should unicode-escape U+2029 paragraph separator" + ); } } } From 0f67a8d9d24368513a7ba68a9fa8e7cf3df20f9f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 13 May 2026 14:36:58 +0530 Subject: [PATCH 054/195] Added footer slot id --- creative-opportunities.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/creative-opportunities.toml b/creative-opportunities.toml index 3cd27f2b1..95ea849a5 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -38,3 +38,22 @@ slot_id = "aps-slot-homepage-header" [slot.providers.pbs.bidders] mocktioneer = { bid = 2.00 } criteo = { networkId = 123456, pubid = "123456" } + +[[slot]] +id = "homepage_footer_ad" +gam_unit_path = "/88059007/autoblog/homepage" +div_id = "ad-fixed_bottom-0-_R_klubtak5lb_" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +floor_price = 0.50 + +[slot.targeting] +pos = "btf" +zone = "fixedBottom" + +[slot.providers.aps] +slot_id = "aps-slot-homepage-footer" + +[slot.providers.pbs.bidders] +mocktioneer = { bid = 1.50 } +criteo = { networkId = 123456, pubid = "123456" } From a7e87512c3bfe7b1fe0296a45e624edfb2498f0a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 15:37:11 +0530 Subject: [PATCH 055/195] Fix page-bids auction context, protect Cache-Control from operator override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass the real incoming request to AuctionContext in handle_page_bids instead of a placeholder — SSPs now receive browser UA, referer, and cookies on SPA navigation bids. Guard Cache-Control in finalize_response so operator response_headers cannot overwrite the private/no-store directives set for per-user HTML and page-bids responses. Disable auction_html_comment debug flag in trusted-server.toml. --- crates/trusted-server-adapter-fastly/src/main.rs | 10 ++++++++++ crates/trusted-server-core/src/publisher.rs | 3 +-- trusted-server.toml | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index fc66fcfdc..24c447d3d 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -404,6 +404,16 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: } for (key, value) in &settings.response_headers { + // Never overwrite a privacy-critical Cache-Control header (private, no-store, etc.) + // that was set for per-user responses (HTML or page-bids). + if **key == header::CACHE_CONTROL + && response + .get_header(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("private")) + { + continue; + } response.set_header(key, value); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 30955f913..5ab8d1aef 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1460,10 +1460,9 @@ pub async fn handle_page_bids( let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); - let placeholder_req = fastly::Request::get("https://placeholder.invalid/"); let auction_context = AuctionContext { settings, - request: &placeholder_req, + request: &req, client_info: services.client_info(), timeout_ms, provider_responses: None, diff --git a/trusted-server.toml b/trusted-server.toml index a71abfdd7..899c8c895 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -208,11 +208,10 @@ endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" timeout_ms = 1000 # Debug configuration (all flags default to false — do not enable in production) -# TODO: remove [debug] block before merging to main [debug] # Inject before . # Visible in page source. Disable after investigation. -auction_html_comment = true +# auction_html_comment = true # Enable the JA4/TLS fingerprint debug endpoint at GET /_ts/debug/ja4. # Returns a plain-text response with the following fields (Fastly-observed values): # ja4 — JA4 TLS client fingerprint From 401136378c6026aa445366b8c0225f132e90ab5a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:14:24 +0530 Subject: [PATCH 056/195] Restore nurl/burl/ad_id through adserver_mock mediation path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mock mediator endpoint does not echo nurl/burl/ad_id back in its response. Build a bid index in request_bids keyed by (provider, slot_id, bidder) — where bidder is recovered from the echoed crid field — and restore the fields in parse_mediation_response from the original SSP bids. Fixes the spec requirement: both nurl and burl must travel in __ts_bids for client-side sendBeacon firing on slotRenderEnded (§4.5). --- .../src/integrations/adserver_mock.rs | 81 +++++++++++++++---- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 3a42ec2a0..c8d0ca7b5 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -10,7 +10,7 @@ use fastly::Request; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as Json}; use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; use validator::Validate; @@ -88,16 +88,28 @@ impl IntegrationConfig for AdServerMockConfig { // Provider // ============================================================================ +/// Lookup index built from original SSP bids during `request_bids`, consumed +/// during `parse_response` to restore `nurl`/`burl`/`ad_id` that the mock +/// mediator endpoint does not echo back. +/// +/// Keyed by `(provider_name, slot_id, bidder_name)`. +type BidIndex = HashMap<(String, String, String), Bid>; + /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, + /// Bridges SSP bid metadata (nurl/burl/ad_id) from request_bids to parse_response. + bid_index: Mutex>, } impl AdServerMockProvider { /// Create a new mock ad server provider. #[must_use] pub fn new(config: AdServerMockConfig) -> Self { - Self { config } + Self { + config, + bid_index: Mutex::new(None), + } } /// Build the mediation endpoint URL, appending context values as query @@ -212,8 +224,17 @@ impl AdServerMockProvider { /// Parse `OpenRTB` response from mediation endpoint. /// Mediation returns decoded prices for all bids (including APS bids that were encoded). - fn parse_mediation_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { - // Parse OpenRTB response + /// + /// `bid_index` is the SSP-bid lookup built in `request_bids`. The mock mediator + /// does not echo `nurl`/`burl`/`ad_id` back, so they are restored from the index + /// using `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` + /// field (`"{bidder}-creative"` format set during request construction). + fn parse_mediation_response( + &self, + json: &Json, + response_time_ms: u64, + bid_index: &BidIndex, + ) -> AuctionResponse { let empty_array = vec![]; let seatbid = json["seatbid"].as_array().unwrap_or(&empty_array); @@ -225,10 +246,18 @@ impl AdServerMockProvider { let bids = seat["bid"].as_array().unwrap_or(&empty_bids); for bid in bids { - // Mediation layer returns decoded prices for all bids + let slot_id = bid["impid"].as_str().unwrap_or("").to_string(); + + // Recover bidder name from crid ("{bidder}-creative") to look up the + // original SSP bid and restore nurl/burl/ad_id the mediator drops. + let crid = bid["crid"].as_str().unwrap_or(""); + let bidder = crid.strip_suffix("-creative").unwrap_or(""); + let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); + let original = bid_index.get(&key); + all_bids.push(Bid { - slot_id: bid["impid"].as_str().unwrap_or("").to_string(), - price: bid["price"].as_f64(), // Now properly decoded by mediation + slot_id, + price: bid["price"].as_f64(), currency: "USD".to_string(), creative: bid["adm"].as_str().map(String::from), width: bid["w"].as_u64().unwrap_or(0) as u32, @@ -239,9 +268,9 @@ impl AdServerMockProvider { .filter_map(|v| v.as_str().map(String::from)) .collect() }), - nurl: None, - burl: None, - ad_id: None, + nurl: original.and_then(|b| b.nurl.clone()), + burl: original.and_then(|b| b.burl.clone()), + ad_id: original.and_then(|b| b.ad_id.clone()), metadata: HashMap::new(), }); } @@ -274,6 +303,19 @@ impl AuctionProvider for AdServerMockProvider { bidder_responses.len() ); + // Build bid index so parse_response can restore nurl/burl/ad_id from + // the original SSP bids (the mock mediator does not echo these fields). + let mut index = BidIndex::new(); + for response in bidder_responses { + for bid in &response.bids { + index.insert( + (response.provider.clone(), bid.slot_id.clone(), bid.bidder.clone()), + bid.clone(), + ); + } + } + *self.bid_index.lock().expect("should lock bid index") = Some(index); + // Build mediation request let mediation_req = self .build_mediation_request(request, bidder_responses) @@ -349,7 +391,15 @@ impl AuctionProvider for AdServerMockProvider { log::trace!("AdServer Mock response: {:?}", response_json); - let auction_response = self.parse_mediation_response(&response_json, response_time_ms); + let bid_index = self + .bid_index + .lock() + .expect("should lock bid index") + .take() + .unwrap_or_default(); + + let auction_response = + self.parse_mediation_response(&response_json, response_time_ms, &bid_index); log::info!( "AdServer Mock returned {} bids in {}ms", @@ -571,7 +621,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 200); + let auction_response = + provider.parse_mediation_response(&mediation_response, 200, &BidIndex::new()); assert_eq!(auction_response.provider, "adserver_mock"); assert_eq!(auction_response.status, BidStatus::Success); @@ -597,7 +648,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 100); + let auction_response = + provider.parse_mediation_response(&mediation_response, 100, &BidIndex::new()); assert_eq!(auction_response.status, BidStatus::NoBid); assert_eq!(auction_response.bids.len(), 0); @@ -791,7 +843,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 200); + let auction_response = + provider.parse_mediation_response(&mediation_response, 200, &BidIndex::new()); assert_eq!(auction_response.status, BidStatus::Success); assert_eq!(auction_response.bids.len(), 2); From 8516caa8e5cd5369755f5edad665311f98fca2dd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:17:40 +0530 Subject: [PATCH 057/195] Populate device.user_agent in auction request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit APS reads user agent from request.device — without it, real APS bids arrive with wrong or missing device targeting. Pass the incoming UA from both the page-load and page-bids auction paths. --- crates/trusted-server-core/src/publisher.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 5ab8d1aef..18fc62c8e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -20,7 +20,7 @@ use fastly::{Body, Request, Response}; use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; use crate::auction::types::{ - AuctionContext, AuctionRequest, Bid, PublisherInfo, SiteInfo, UserInfo, + AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::backend::BackendConfig; use crate::compat; @@ -935,6 +935,7 @@ pub async fn handle_publisher_request( &request_info, &request_path, co_config, + req.get_header_str("user-agent"), ); let auction_context = AuctionContext { settings, @@ -1180,6 +1181,7 @@ pub(crate) fn build_auction_request( request_info: &crate::http_util::RequestInfo, request_path: &str, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + user_agent: Option<&str>, ) -> AuctionRequest { let slots = matched_slots .iter() @@ -1201,7 +1203,11 @@ pub(crate) fn build_auction_request( fresh_id: ec_id.to_string(), consent: Some(consent_context.clone()), }, - device: None, + device: user_agent.filter(|ua| !ua.is_empty()).map(|ua| DeviceInfo { + user_agent: Some(ua.to_string()), + ip: None, + geo: None, + }), site: Some(SiteInfo { domain: request_info.host.clone(), page: page_url, @@ -1456,6 +1462,7 @@ pub async fn handle_page_bids( &request_info, &path_param, co_config, + req.get_header_str("user-agent"), ); let timeout_ms = co_config .auction_timeout_ms From 9e0ec5ba566dd925e01dc967b265142b37e782fb Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:31:42 +0530 Subject: [PATCH 058/195] Fix __tsAdInit fallback: look up bids by slot id not div id slotRenderEnded gives a div element id via getSlotElementId(), but __ts_bids is keyed by slot id. Build a divToSlotId map during slot setup (matching the TS implementation) and use it in the event handler. Without this, nurl/burl beacons and hb_adid match checks silently fail in the server-rendered fallback whenever div_id != id. --- crates/trusted-server-core/src/integrations/gpt.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 796d633e1..690ba0486 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -447,22 +447,24 @@ impl IntegrationHeadInjector for GptIntegration { "window.__tsAdInit=function(){", "var slots=window.__ts_ad_slots||[];", "var bids=window.__ts_bids||{};", + "var divToSlotId={};", "googletag.cmd.push(function(){", - "var gptSlots=slots.map(function(slot){", + "slots.map(function(slot){", "var s=googletag.defineSlot(slot.gam_unit_path,slot.formats,slot.div_id);", - "if(!s)return null;", + "if(!s)return;", "s.addService(googletag.pubads());", "Object.entries(slot.targeting||{}).forEach(function(e){s.setTargeting(e[0],e[1]);});", "var b=bids[slot.id]||{};", "[\"hb_pb\",\"hb_bidder\",\"hb_adid\"].forEach(function(k){if(b[k])s.setTargeting(k,b[k]);});", "s.setTargeting(\"ts_initial\",\"1\");", - "return{id:slot.id,gptSlot:s};", - "}).filter(Boolean);", + "divToSlotId[slot.div_id]=slot.id;", + "});", "googletag.pubads().enableSingleRequest();", "googletag.enableServices();", "googletag.pubads().addEventListener(\"slotRenderEnded\",function(ev){", - "var id=ev.slot.getSlotElementId();", - "var b=bids[id]||{};", + "var divId=ev.slot.getSlotElementId();", + "var slotId=divToSlotId[divId]||divId;", + "var b=bids[slotId]||{};", "var ourBidWon=!ev.isEmpty&&b.hb_adid&&ev.slot.getTargeting(\"hb_adid\")[0]===b.hb_adid;", "if(ourBidWon){", "if(b.nurl)navigator.sendBeacon(b.nurl);", From d27a329919e48ea35afc66ad91115f898a3fd10c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:32:12 +0530 Subject: [PATCH 059/195] Format lint using cargo fmt --- .../trusted-server-core/src/integrations/adserver_mock.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index c8d0ca7b5..a8f7cadfa 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -309,7 +309,11 @@ impl AuctionProvider for AdServerMockProvider { for response in bidder_responses { for bid in &response.bids { index.insert( - (response.provider.clone(), bid.slot_id.clone(), bid.bidder.clone()), + ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), + ), bid.clone(), ); } From 790c1232f0efa050eff1ecc5c84a7cd9307a78ea Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 14 May 2026 16:44:22 +0530 Subject: [PATCH 060/195] Fix clippy doc-markdown lint in adserver_mock --- crates/trusted-server-core/src/integrations/adserver_mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index a8f7cadfa..4330ea660 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -98,7 +98,7 @@ type BidIndex = HashMap<(String, String, String), Bid>; /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, - /// Bridges SSP bid metadata (nurl/burl/ad_id) from request_bids to parse_response. + /// Bridges SSP bid metadata (`nurl`/`burl`/`ad_id`) from `request_bids` to `parse_response`. bid_index: Mutex>, } From 299f6ba95704dcf0b35a56f2f28d74a7075c6a55 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 15:59:24 +0530 Subject: [PATCH 061/195] Remove inline PBS bidder params from creative-opportunities.toml PBS bidder credentials (mocktioneer, criteo placeholder params) were being sent directly to PBS on every auction request. Per the design spec, PBS bidder params belong in PBS stored requests keyed by slot ID, not in the edge config file. Removes PbsSlotParams struct, SlotProviders.pbs field, the to_ad_slot wiring block, and the corresponding test. Slots without inline bidder params trigger the existing storedrequest fallback path in the Prebid provider. Closes #697 --- .../src/creative_opportunities.rs | 65 ------------------- creative-opportunities.toml | 12 ---- 2 files changed, 77 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 12957d4b8..25add829a 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -125,11 +125,6 @@ impl CreativeOpportunitySlot { serde_json::json!({ "slotID": aps.slot_id }), ); } - if let Some(ref pbs) = self.providers.pbs { - for (bidder_name, params) in &pbs.bidders { - bidders.insert(bidder_name.clone(), params.clone()); - } - } AdSlot { id: self.id.clone(), formats: self @@ -175,8 +170,6 @@ impl CreativeOpportunityFormat { pub struct SlotProviders { /// Amazon Publisher Services (APS/TAM) slot parameters. pub aps: Option, - /// Prebid Server (PBS) slot parameters. - pub pbs: Option, } /// APS-specific parameters for a slot. @@ -186,24 +179,6 @@ pub struct ApsSlotParams { pub slot_id: String, } -/// PBS-specific parameters for a slot. -/// -/// Bidder params are sent inline to Prebid Server so bidder credentials -/// stay in `creative-opportunities.toml` rather than in PBS stored requests. -#[derive(Debug, Clone, Default, Deserialize)] -pub struct PbsSlotParams { - /// Per-bidder params keyed by bidder name (must match PBS adapter name). - /// - /// Example in TOML: - /// ```toml - /// [slot.providers.pbs.bidders] - /// mocktioneer = { bid = 2.00 } - /// criteo = { networkId = 123456, pubid = "123456" } - /// ``` - #[serde(default)] - pub bidders: HashMap, -} - /// TOML file structure for creative opportunity slot definitions. #[derive(Debug, Clone, Deserialize, Default)] pub struct CreativeOpportunitiesFile { @@ -333,46 +308,6 @@ mod tests { ); } - #[test] - fn to_ad_slot_wires_pbs_bidder_params_into_bidders() { - let mut slot = make_slot("atf_sidebar_ad", vec!["/"]); - slot.providers.pbs = Some(PbsSlotParams { - bidders: [ - ( - "mocktioneer".to_string(), - serde_json::json!({ "bid": 2.00 }), - ), - ( - "criteo".to_string(), - serde_json::json!({ "networkId": 123456, "pubid": "123456" }), - ), - ] - .into_iter() - .collect(), - }); - let ad_slot = slot.to_ad_slot("88059007"); - let mock_params = ad_slot - .bidders - .get("mocktioneer") - .expect("should have mocktioneer bidder"); - assert_eq!( - mock_params.get("bid").and_then(serde_json::Value::as_f64), - Some(2.0), - "should wire mocktioneer bid param" - ); - let criteo_params = ad_slot - .bidders - .get("criteo") - .expect("should have criteo bidder"); - assert_eq!( - criteo_params - .get("networkId") - .and_then(serde_json::Value::as_i64), - Some(123456), - "should wire criteo networkId param" - ); - } - #[test] fn to_ad_slot_sets_floor_price_and_formats() { let slot = make_slot("atf", vec!["/"]); diff --git a/creative-opportunities.toml b/creative-opportunities.toml index 95ea849a5..b6ed8900f 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -16,10 +16,6 @@ zone = "atfSidebar" [slot.providers.aps] slot_id = "aps-slot-atf-sidebar" -[slot.providers.pbs.bidders] -mocktioneer = { bid = 2.00 } -criteo = { networkId = 123456, pubid = "123456" } - [[slot]] id = "homepage_header_ad" gam_unit_path = "/88059007/autoblog/homepage" @@ -35,10 +31,6 @@ zone = "header" [slot.providers.aps] slot_id = "aps-slot-homepage-header" -[slot.providers.pbs.bidders] -mocktioneer = { bid = 2.00 } -criteo = { networkId = 123456, pubid = "123456" } - [[slot]] id = "homepage_footer_ad" gam_unit_path = "/88059007/autoblog/homepage" @@ -53,7 +45,3 @@ zone = "fixedBottom" [slot.providers.aps] slot_id = "aps-slot-homepage-footer" - -[slot.providers.pbs.bidders] -mocktioneer = { bid = 1.50 } -criteo = { networkId = 123456, pubid = "123456" } From 03d39f29bc2d643c25b9d85ccbae9fc304ee5d5c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 16:16:51 +0530 Subject: [PATCH 062/195] Clarify and test APS floor price enforcement in mediation path - Rewrite misleading comment in apply_floor_prices: price=None bids pass through in the parallel-only path because decoding is deferred; in the mediation path the mediator decodes prices before this function runs - Add test: decoded APS bid below slot floor is dropped - Add test: decoded APS bid at or above slot floor is kept Closes #698 --- .../src/auction/orchestrator.rs | 83 ++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 17fe405dd..58f46b149 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -542,7 +542,11 @@ impl AuctionOrchestrator { let starting_count = winning_bids.len(); winning_bids.retain(|slot_id, bid| match floor_prices.get(slot_id) { Some(floor) => { - // Bids without price (e.g., APS) pass through - floor checked in mediation + // price=None means the SSP returned an encoded price (e.g. APS amznbid). + // In the parallel-only path this bid cannot yet be floor-checked; it passes + // through and will be decoded (and re-checked) by the mediation layer. + // In the mediation path, mediation decodes prices before calling this + // function, so any bid still carrying price=None is dropped upstream. match bid.price { Some(price) if price >= *floor => true, Some(_) => { @@ -554,7 +558,7 @@ impl AuctionOrchestrator { } None => { log::debug!( - "Passing bid with encoded price for slot '{}' - floor check deferred to mediation", + "Passing encoded-price bid for slot '{}' - price not yet decoded", slot_id ); true @@ -1305,4 +1309,79 @@ mod tests { "Price should still be None (not decoded yet)" ); } + + #[test] + fn test_apply_floor_prices_drops_decoded_aps_bid_below_floor() { + // After mediation decodes an APS bid, apply_floor_prices must enforce the + // slot floor on the resulting price=Some(x) value. This test simulates the + // state of a bid after mediator decoding: price is Some, amznbid is gone. + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let mut floor_prices = HashMap::new(); + floor_prices.insert("atf".to_string(), 0.50); + + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf".to_string(), + Bid { + slot_id: "atf".to_string(), + price: Some(0.30), // decoded APS price — below $0.50 floor + currency: "USD".to_string(), + creative: Some("
APS Ad
".to_string()), + adomain: None, + bidder: "aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: HashMap::new(), + }, + ); + + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + + assert!( + filtered.is_empty(), + "Decoded APS bid below slot floor should be dropped" + ); + } + + #[test] + fn test_apply_floor_prices_keeps_decoded_aps_bid_at_or_above_floor() { + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let mut floor_prices = HashMap::new(); + floor_prices.insert("atf".to_string(), 0.50); + + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf".to_string(), + Bid { + slot_id: "atf".to_string(), + price: Some(0.75), // decoded APS price — above floor + currency: "USD".to_string(), + creative: Some("
APS Ad
".to_string()), + adomain: None, + bidder: "aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: HashMap::new(), + }, + ); + + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + + assert_eq!( + filtered.len(), + 1, + "Decoded APS bid at or above floor should be kept" + ); + assert_eq!( + filtered.get("atf").expect("atf should be present").price, + Some(0.75), + "Price should be preserved" + ); + } } From f09eb34ffac5b3230a7a1c4af5804890e9b4954b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 16:24:28 +0530 Subject: [PATCH 063/195] Document and test /auction API contract for non-Prebid.js callers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Expand handle_auction doc: inline-params vs stored-request paths, config passthrough and allowed_context_keys, response headers - Document AdRequest, AdUnit, BidConfig with the stored-request contract: absent/empty bids → empty bidders map → PBS stored-request fallback - Add tests for convert_tsjs_to_auction_request: - No bids → empty bidders map (stored-request path) - Inline bids → bidders map populated - Allowed config key passes through; disallowed key dropped - Invalid 3-element banner size returns error Closes #699 --- .../src/auction/endpoints.rs | 41 +++- .../src/auction/formats.rs | 195 +++++++++++++++++- 2 files changed, 230 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 0430f08ba..5a9ac6f10 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -16,11 +16,44 @@ use super::formats::{convert_to_openrtb_response, convert_tsjs_to_auction_reques use super::types::AuctionContext; use super::AuctionOrchestrator; -/// Handle auction request from /auction endpoint. +/// Handle auction request from `POST /auction`. /// -/// This is the main entry point for running header bidding auctions. -/// It orchestrates bids from multiple providers (Prebid, APS, GAM, etc.) and returns -/// the winning bids in `OpenRTB` format with creative HTML inline in the `adm` field. +/// Accepts a JSON body matching [`AdRequest`][`super::formats::AdRequest`]. +/// The minimum valid request is: +/// +/// ```json +/// { +/// "adUnits": [{ +/// "code": "atf_sidebar_ad", +/// "mediaTypes": { "banner": { "sizes": [[300, 250]] } } +/// }] +/// } +/// ``` +/// +/// ## Bidder params: inline vs. stored-request +/// +/// Each ad unit's `bids` array is **optional**. When absent or empty the PBS +/// integration falls back to a stored-request keyed by the unit's `code` +/// field (`imp.ext.prebid.storedrequest = { id: "" }`). A PBS stored +/// request must therefore exist for every slot code that omits inline params. +/// +/// When `bids` is supplied, each entry's `bidder`/`params` pair is forwarded +/// directly as `imp.ext.prebid.bidder.`. +/// +/// ## Context passthrough (`config`) +/// +/// The optional `config` object is filtered through +/// [`auction.allowed_context_keys`][`crate::settings::AuctionConfig::allowed_context_keys`]. +/// Only keys listed there reach the auction providers (e.g. `"permutive_segments"`). +/// All other keys are silently dropped. Values must be either strings or arrays of +/// strings. +/// +/// ## Response +/// +/// Returns an `OpenRTB 2.x` response. Creative HTML is inlined in each bid's +/// `adm` field after sanitisation and first-party URL rewriting. Response +/// headers include `X-TS-EC` (the caller's Edge Cookie ID) and +/// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). /// /// # Errors /// diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 5237921a7..53c6474a0 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -28,7 +28,11 @@ use super::types::{ PublisherInfo, SiteInfo, UserInfo, }; -/// Request body format for auction endpoints (tsjs/Prebid.js format). +/// Request body for `POST /auction` (tsjs / Prebid.js wire format). +/// +/// `adUnits` lists the placements to bid on. `config` carries optional +/// context values (e.g. audience segments) filtered through +/// [`auction.allowed_context_keys`][`crate::settings::AuctionConfig::allowed_context_keys`]. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdRequest { @@ -36,6 +40,15 @@ pub struct AdRequest { pub config: Option, } +/// A single ad placement in an [`AdRequest`]. +/// +/// `code` identifies the slot (e.g. `"atf_sidebar_ad"`) and becomes the +/// impression ID in the outgoing `OpenRTB` request. +/// +/// `bids` is optional. When absent or empty the PBS provider falls back to +/// a stored-request keyed by `code` (`imp.ext.prebid.storedrequest.id`). +/// When present, each entry's params are forwarded inline to PBS as +/// `imp.ext.prebid.bidder.`. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdUnit { @@ -44,7 +57,11 @@ pub struct AdUnit { pub bids: Option>, } -/// Bidder configuration from the request. +/// Inline bidder params for one SSP within an [`AdUnit`]. +/// +/// `params` is passed verbatim to the corresponding PBS bidder adapter. +/// When the `bids` array is absent, the slot falls back to PBS stored +/// requests — see [`AdUnit`] for details. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BidConfig { @@ -318,3 +335,177 @@ pub fn convert_to_openrtb_response( .with_header(HEADER_X_TS_EC_FRESH, &auction_request.user.fresh_id) .with_body(body_bytes)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::ConsentContext; + use crate::platform::test_support::noop_services; + use crate::test_support::tests::crate_test_settings_str; + use fastly::http::Method; + use fastly::Request; + + fn make_settings() -> Settings { + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") + } + + fn make_req() -> Request { + Request::new(Method::POST, "https://test-publisher.com/auction") + } + + fn call_convert(body: &AdRequest) -> AuctionRequest { + let settings = make_settings(); + let services = noop_services(); + let req = make_req(); + convert_tsjs_to_auction_request( + body, + &settings, + &services, + &req, + ConsentContext::default(), + "test-ec-id", + None, + ) + .expect("should convert without error") + } + + #[test] + fn no_bids_produces_empty_bidders_map() { + // An ad unit with no `bids` array must produce an empty bidders map. + // An empty bidders map triggers the PBS stored-request fallback: + // the PBS provider sets imp.ext.prebid.storedrequest = { id: "" }. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "atf_sidebar_ad".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250]], + }), + }), + bids: None, + }], + config: None, + }; + + let auction_request = call_convert(&body); + + assert_eq!(auction_request.slots.len(), 1, "should have one slot"); + let slot = &auction_request.slots[0]; + assert_eq!(slot.id, "atf_sidebar_ad", "slot id should match unit code"); + assert!( + slot.bidders.is_empty(), + "absent bids array should yield empty bidders map (PBS stored-request path)" + ); + } + + #[test] + fn inline_bids_populate_bidders_map() { + // When bids are supplied, each bidder+params pair should appear in the + // slot's bidders map so PBS receives inline params. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "homepage_header_ad".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![970, 90]], + }), + }), + bids: Some(vec![BidConfig { + bidder: "kargo".to_string(), + params: serde_json::json!({ "placementId": "client_123" }), + }]), + }], + config: None, + }; + + let auction_request = call_convert(&body); + + let slot = &auction_request.slots[0]; + assert!( + slot.bidders.contains_key("kargo"), + "kargo bidder should be present in slot bidders map" + ); + assert_eq!( + slot.bidders["kargo"]["placementId"], "client_123", + "bidder params should be forwarded verbatim" + ); + } + + #[test] + fn config_allowed_key_passes_through() { + // Keys in auction.allowed_context_keys must reach the auction context. + // The test settings do not set allowed_context_keys so the default + // (empty) applies — verify a key is NOT present rather than IS. + // To test the allow-list, inject a key via a custom settings string. + let settings_str = format!( + "{}\n[auction]\nallowed_context_keys = [\"permutive_segments\"]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_str).expect("should parse"); + let services = noop_services(); + let req = make_req(); + + let body = AdRequest { + ad_units: vec![], + config: Some(serde_json::json!({ + "permutive_segments": ["seg1", "seg2"], + "disallowed_key": "should be dropped", + })), + }; + + let auction_request = convert_tsjs_to_auction_request( + &body, + &settings, + &services, + &req, + ConsentContext::default(), + "test-ec-id", + None, + ) + .expect("should convert"); + + assert!( + auction_request.context.contains_key("permutive_segments"), + "allowed key should be in auction context" + ); + assert!( + !auction_request.context.contains_key("disallowed_key"), + "unlisted key should be dropped" + ); + } + + #[test] + fn invalid_banner_size_returns_error() { + // Banner sizes must be [width, height] pairs; a 3-element size is invalid. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "bad_slot".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250, 99]], // invalid — 3 elements + }), + }), + bids: None, + }], + config: None, + }; + + let settings = make_settings(); + let services = noop_services(); + let req = make_req(); + let result = convert_tsjs_to_auction_request( + &body, + &settings, + &services, + &req, + ConsentContext::default(), + "test-ec-id", + None, + ); + + assert!( + result.is_err(), + "3-element banner size should return an error" + ); + } +} From a03c70a8cbbf6065eacb34c5a1ee1ad53c556025 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 17:07:35 +0530 Subject: [PATCH 064/195] Verify and document graceful degradation when no slots match URL - Add debug log at no-match gate in handle_publisher_request and handle_page_bids so operators can confirm the feature is inactive on non-article URLs without reading source code - Add test: empty slots file (kill-switch) returns slots:[] bids:{} - Add test: URL not matching any slot pattern returns slots:[] bids:{} Closes #700 --- crates/trusted-server-core/src/publisher.rs | 117 ++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 18fc62c8e..a4773a629 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -905,6 +905,13 @@ pub async fn handle_publisher_request( let should_run_auction = is_get && !is_prefetch && !is_bot && !matched_slots.is_empty() && consent_allows_auction; + if matched_slots.is_empty() && settings.creative_opportunities.is_some() { + log::debug!( + "No creative opportunity slots matched path '{}' — skipping auction and injection", + request_path + ); + } + let auction_timeout_ms = settings .creative_opportunities .as_ref() @@ -1454,6 +1461,13 @@ pub async fn handle_page_bids( .as_ref() .is_some_and(|tcf| tcf.has_purpose_consent(1)); + if matched_slots.is_empty() { + log::debug!( + "No creative opportunity slots matched path '{}' — skipping auction", + path_param + ); + } + let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { let auction_request = build_auction_request( &matched_slots, @@ -2673,4 +2687,107 @@ mod tests { ); } } + + mod page_bids_no_match_tests { + use super::super::*; + use crate::auction::AuctionOrchestrator; + use crate::creative_opportunities::{ + CreativeOpportunitiesFile, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + use crate::platform::test_support::noop_services; + use crate::test_support::tests::crate_test_settings_str; + use fastly::http::Method; + use fastly::Request; + + fn settings_with_co() -> Settings { + let toml = format!( + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") + } + + fn file_with_article_slot() -> CreativeOpportunitiesFile { + CreativeOpportunitiesFile { + slots: vec![CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + }], + } + } + + fn make_page_bids_request(path: &str) -> Request { + Request::new( + Method::GET, + format!("https://test-publisher.com/_ts/page-bids?path={path}"), + ) + } + + #[tokio::test] + async fn empty_slots_file_returns_empty_slots_and_bids() { + // Spec §8 kill-switch: creative-opportunities.toml with zero slots disables + // all server-side auction activity and injection. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = CreativeOpportunitiesFile { slots: vec![] }; + let req = make_page_bids_request("/2024/01/my-article/"); + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"].as_array().expect("slots should be array").len(), + 0, + "empty slots file should produce zero injected slots" + ); + assert_eq!( + body["bids"].as_object().expect("bids should be object").len(), + 0, + "empty slots file should produce zero bids" + ); + } + + #[tokio::test] + async fn url_not_matching_any_pattern_returns_empty_response() { + // Slots exist but request path does not match — no auction, no injection. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = file_with_article_slot(); // slot matches /20** only + let req = make_page_bids_request("/about"); // does not match + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"].as_array().expect("slots should be array").len(), + 0, + "non-matching URL should produce zero injected slots" + ); + assert_eq!( + body["bids"].as_object().expect("bids should be object").len(), + 0, + "non-matching URL should produce zero bids" + ); + } + } } From 421833399efc17692a869e7355d5f105e6b99944 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 17:10:45 +0530 Subject: [PATCH 065/195] Format publisher.rs with cargo fmt --- crates/trusted-server-core/src/publisher.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a4773a629..235be8178 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2751,12 +2751,18 @@ mod tests { serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); assert_eq!( - body["slots"].as_array().expect("slots should be array").len(), + body["slots"] + .as_array() + .expect("slots should be array") + .len(), 0, "empty slots file should produce zero injected slots" ); assert_eq!( - body["bids"].as_object().expect("bids should be object").len(), + body["bids"] + .as_object() + .expect("bids should be object") + .len(), 0, "empty slots file should produce zero bids" ); @@ -2779,12 +2785,18 @@ mod tests { serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); assert_eq!( - body["slots"].as_array().expect("slots should be array").len(), + body["slots"] + .as_array() + .expect("slots should be array") + .len(), 0, "non-matching URL should produce zero injected slots" ); assert_eq!( - body["bids"].as_object().expect("bids should be object").len(), + body["bids"] + .as_object() + .expect("bids should be object") + .len(), 0, "non-matching URL should produce zero bids" ); From 0762999f37b977ffe7d61705cac5bb266ce32140 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 15 May 2026 17:13:35 +0530 Subject: [PATCH 066/195] Document scroll/refresh handoff contract between TS and slim-Prebid - Clarify in handle_auction doc that /auction is for initial render and programmatic callers; scroll/refresh/SPA navigation is slim-Prebid's domain in Phase 1 - Note Phase 2 slot-template-aware refresh API as deferred future work - Add head_inserts doc clarifying __tsAdInit handles initial render only; slotRenderEnded fires win beacons but does not trigger refresh auctions Closes #702 --- .../trusted-server-core/src/auction/endpoints.rs | 14 ++++++++++++++ crates/trusted-server-core/src/integrations/gpt.rs | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 5a9ac6f10..5d5bb292c 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -55,6 +55,20 @@ use super::AuctionOrchestrator; /// headers include `X-TS-EC` (the caller's Edge Cookie ID) and /// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). /// +/// ## Scroll, refresh, and SPA navigation +/// +/// This endpoint is intended for **initial page render** and **programmatic +/// callers** (e.g. slim-Prebid, native apps, server-to-server integrations). +/// It is **not** the intended path for scroll or GPT refresh events. +/// +/// In Phase 1, slim-Prebid owns scroll and refresh: it runs post-`window.load`, +/// listens for GPT refresh events, and runs client-side auctions independently +/// of this endpoint. SPAs that use pushState routing do not trigger TS page-level +/// auctions — slim-Prebid handles those cases too. +/// +/// A slot-template-aware refresh API (`POST /auction/refresh`) is deferred to a +/// future phase and not designed here. +/// /// # Errors /// /// Returns an error if: diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 690ba0486..85ea800ea 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -437,6 +437,19 @@ impl IntegrationHeadInjector for GptIntegration { GPT_INTEGRATION_ID } + /// Injects the `__tsAdInit` bootstrap script into ``. + /// + /// ## Scroll / refresh handoff contract (Phase 1) + /// + /// `__tsAdInit` handles **initial render only**: it wires server-side bid + /// targeting into GPT slots and fires win beacons (`nurl`/`burl`) via + /// `slotRenderEnded`. It does **not** trigger refresh auctions or handle + /// GPT slot refresh events. + /// + /// Post-`window.load`, slim-Prebid takes over: it listens for GPT refresh + /// events, runs client-side auctions, and sets targeting for subsequent + /// impressions. SPA pushState navigation is also slim-Prebid's domain. + /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { vec![ ""# @@ -606,7 +624,7 @@ mod tests { request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), ad_slots_script: None, - ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), } } @@ -1263,7 +1281,7 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), - ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), }; let mut processor = create_html_processor(config); let output = processor @@ -1287,7 +1305,7 @@ mod tests { fn injects_ts_bids_before_body_close() { let bids_script = r#""#; - let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1316,7 +1334,7 @@ mod tests { fn injects_ts_bids_only_once_with_multiple_body_elements() { let bids_script = r#""#; - let state = std::sync::Arc::new(std::sync::RwLock::new(Some(bids_script.to_string()))); + let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1342,7 +1360,7 @@ mod tests { fn injects_empty_ts_bids_when_slots_matched_but_auction_returned_nothing() { // Slots matched (ad_slots_script is Some) but auction task never wrote a result // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. - let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1367,7 +1385,7 @@ mod tests { // No slots matched this URL — ad_slots_script is None. __ts_bids must be // omitted entirely so the publisher's existing client-side GPT flow is // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). - let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 850798e43..71ef2bbe7 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -456,10 +456,16 @@ impl ApsAuctionProvider { aps_response.contextual.slots.len() ); - let slot_map = self - .slot_id_map - .lock() - .expect("should lock APS slot id map"); + // Take the map by value so it does not linger on the provider + // across requests if the Fastly Compute runtime ever reuses Wasm + // instances. Today each request gets its own instance so this is + // belt-and-suspenders; tomorrow it may not be. + let slot_map = std::mem::take( + &mut *self + .slot_id_map + .lock() + .expect("should lock APS slot id map"), + ); for slot in aps_response.contextual.slots { match self.parse_aps_slot(&slot) { Ok(mut bid) => { diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 85ea800ea..cb0994029 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -455,44 +455,21 @@ impl IntegrationHeadInjector for GptIntegration { "" .to_string(), - concat!( - "" - ).to_string(), + format!("", GPT_BOOTSTRAP_JS), ] } } +/// Inline `window.__tsAdInit` bootstrap injected at `` so the bids +/// script at `` can call it before the TSJS bundle has loaded. +/// +/// The bundle's idempotent implementation in +/// `crates/js/lib/src/integrations/gpt/index.ts` later overwrites this stub. +/// Both implementations guard the one-time-per-page setup with +/// `window.__tsServicesEnabled` so neither double-enables services if the +/// publisher's own init code also calls `googletag.enableServices()`. +const GPT_BOOTSTRAP_JS: &str = include_str!("gpt_bootstrap.js"); + // Default value functions fn default_enabled() -> bool { @@ -1120,6 +1097,32 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("__tsServicesEnabled"), + "should guard enableServices/enableSingleRequest with the __tsServicesEnabled flag" + ); + assert!( + combined.contains("window.__tsAdInit"), + "should install __tsAdInit on window" + ); + assert!( + !combined.contains("googletag.pubads().refresh()"), + "should never call unbounded refresh() — only refresh(newSlots)" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js new file mode 100644 index 000000000..a3d28a286 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -0,0 +1,78 @@ +// Edge-injected GPT auction bootstrap. +// +// This is the minimal `window.__tsAdInit` that runs on first page load +// before the TSJS bundle has had a chance to install its richer +// idempotent implementation. The bundle in +// crates/js/lib/src/integrations/gpt/index.ts overwrites `__tsAdInit` +// once it loads. +// +// Contract with the bundle: +// - Both implementations must set `window.__tsServicesEnabled = true` +// after calling `enableSingleRequest()`/`enableServices()` so a +// subsequent call from any source (the bundle's `__tsAdInit`, the +// publisher's own GPT init code) becomes a no-op. +// - `refresh()` is called only for the slots defined in this pass, +// never the global slot list, so we never accidentally refresh +// publisher-managed slots that we don't own. +// +// Only installed if `window.__tsAdInit` isn't already defined — that +// way the bundle (or anything else) can preempt this fallback by +// installing first. +(function () { + if (typeof window === "undefined" || window.__tsAdInit) { + return; + } + window.__tsAdInit = function () { + var slots = window.__ts_ad_slots || []; + var bids = window.__ts_bids || {}; + var divToSlotId = {}; + googletag.cmd.push(function () { + var newSlots = []; + slots.forEach(function (slot) { + var s = googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + slot.div_id, + ); + if (!s) return; + s.addService(googletag.pubads()); + Object.entries(slot.targeting || {}).forEach(function (e) { + s.setTargeting(e[0], e[1]); + }); + var b = bids[slot.id] || {}; + ["hb_pb", "hb_bidder", "hb_adid"].forEach(function (k) { + if (b[k]) s.setTargeting(k, b[k]); + }); + s.setTargeting("ts_initial", "1"); + divToSlotId[slot.div_id] = slot.id; + newSlots.push(s); + }); + // Guard the one-time-per-page setup so a follow-up call (e.g. + // publisher's own init code or the bundle's `__tsAdInit` after + // it overwrites this stub) doesn't double-enable services. + if (!window.__tsServicesEnabled) { + googletag.pubads().enableSingleRequest(); + googletag.enableServices(); + window.__tsServicesEnabled = true; + googletag + .pubads() + .addEventListener("slotRenderEnded", function (ev) { + var divId = ev.slot.getSlotElementId(); + var slotId = divToSlotId[divId] || divId; + var b = (window.__ts_bids || {})[slotId] || {}; + var ourBidWon = + !ev.isEmpty && + b.hb_adid && + ev.slot.getTargeting("hb_adid")[0] === b.hb_adid; + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl); + if (b.burl) navigator.sendBeacon(b.burl); + } + }); + } + if (newSlots.length > 0) { + googletag.pubads().refresh(newSlots); + } + }); + }; +})(); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d7711d61e..b74b234ca 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -164,10 +164,6 @@ pub struct PrebidIntegrationConfig { /// - `both` — consent in both cookies and body (default) #[serde(default)] pub consent_forwarding: ConsentForwardingMode, - /// When true, suppresses client-side nurl firing. - /// Use for PBS deployments that fire nurl internally. - #[serde(default)] - pub suppress_nurl: bool, } impl IntegrationConfig for PrebidIntegrationConfig { @@ -1661,16 +1657,9 @@ mod tests { bid_param_overrides: HashMap::default(), bid_param_override_rules: Vec::new(), consent_forwarding: ConsentForwardingMode::Both, - suppress_nurl: false, } } - #[test] - fn prebid_config_suppress_nurl_defaults_to_false() { - let config = base_config(); - assert!(!config.suppress_nurl, "should not suppress nurl by default"); - } - fn create_test_auction_request() -> AuctionRequest { AuctionRequest { id: "auction-123".to_string(), diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs index b683020bf..cfdca9eb4 100644 --- a/crates/trusted-server-core/src/price_bucket.rs +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -20,7 +20,10 @@ impl PriceGranularity { #[must_use] pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { - if cpm <= 0.0 { + // Reject NaN / Inf early so the `(x * 100.0).floor() as u64` cast below + // can never see a non-finite value (the cast's behaviour for NaN/Inf is + // implementation-defined in Rust and "saturate to 0" only by convention). + if !cpm.is_finite() || cpm <= 0.0 { return "0.00".to_string(); } match granularity { @@ -125,4 +128,31 @@ mod tests { price_bucket(2.53, PriceGranularity::Dense) ); } + + #[test] + fn non_finite_cpm_returns_zero_bucket() { + for granularity in [ + PriceGranularity::Dense, + PriceGranularity::Low, + PriceGranularity::Medium, + PriceGranularity::High, + PriceGranularity::Auto, + ] { + assert_eq!( + price_bucket(f64::NAN, granularity), + "0.00", + "NaN cpm should bucket to 0.00 for granularity {granularity:?}" + ); + assert_eq!( + price_bucket(f64::INFINITY, granularity), + "0.00", + "+Inf cpm should bucket to 0.00 for granularity {granularity:?}" + ); + assert_eq!( + price_bucket(f64::NEG_INFINITY, granularity), + "0.00", + "-Inf cpm should bucket to 0.00 for granularity {granularity:?}" + ); + } + } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 494f06eb5..8908eaf09 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -12,7 +12,7 @@ //! content-rewriting concern. use std::io::Write; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex}; use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; @@ -39,6 +39,11 @@ use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, S use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; +/// Read buffer size for streaming body processing and brotli internal buffers. +/// Both the `Decompressor` and `CompressorWriter` use this value so all +/// brotli I/O layers operate on consistently-sized chunks. +const STREAM_CHUNK_SIZE: usize = 8192; + fn restrict_accept_encoding(req: &mut Request) { // If the client sent no Accept-Encoding, leave the request unchanged so the // origin responds without compression. Adding encodings here would cause the @@ -194,7 +199,7 @@ struct ProcessResponseParams<'a> { content_type: &'a str, integration_registry: &'a IntegrationRegistry, ad_slots_script: Option<&'a str>, - ad_bids_state: &'a Arc>>, + ad_bids_state: &'a Arc>>, } /// Process response body through the streaming pipeline. @@ -262,26 +267,32 @@ fn process_response_streaming( Ok(()) } -/// Create a unified HTML stream processor +/// Create a unified HTML stream processor. +/// +/// Builds the config via [`HtmlProcessorConfig::from_settings`] and then +/// layers the auction-hold streaming fields on top via +/// [`HtmlProcessorConfig::with_ad_state`], so the canonical builder stays the +/// single source of truth: a future field added to `from_settings` is +/// inherited here automatically. fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, - _settings: &Settings, + settings: &Settings, integration_registry: &IntegrationRegistry, ad_slots_script: Option, - ad_bids_state: Arc>>, + ad_bids_state: Arc>>, ) -> Result> { use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; - let config = HtmlProcessorConfig { - origin_host: origin_host.to_string(), - request_host: request_host.to_string(), - request_scheme: request_scheme.to_string(), - integrations: integration_registry.clone(), - ad_slots_script, - ad_bids_state, - }; + let config = HtmlProcessorConfig::from_settings( + settings, + integration_registry, + origin_host, + request_host, + request_scheme, + ) + .with_ad_state(ad_slots_script, ad_bids_state); Ok(create_html_processor(config)) } @@ -412,7 +423,7 @@ pub struct OwnedProcessResponseParams { pub(crate) request_scheme: String, pub(crate) content_type: String, pub(crate) ad_slots_script: Option, - pub(crate) ad_bids_state: Arc>>, + pub(crate) ad_bids_state: Arc>>, /// In-flight SSP bids dispatched before `pending_origin.wait()`. /// The streaming phase collects these and writes bids to `ad_bids_state` /// before processing the last body chunk, so `` injection sees live bids. @@ -493,7 +504,7 @@ pub async fn stream_publisher_body_async( if !is_html { // Non-HTML: collect auction first, then stream. There is no // to hold, so delaying the entire body until collection is acceptable. - let placeholder = Request::get("https://placeholder.invalid/"); + let placeholder = Request::get(crate::auction::types::MEDIATOR_PLACEHOLDER_URL); let result = orchestrator .collect_dispatched_auction( dispatched, @@ -540,16 +551,25 @@ pub async fn stream_publisher_body_async( .await } -/// Build a minimal [`AuctionContext`] for the mediator call in collection. +/// Build a minimal [`AuctionContext`] for the collect phase. /// -/// The `request` field is a short-lived placeholder (providers use it only for -/// header extraction; the placeholder is functionally equivalent to the original -/// since `req` was already consumed by `send_async` before dispatch). +/// See [`AuctionContext::request`]: the orchestrator's collect path runs +/// after `send_async` has already consumed the real client request, so this +/// context carries a synthetic placeholder. The orchestrator itself +/// instantiates a fresh placeholder when it actually invokes a mediator — +/// this argument is plumbing for the (presently unused) case where the +/// orchestrator needs the caller's request shape. fn make_collect_context<'a>( settings: &'a Settings, services: &'a RuntimeServices, placeholder: &'a Request, ) -> AuctionContext<'a> { + debug_assert_eq!( + placeholder.get_url_str(), + crate::auction::types::MEDIATOR_PLACEHOLDER_URL, + "make_collect_context must be given the canonical placeholder; \ + callers must not forward a real client request through the collect path" + ); AuctionContext { settings, request: placeholder, @@ -560,27 +580,87 @@ fn make_collect_context<'a>( } } +/// Well-known crawler User-Agent fragments. Best-effort: an attacker can +/// trivially spoof their UA, so this is for opt-out signalling to honest +/// crawlers (preventing SSP auctions burning partner quota on their behalf), +/// not security. +pub(crate) const BOT_USER_AGENT_FRAGMENTS: &[&str] = + &["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"]; + +/// Returns true when the request's User-Agent matches any well-known crawler +/// fragment in [`BOT_USER_AGENT_FRAGMENTS`]. +pub(crate) fn is_bot_user_agent(req: &Request) -> bool { + let ua = req.get_header_str("user-agent").unwrap_or(""); + BOT_USER_AGENT_FRAGMENTS + .iter() + .any(|frag| ua.contains(frag)) +} + +/// Returns true when the request advertises itself as a prefetch via either +/// the standard `Sec-Purpose` or the legacy `Purpose` header. +pub(crate) fn is_prefetch_request(req: &Request) -> bool { + req.get_header_str("sec-purpose") + .is_some_and(|v| v.contains("prefetch")) + || req + .get_header_str("purpose") + .is_some_and(|v| v.contains("prefetch")) +} + /// Write winning bids from an auction result into the shared `ad_bids_state` lock. pub(crate) fn write_bids_to_state( winning_bids: &std::collections::HashMap, price_granularity: PriceGranularity, - ad_bids_state: &Arc>>, + ad_bids_state: &Arc>>, ) { - log::info!( + log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), winning_bids.keys().cloned().collect::>().join(", ") ); let bid_map = build_bid_map(winning_bids, price_granularity); let bids_script = build_bids_script(&bid_map); - *ad_bids_state.write().expect("should write bid state") = Some(bids_script); + *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); +} + +/// Prepend an HTML comment summarising the auction result onto the shared +/// `ad_bids_state` so it lands directly before the injected bids `` sequences inside the string. pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { - let json = serde_json::to_string(bid_map).unwrap_or_else(|_| "{}".to_string()); + let json = serde_json::to_string(bid_map) + .expect("serde_json::to_string of Map should be infallible"); let escaped = html_escape_for_script(&json); format!( "", @@ -1338,7 +1385,8 @@ pub(crate) fn build_ad_slots_script( }) }) .collect(); - let json = serde_json::to_string(&slots).unwrap_or_else(|_| "[]".to_string()); + let json = serde_json::to_string(&slots) + .expect("serde_json::to_string of Vec should be infallible"); let escaped = html_escape_for_script(&json); format!( "", @@ -1473,58 +1521,74 @@ pub async fn handle_page_bids( .as_ref() .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Same bot / prefetch guards the publisher path uses — without them this + // endpoint would fire real SSP auctions on Sec-Purpose=prefetch warm-up + // navigations and known crawler UA scans, burning partner request quota. + let is_prefetch = is_prefetch_request(&req); + let is_bot = is_bot_user_agent(&req); + if matched_slots.is_empty() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction", path_param ); + } else if is_bot || is_prefetch { + log::debug!( + "page-bids: skipping auction for path '{}' (is_bot={}, is_prefetch={})", + path_param, + is_bot, + is_prefetch + ); } - let winning_bids = if !matched_slots.is_empty() && consent_allows_auction { - let mut auction_request = build_auction_request( - &matched_slots, - &ec_id, - &consent_context, - &request_info, - &path_param, - co_config, - req.get_header_str("user-agent"), - ); - auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); - let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } - let timeout_ms = co_config - .auction_timeout_ms - .unwrap_or(settings.auction.timeout_ms); - let auction_context = AuctionContext { - settings, - request: &req, - client_info: services.client_info(), - timeout_ms, - provider_responses: None, - services, - }; - match orchestrator - .run_auction(&auction_request, &auction_context, services) - .await - { - Ok(result) => result.winning_bids, - Err(e) => { - log::warn!("page-bids auction failed: {e:?}"); - std::collections::HashMap::new() + let winning_bids = + if !matched_slots.is_empty() && consent_allows_auction && !is_bot && !is_prefetch { + let slots_ctx = MatchedSlotsContext { + matched_slots: &matched_slots, + request_path: &path_param, + }; + let mut auction_request = build_auction_request( + &slots_ctx, + &ec_id, + &consent_context, + &request_info, + req.get_header_str("user-agent"), + ); + auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); + let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); + if client_ip.is_some() || geo.is_some() { + let device = auction_request.device.get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }); + device.ip = client_ip; + device.geo = geo.clone(); } - } - } else { - std::collections::HashMap::new() - }; + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let auction_context = AuctionContext { + settings, + request: &req, + client_info: services.client_info(), + timeout_ms, + provider_responses: None, + services, + }; + match orchestrator + .run_auction(&auction_request, &auction_context, services) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; let bid_map = build_bid_map(&winning_bids, co_config.price_granularity); @@ -2223,7 +2287,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2268,7 +2332,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2304,7 +2368,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2407,7 +2471,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2461,7 +2525,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(RwLock::new(None)), + ad_bids_state: Arc::new(Mutex::new(None)), dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), }; @@ -2527,6 +2591,7 @@ mod tests { .into_iter() .collect(), providers: Default::default(), + compiled_patterns: Vec::new(), } } @@ -2745,6 +2810,7 @@ mod tests { floor_price: Some(0.50), targeting: Default::default(), providers: Default::default(), + compiled_patterns: Vec::new(), }], } } @@ -2791,6 +2857,79 @@ mod tests { ); } + #[tokio::test] + async fn bot_user_agent_returns_slots_but_no_bids() { + // Crawlers should get slot definitions (so HTML structure is unchanged) + // but the server must not burn SSP request quota running a real auction + // for them. Same gate the publisher path applies. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = file_with_article_slot(); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "bot request should still get slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "bot request must not run an auction (no SSP cost burned for crawlers)" + ); + } + + #[tokio::test] + async fn prefetch_request_returns_slots_but_no_bids() { + // Navigations triggered by Sec-Purpose=prefetch should not fire real + // SSP auctions — the user has not yet visited the page. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let slots_file = file_with_article_slot(); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.set_header("sec-purpose", "prefetch"); + + let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + .await + .expect("should return ok response"); + + let body: serde_json::Value = + serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "prefetch request should still get slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "prefetch request must not run an auction" + ); + } + #[tokio::test] async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. From 93c1678d2ce13be9a3cad7e30954ed7a4cba0394 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 23 May 2026 23:02:54 +0530 Subject: [PATCH 070/195] Address PR review findings from #680 - Fix gpt_bootstrap.js APS beacon miss: listener now uses hb_bidder fallback when hb_adid is absent, matching the bundle's slotRenderEnded logic - Fix stale divToSlotId in inline listener: read from window.__tsDivToSlotId dynamically instead of local closure so SPA navigation updates are seen; early-return for slots not managed by Trusted Server - Populate window.__tsPrevGptSlots and window.__tsDivToSlotId from inline bootstrap so bundle's destroySlots and SPA nav path have correct state - Call installSlimPrebidLoader() in module init so the slim-Prebid lazy loader activates when __tsjs_slim_prebid_url is set; add three Vitest cases - Update /auction doc comment to distinguish /__ts/page-bids (SPA navigation) from /auction (initial render) and slim-Prebid (scroll/refresh) --- crates/js/lib/src/integrations/gpt/index.ts | 1 + .../lib/test/integrations/gpt/index.test.ts | 51 +++++++++++++++++++ .../src/auction/endpoints.rs | 12 +++-- .../src/integrations/gpt_bootstrap.js | 17 +++++-- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index fee79c1b6..611b0aeac 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -396,4 +396,5 @@ if (typeof window !== 'undefined') { installTsAdInit(); installSpaAuctionHook(); + installSlimPrebidLoader(); } diff --git a/crates/js/lib/test/integrations/gpt/index.test.ts b/crates/js/lib/test/integrations/gpt/index.test.ts index 57c4015dc..839b121d6 100644 --- a/crates/js/lib/test/integrations/gpt/index.test.ts +++ b/crates/js/lib/test/integrations/gpt/index.test.ts @@ -165,6 +165,57 @@ describe('GPT shim – patchCommandQueue', () => { }); }); +describe('GPT – installSlimPrebidLoader', () => { + type SlimWindow = Window & { __tsjs_slim_prebid_url?: string }; + + afterEach(() => { + delete (window as SlimWindow).__tsjs_slim_prebid_url; + }); + + it('is a no-op when __tsjs_slim_prebid_url is not set', async () => { + const { installSlimPrebidLoader } = await import('../../../src/integrations/gpt/index'); + const addEventListenerSpy = vi.spyOn(window, 'addEventListener'); + installSlimPrebidLoader(); + expect(addEventListenerSpy).not.toHaveBeenCalledWith('load', expect.any(Function)); + addEventListenerSpy.mockRestore(); + }); + + it('appends a deferred script tag when __tsjs_slim_prebid_url is set and load fires', async () => { + (window as SlimWindow).__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid.js'; + const { installSlimPrebidLoader } = await import('../../../src/integrations/gpt/index'); + + installSlimPrebidLoader(); + + // Simulate the window load event. + window.dispatchEvent(new Event('load')); + + const scripts = Array.from(document.querySelectorAll('script[defer]')); + const injected = scripts.find( + (s) => (s as HTMLScriptElement).src === 'https://cdn.example.com/slim-prebid.js' + ); + expect(injected).toBeDefined(); + + // Clean up + injected?.parentNode?.removeChild(injected); + }); + + it('module init calls installSlimPrebidLoader — script injected when URL is preset', async () => { + vi.resetModules(); + (window as SlimWindow).__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid-init.js'; + + await import('../../../src/integrations/gpt/index'); + window.dispatchEvent(new Event('load')); + + const scripts = Array.from(document.querySelectorAll('script[defer]')); + const injected = scripts.find( + (s) => (s as HTMLScriptElement).src === 'https://cdn.example.com/slim-prebid-init.js' + ); + expect(injected).toBeDefined(); + + injected?.parentNode?.removeChild(injected); + }); +}); + describe('GPT shim – runtime gating', () => { type GatedWindow = Window & { __tsjs_gpt_enabled?: boolean; diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 5b4e7b259..22fc11e8e 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -61,10 +61,14 @@ use super::AuctionOrchestrator; /// callers** (e.g. slim-Prebid, native apps, server-to-server integrations). /// It is **not** the intended path for scroll or GPT refresh events. /// -/// In Phase 1, slim-Prebid owns scroll and refresh: it runs post-`window.load`, -/// listens for GPT refresh events, and runs client-side auctions independently -/// of this endpoint. SPAs that use pushState routing do not trigger TS page-level -/// auctions — slim-Prebid handles those cases too. +/// **SPA navigation** is handled by `GET /__ts/page-bids`: the client-side SPA +/// hook (`installSpaAuctionHook`) intercepts `pushState`/`replaceState`/`popstate` +/// events and calls that endpoint to fetch fresh slots and bids for each new +/// route, then invokes `window.__tsAdInit()` with the updated data. +/// +/// **Scroll and GPT refresh** are owned by slim-Prebid in Phase 1: it runs +/// post-`window.load`, listens for GPT refresh events, and runs client-side +/// auctions independently of this endpoint. /// /// A slot-template-aware refresh API (`POST /auction/refresh`) is deferred to a /// future phase and not designed here. diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index a3d28a286..85109d724 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -47,6 +47,11 @@ divToSlotId[slot.div_id] = slot.id; newSlots.push(s); }); + // Expose slot metadata on window so later calls (SPA navigation, + // the bundle's __tsAdInit) can destroy stale slots and the render + // listener can resolve slot IDs after navigation updates these maps. + window.__tsPrevGptSlots = newSlots; + window.__tsDivToSlotId = divToSlotId; // Guard the one-time-per-page setup so a follow-up call (e.g. // publisher's own init code or the bundle's `__tsAdInit` after // it overwrites this stub) doesn't double-enable services. @@ -58,12 +63,18 @@ .pubads() .addEventListener("slotRenderEnded", function (ev) { var divId = ev.slot.getSlotElementId(); - var slotId = divToSlotId[divId] || divId; + // Read from window so SPA navigation updates are picked up; + // early-return for slots not managed by Trusted Server. + var slotId = (window.__tsDivToSlotId || {})[divId]; + if (!slotId) return; var b = (window.__ts_bids || {})[slotId] || {}; + // Prebid: verify the specific creative via hb_adid targeting. + // APS: no hb_adid — fire if any TS bidder is present and slot is non-empty. var ourBidWon = !ev.isEmpty && - b.hb_adid && - ev.slot.getTargeting("hb_adid")[0] === b.hb_adid; + (b.hb_adid + ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid + : !!b.hb_bidder); if (ourBidWon) { if (b.nurl) navigator.sendBeacon(b.nurl); if (b.burl) navigator.sendBeacon(b.burl); From 0346330905b3e5e8487ecb566baac6c2504d5214 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Tue, 26 May 2026 09:42:33 -0700 Subject: [PATCH 071/195] Formatting --- crates/trusted-server-core/src/integrations/sourcepoint.rs | 4 ++-- creative-opportunities.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index a911b5d5a..adea7b446 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -1073,9 +1073,9 @@ mod tests { let integration = SourcepointIntegration::new(Arc::new(config(true))); let document_state = IntegrationDocumentState::default(); let ctx = IntegrationHtmlContext { - request_host: "ts.autoblog.com", + request_host: "ts.examnple.com", request_scheme: "https", - origin_host: "origin.autoblog.com", + origin_host: "origin.examnple.com", document_state: &document_state, }; diff --git a/creative-opportunities.toml b/creative-opportunities.toml index b6ed8900f..da1ed23e7 100644 --- a/creative-opportunities.toml +++ b/creative-opportunities.toml @@ -3,7 +3,7 @@ [[slot]] id = "atf_sidebar_ad" -gam_unit_path = "/88059007/autoblog/news" +gam_unit_path = "/a/b/news" div_id = "ad-atf_sidebar-0-_r_2_" page_patterns = ["/20**", "/news/**"] formats = [{ width = 300, height = 250 }] @@ -18,7 +18,7 @@ slot_id = "aps-slot-atf-sidebar" [[slot]] id = "homepage_header_ad" -gam_unit_path = "/88059007/autoblog/homepage" +gam_unit_path = "/a/b/homepage" div_id = "ad-header-0-_R_jpalubtak5lb_" page_patterns = ["/"] formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }, { width = 970, height = 250 }] @@ -33,7 +33,7 @@ slot_id = "aps-slot-homepage-header" [[slot]] id = "homepage_footer_ad" -gam_unit_path = "/88059007/autoblog/homepage" +gam_unit_path = "/a/b/homepage" div_id = "ad-fixed_bottom-0-_R_klubtak5lb_" page_patterns = ["/"] formats = [{ width = 728, height = 90 }] From ca98985b003eb70931e5f52d8ff734deac92b140 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 28 May 2026 15:36:29 +0530 Subject: [PATCH 072/195] Address pass-4 review findings (#680) - Fix examnple.com typo in sourcepoint.rs test fixture - Guard SPA navigation race: check inflight controller identity after await res.json() before writing __ts_ad_slots/__ts_bids - Extract build_empty_bids_script() helper; html_processor.rs now calls it instead of duplicating the inline literal - Add invariant comment to unreachable None branch of prepend_auction_debug_comment - Cap parse_ts_eids_cookie to 32 eids / 32 uids per eid; log and return None when exceeded - Add #[serde(deny_unknown_fields)] to openrtb::Eid and Uid - Add #[serde(deny_unknown_fields)] to CreativeOpportunitiesFile and CreativeOpportunitySlot - Log debug message when adserver_mock crid does not match -creative convention - Skip zero-dimension bids in adserver_mock with debug log - Fail closed in APS parse_aps_slot on malformed size string instead of producing 0x0 bid --- crates/js/lib/src/integrations/gpt/index.ts | 1 + crates/trusted-server-core/src/cookies.rs | 8 +++++++- .../src/creative_opportunities.rs | 2 ++ .../trusted-server-core/src/html_processor.rs | 11 +++++----- .../src/integrations/adserver_mock.rs | 20 ++++++++++++++++--- .../src/integrations/aps.rs | 12 ++++++++++- .../src/integrations/sourcepoint.rs | 4 ++-- crates/trusted-server-core/src/openrtb.rs | 2 ++ crates/trusted-server-core/src/publisher.rs | 10 ++++++++++ 9 files changed, 57 insertions(+), 13 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 611b0aeac..e1a1ee267 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -328,6 +328,7 @@ export function installSpaAuctionHook(): void { }); if (!res.ok) return; const data = (await res.json()) as PageBidsResponse; + if (inflight !== controller) return; win.__ts_ad_slots = data.slots; win.__ts_bids = data.bids; win.__tsAdInit?.(); diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 4f0e7f9c0..91f92d830 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -141,7 +141,13 @@ pub(crate) fn parse_ts_eids_cookie(jar: Option<&CookieJar>) -> Option>(&decoded) { - Ok(eids) if !eids.is_empty() => Some(eids), + Ok(eids) if !eids.is_empty() => { + if eids.len() > 32 || eids.iter().any(|e| e.uids.len() > 32) { + log::debug!("ts-eids cookie: too many eids or uids, rejecting"); + return None; + } + Some(eids) + } Ok(_) => None, Err(e) => { log::debug!("ts-eids cookie: JSON parse failed: {e}"); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cbf79b114..95180041e 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -42,6 +42,7 @@ pub struct CreativeOpportunitiesConfig { /// A single ad placement opportunity on the publisher's site. #[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunitySlot { /// Unique identifier for the slot (e.g., `"atf"`, `"below-fold-sidebar"`). pub id: String, @@ -224,6 +225,7 @@ pub struct ApsSlotParams { /// TOML file structure for creative opportunity slot definitions. #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesFile { /// All slot definitions in the file (mapped from `[[slot]]` TOML arrays). #[serde(rename = "slot", default)] diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index c54c46897..6005e3cc3 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -34,6 +34,7 @@ use crate::integrations::{ IntegrationHtmlContext, IntegrationHtmlPostProcessor, IntegrationRegistry, IntegrationScriptContext, ScriptRewriteAction, }; +use crate::publisher::build_empty_bids_script; use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; use crate::tsjs; @@ -328,21 +329,19 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let state = state.clone(); let injected_bids = injected_bids.clone(); if let Some(handlers) = el.end_tag_handlers() { - let handler: EndTagHandler<'static> = Box::new( - move |end_tag: &mut EndTag<'_>| { + let handler: EndTagHandler<'static> = + Box::new(move |end_tag: &mut EndTag<'_>| { if injected_bids.swap(true, Ordering::SeqCst) { return Ok(()); } let script_guard = state.lock().expect("should lock bid state"); let bids_script = match &*script_guard { Some(s) => s.clone(), - None => r#""# - .to_string(), + None => build_empty_bids_script(), }; end_tag.before(&bids_script, ContentType::Html); Ok(()) - }, - ); + }); handlers.push(handler); } Ok(()) diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 1d968484c..beacef1df 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -251,17 +251,31 @@ impl AdServerMockProvider { // Recover bidder name from crid ("{bidder}-creative") to look up the // original SSP bid and restore nurl/burl/ad_id the mediator drops. let crid = bid["crid"].as_str().unwrap_or(""); - let bidder = crid.strip_suffix("-creative").unwrap_or(""); + let bidder = crid.strip_suffix("-creative").unwrap_or_else(|| { + log::debug!( + "adserver_mock: crid '{crid}' does not match '-creative' — dropping nurl/burl/ad_id" + ); + "" + }); let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); let original = bid_index.get(&key); + let width = bid["w"].as_u64().unwrap_or(0) as u32; + let height = bid["h"].as_u64().unwrap_or(0) as u32; + if width == 0 || height == 0 { + log::debug!( + "adserver_mock: bid for slot '{slot_id}' has zero dimension ({width}×{height}), skipping" + ); + continue; + } + all_bids.push(Bid { slot_id, price: bid["price"].as_f64(), currency: "USD".to_string(), creative: bid["adm"].as_str().map(String::from), - width: bid["w"].as_u64().unwrap_or(0) as u32, - height: bid["h"].as_u64().unwrap_or(0) as u32, + width, + height, bidder: seat_name.to_string(), adomain: bid["adomain"].as_array().map(|arr| { arr.iter() diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 71ef2bbe7..304f61a06 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -406,7 +406,17 @@ impl ApsAuctionProvider { } // Parse size from "WxH" format - let (width, height) = Self::parse_size(&slot.size).unwrap_or((0, 0)); + let (width, height) = match Self::parse_size(&slot.size) { + Some(dims) => dims, + None => { + log::debug!( + "APS: slot '{}' has malformed size '{}', skipping", + slot.slot_id, + slot.size + ); + return Err(()); + } + }; // Build metadata from targeting keys - includes encoded price for mediation let mut metadata = HashMap::new(); diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index adea7b446..b48075a6a 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -1073,9 +1073,9 @@ mod tests { let integration = SourcepointIntegration::new(Arc::new(config(true))); let document_state = IntegrationDocumentState::default(); let ctx = IntegrationHtmlContext { - request_host: "ts.examnple.com", + request_host: "ts.example.com", request_scheme: "https", - origin_host: "origin.examnple.com", + origin_host: "origin.example.com", document_state: &document_state, }; diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 63d63435c..aff580608 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -76,6 +76,7 @@ pub struct ConsentedProvidersSettings { /// An Extended User ID entry from an identity provider. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, @@ -85,6 +86,7 @@ pub struct Eid { /// A single user identifier within an [`Eid`] entry. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Uid { /// The identifier value. pub id: String, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8908eaf09..c6ffa7761 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -651,6 +651,8 @@ pub(crate) fn prepend_auction_debug_comment( *script = format!("{debug_comment}\n{script}"); } None => { + // invariant: write_bids_to_state is always called before this and + // always sets Some(_); this branch is unreachable in production. *state = Some(debug_comment); } } @@ -1353,6 +1355,14 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map` tag used when no bids were returned. +/// +/// Shares the same shape as [`build_bids_script`] so any change to the script +/// format stays in one place. +pub(crate) fn build_empty_bids_script() -> String { + build_bids_script(&serde_json::Map::new()) +} + /// Build the `__ts_ad_slots` ``. + /// Pre-computed ``. /// Injected at `` open. `None` when no slots matched. pub ad_slots_script: Option, /// Shared auction result — written by auction task before HTML processing begins. /// Handler reads this in `el.on_end_tag()` on the body element. - /// `None` means no auction ran; inject empty `__ts_bids = {}` as fallback. + /// `None` means no auction ran; inject empty `tsjs.bids = {}` as fallback. pub ad_bids_state: std::sync::Arc>>, } @@ -311,10 +311,10 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) } }), - // Inject __ts_bids before via end_tag_handlers — only when + // Inject tsjs.bids before via end_tag_handlers — only when // slots matched this URL. When no slots matched, skip injection entirely // so the publisher's existing client-side Prebid/GPT flow is unmodified - // (dual-mode rollout: calling __tsAdInit with empty slots would invoke + // (dual-mode rollout: calling tsjs.adInit with empty slots would invoke // enableSingleRequest/enableServices and conflict with the publisher's GPT init). // Guard with AtomicBool so the script is only injected once even if // the origin HTML contains multiple elements (e.g. template fragments). @@ -1278,7 +1278,8 @@ mod tests { request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: Some( - r#""#.to_string(), + r#""# + .to_string(), ), ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), }; @@ -1291,8 +1292,12 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - html.contains("window.__ts_ad_slots"), - "should inject ad slots at head-open" + html.contains("window.tsjs=window.tsjs||{}"), + "should inject ad slots namespace at head-open" + ); + assert!( + html.contains(".adSlots=JSON.parse"), + "should inject adSlots at head-open" ); assert!( !html.contains("__ts_request_id"), @@ -1302,15 +1307,16 @@ mod tests { #[test] fn injects_ts_bids_before_body_close() { - let bids_script = - r#""#; + let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: Some("".to_string()), + ad_slots_script: Some( + r#""#.to_string(), + ), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1319,27 +1325,32 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - html.contains("window.__ts_bids"), + html.contains("window.tsjs=window.tsjs||{}"), + "should inject _ts namespace for bids before " + ); + assert!( + html.contains(".bids=JSON.parse"), "should inject bids before " ); let bids_pos = html - .find("window.__ts_bids") - .expect("bids should be in output"); + .find("window.tsjs=window.tsjs||{}") + .expect("bids namespace should be in output"); let body_close_pos = html.find("").expect(" should be in output"); assert!(bids_pos < body_close_pos, "bids must appear before "); } #[test] fn injects_ts_bids_only_once_with_multiple_body_elements() { - let bids_script = - r#""#; + let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: Some("".to_string()), + ad_slots_script: Some( + r#""#.to_string(), + ), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1349,9 +1360,9 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert_eq!( - html.matches("window.__ts_bids").count(), + html.matches(".bids=JSON.parse").count(), 1, - "should inject __ts_bids exactly once even with multiple elements" + "should inject tsjs.bids exactly once even with multiple elements" ); } @@ -1365,7 +1376,9 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), - ad_slots_script: Some("".to_string()), + ad_slots_script: Some( + r#""#.to_string(), + ), ad_bids_state: state, }; let mut processor = create_html_processor(config); @@ -1374,14 +1387,14 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - html.contains("__ts_bids=JSON.parse(\"{}\")"), + html.contains(".bids=JSON.parse(\"{}\")"), "should inject empty bids fallback when auction produced nothing" ); } #[test] fn does_not_inject_ts_bids_when_no_slots_matched() { - // No slots matched this URL — ad_slots_script is None. __ts_bids must be + // No slots matched this URL — ad_slots_script is None. tsjs.bids must be // omitted entirely so the publisher's existing client-side GPT flow is // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); @@ -1399,8 +1412,8 @@ mod tests { .expect("should process"); let html = std::str::from_utf8(&output).expect("should be utf8"); assert!( - !html.contains("__ts_bids"), - "should NOT inject __ts_bids when no slots matched" + !html.contains(".bids=JSON.parse"), + "should NOT inject tsjs.bids when no slots matched" ); } } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index beacef1df..483e4499c 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -89,7 +89,7 @@ impl IntegrationConfig for AdServerMockConfig { // ============================================================================ /// Lookup index built from original SSP bids during `request_bids`, consumed -/// during `parse_response` to restore `nurl`/`burl`/`ad_id` that the mock +/// during `parse_response` to restore render/accounting fields that the mock /// mediator endpoint does not echo back. /// /// Keyed by `(provider_name, slot_id, bidder_name)`. @@ -98,7 +98,7 @@ type BidIndex = HashMap<(String, String, String), Bid>; /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, - /// Bridges SSP bid metadata (`nurl`/`burl`/`ad_id`) from `request_bids` to `parse_response`. + /// Bridges SSP bid metadata from `request_bids` to `parse_response`. bid_index: Mutex>, } @@ -226,7 +226,7 @@ impl AdServerMockProvider { /// Mediation returns decoded prices for all bids (including APS bids that were encoded). /// /// `bid_index` is the SSP-bid lookup built in `request_bids`. The mock mediator - /// does not echo `nurl`/`burl`/`ad_id` back, so they are restored from the index + /// does not echo render/accounting fields back, so they are restored from the index /// using `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` /// field (`"{bidder}-creative"` format set during request construction). fn parse_mediation_response( @@ -249,16 +249,18 @@ impl AdServerMockProvider { let slot_id = bid["impid"].as_str().unwrap_or("").to_string(); // Recover bidder name from crid ("{bidder}-creative") to look up the - // original SSP bid and restore nurl/burl/ad_id the mediator drops. + // original SSP bid and restore render/accounting fields the mediator drops. let crid = bid["crid"].as_str().unwrap_or(""); let bidder = crid.strip_suffix("-creative").unwrap_or_else(|| { log::debug!( - "adserver_mock: crid '{crid}' does not match '-creative' — dropping nurl/burl/ad_id" + "adserver_mock: crid '{crid}' does not match '-creative'; render/accounting fields may be missing" ); "" }); let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); let original = bid_index.get(&key); + let restored_bidder = + original.map_or_else(|| seat_name.to_string(), |b| b.bidder.clone()); let width = bid["w"].as_u64().unwrap_or(0) as u32; let height = bid["h"].as_u64().unwrap_or(0) as u32; @@ -276,7 +278,7 @@ impl AdServerMockProvider { creative: bid["adm"].as_str().map(String::from), width, height, - bidder: seat_name.to_string(), + bidder: restored_bidder, adomain: bid["adomain"].as_array().map(|arr| { arr.iter() .filter_map(|v| v.as_str().map(String::from)) @@ -285,6 +287,9 @@ impl AdServerMockProvider { nurl: original.and_then(|b| b.nurl.clone()), burl: original.and_then(|b| b.burl.clone()), ad_id: original.and_then(|b| b.ad_id.clone()), + cache_id: original.and_then(|b| b.cache_id.clone()), + cache_host: original.and_then(|b| b.cache_host.clone()), + cache_path: original.and_then(|b| b.cache_path.clone()), metadata: HashMap::new(), }); } @@ -563,6 +568,9 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }], response_time_ms: 150, @@ -583,6 +591,9 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }], response_time_ms: 120, @@ -656,6 +667,98 @@ mod tests { assert_eq!(bid.height, 90); } + #[test] + fn parse_mediation_response_restores_original_bid_render_fields() { + let provider = AdServerMockProvider::new(AdServerMockConfig::default()); + let mediation_response = json!({ + "id": "test-auction-123", + "seatbid": [ + { + "seat": "prebid", + "bid": [ + { + "id": "mediated-bid-001", + "impid": "header-banner", + "price": 0.20, + "adm": "
Mediated Ad
", + "w": 728, + "h": 90, + "crid": "mocktioneer-creative", + "adomain": ["example.com"] + } + ] + } + ], + "cur": "USD" + }); + let mut bid_index = BidIndex::new(); + bid_index.insert( + ( + "prebid".to_string(), + "header-banner".to_string(), + "mocktioneer".to_string(), + ), + Bid { + slot_id: "header-banner".to_string(), + price: Some(0.20), + currency: "USD".to_string(), + creative: Some("
Original Ad
".to_string()), + adomain: Some(vec!["example.com".to_string()]), + bidder: "mocktioneer".to_string(), + width: 728, + height: 90, + nurl: Some("https://ssp.example/win".to_string()), + burl: Some("https://ssp.example/bill".to_string()), + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example".to_string()), + cache_path: Some("/cache".to_string()), + metadata: HashMap::new(), + }, + ); + + let auction_response = + provider.parse_mediation_response(&mediation_response, 42, &bid_index); + + assert_eq!(auction_response.status, BidStatus::Success); + assert_eq!(auction_response.bids.len(), 1); + let bid = &auction_response.bids[0]; + assert_eq!( + bid.bidder, "mocktioneer", + "should preserve underlying bidder for hb_bidder targeting" + ); + assert_eq!( + bid.nurl.as_deref(), + Some("https://ssp.example/win"), + "should restore nurl" + ); + assert_eq!( + bid.burl.as_deref(), + Some("https://ssp.example/bill"), + "should restore burl" + ); + assert_eq!( + bid.ad_id.as_deref(), + Some("bid-impression-id"), + "should restore ad_id" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid"), + "should restore PBS cache UUID" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("cache.example"), + "should restore PBS cache host" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should restore PBS cache path" + ); + } + #[test] fn test_parse_empty_mediation_response() { let config = AdServerMockConfig::default(); @@ -727,6 +830,9 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: aps_metadata, }], response_time_ms: 100, diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 304f61a06..d1c449bf5 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -451,6 +451,9 @@ impl ApsAuctionProvider { nurl: None, // Real APS uses client-side event tracking burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata, }) } diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index cb0994029..5f88f69c6 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -81,6 +81,16 @@ pub struct GptConfig { /// Whether to rewrite GPT script URLs in publisher HTML. #[serde(default = "default_rewrite_script")] pub rewrite_script: bool, + + /// URL for the slim-Prebid bundle loaded post-window.load. + /// + /// When set, `installSlimPrebidLoader()` in the GPT bundle will load this + /// script after `window.load`, enabling scroll/refresh client-side auctions + /// and userID module warm-up. Set to the publisher's tsjs-prebid bundle URL. + /// + /// Override via env var: `TRUSTED_SERVER__INTEGRATIONS__GPT__SLIM_PREBID_URL` + #[serde(default, skip_serializing_if = "Option::is_none")] + pub slim_prebid_url: Option, } impl IntegrationConfig for GptConfig { @@ -437,11 +447,11 @@ impl IntegrationHeadInjector for GptIntegration { GPT_INTEGRATION_ID } - /// Injects the `__tsAdInit` bootstrap script into ``. + /// Injects the `tsjs.adInit` bootstrap script into ``. /// /// ## Scroll / refresh handoff contract (Phase 1) /// - /// `__tsAdInit` handles **initial render only**: it wires server-side bid + /// `tsjs.adInit` handles **initial render only**: it wires server-side bid /// targeting into GPT slots and fires win beacons (`nurl`/`burl`) via /// `slotRenderEnded`. It does **not** trigger refresh auctions or handle /// GPT slot refresh events. @@ -451,22 +461,31 @@ impl IntegrationHeadInjector for GptIntegration { /// impressions. SPA pushState navigation is also slim-Prebid's domain. /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - vec![ + let mut scripts = vec![ "" .to_string(), format!("", GPT_BOOTSTRAP_JS), - ] + ]; + + if let Some(ref url) = self.config.slim_prebid_url { + scripts.push(format!( + "", + serde_json::to_string(url).expect("should serialize string") + )); + } + + scripts } } -/// Inline `window.__tsAdInit` bootstrap injected at `` so the bids +/// Inline `window.tsjs.adInit` bootstrap injected at `` so the bids /// script at `` can call it before the TSJS bundle has loaded. /// /// The bundle's idempotent implementation in /// `crates/js/lib/src/integrations/gpt/index.ts` later overwrites this stub. /// Both implementations guard the one-time-per-page setup with -/// `window.__tsServicesEnabled` so neither double-enables services if the +/// `window.tsjs.servicesEnabled` so neither double-enables services if the /// publisher's own init code also calls `googletag.enableServices()`. const GPT_BOOTSTRAP_JS: &str = include_str!("gpt_bootstrap.js"); @@ -502,6 +521,7 @@ mod tests { script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, + slim_prebid_url: None, } } @@ -1062,10 +1082,10 @@ mod tests { }; let inserts = integration.head_inserts(&ctx); let combined = inserts.join(""); - assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); + assert!(combined.contains("ts.adInit"), "should define tsjs.adInit"); assert!( - combined.contains("window.__ts_bids"), - "should read window.__ts_bids synchronously" + combined.contains("ts.bids"), + "should read tsjs.bids synchronously" ); assert!( combined.contains("ts_initial"), @@ -1110,13 +1130,10 @@ mod tests { }; let combined = integration.head_inserts(&ctx).join(""); assert!( - combined.contains("__tsServicesEnabled"), - "should guard enableServices/enableSingleRequest with the __tsServicesEnabled flag" - ); - assert!( - combined.contains("window.__tsAdInit"), - "should install __tsAdInit on window" + combined.contains("ts.servicesEnabled"), + "should guard enableServices/enableSingleRequest with the tsjs.servicesEnabled flag" ); + assert!(combined.contains("ts.adInit"), "should install tsjs.adInit"); assert!( !combined.contains("googletag.pubads().refresh()"), "should never call unbounded refresh() — only refresh(newSlots)" @@ -1131,4 +1148,59 @@ mod tests { "gpt" ); } + + #[test] + fn head_inserts_emits_slim_prebid_url_when_configured() { + let config = GptConfig { + slim_prebid_url: Some("https://cdn.example.com/tsjs-prebid.min.js".to_string()), + ..test_config() + }; + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + assert_eq!( + inserts.len(), + 3, + "should emit three head inserts when slim_prebid_url is set" + ); + assert_eq!( + inserts[2], + r#""#, + "should emit the slim-Prebid URL as a JSON-encoded string assignment" + ); + } + + #[test] + fn head_inserts_omits_slim_prebid_url_when_not_configured() { + let integration = GptIntegration::new(test_config()); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + assert_eq!( + inserts.len(), + 2, + "should emit exactly two head inserts when slim_prebid_url is absent" + ); + assert!( + inserts + .iter() + .all(|s| !s.contains("__tsjs_slim_prebid_url")), + "should not emit slim-Prebid URL tag when not configured" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 85109d724..0c7ea0dd2 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -1,88 +1,108 @@ // Edge-injected GPT auction bootstrap. // -// This is the minimal `window.__tsAdInit` that runs on first page load +// This is the minimal `window.tsjs.adInit` that runs on first page load // before the TSJS bundle has had a chance to install its richer // idempotent implementation. The bundle in -// crates/js/lib/src/integrations/gpt/index.ts overwrites `__tsAdInit` +// crates/js/lib/src/integrations/gpt/index.ts overwrites `tsjs.adInit` // once it loads. // // Contract with the bundle: -// - Both implementations must set `window.__tsServicesEnabled = true` +// - Both implementations must set `window.tsjs.servicesEnabled = true` // after calling `enableSingleRequest()`/`enableServices()` so a -// subsequent call from any source (the bundle's `__tsAdInit`, the -// publisher's own GPT init code) becomes a no-op. +// subsequent call becomes a no-op. // - `refresh()` is called only for the slots defined in this pass, -// never the global slot list, so we never accidentally refresh -// publisher-managed slots that we don't own. +// never the global slot list. // -// Only installed if `window.__tsAdInit` isn't already defined — that -// way the bundle (or anything else) can preempt this fallback by -// installing first. +// Only installed if `window.tsjs.adInit` isn't already defined. (function () { - if (typeof window === "undefined" || window.__tsAdInit) { - return; - } - window.__tsAdInit = function () { - var slots = window.__ts_ad_slots || []; - var bids = window.__ts_bids || {}; + if (typeof window === "undefined") return; + var ts = (window.tsjs = window.tsjs || {}); + if (ts.adInit) return; + + ts.adInit = function () { + var slots = ts.adSlots || []; + var bids = ts.bids || {}; var divToSlotId = {}; + googletag.cmd.push(function () { + // Slots TS defined itself — tracked for SPA destroy. Publisher-owned + // slots are reused but never destroyed by TS on navigation. var newSlots = []; + // All slots to refresh (TS-defined + publisher-owned reused). + var slotsToRefresh = []; slots.forEach(function (slot) { - var s = googletag.defineSlot( - slot.gam_unit_path, - slot.formats, - slot.div_id, - ); - if (!s) return; - s.addService(googletag.pubads()); + // Resolve actual div ID: exact match first, then prefix query. + // div_id in config may be a stable prefix (e.g. "ad-header-0-") when + // the suffix is dynamically generated by the framework at render time. + var el = + document.getElementById(slot.div_id) || + document.querySelector( + "[id^='" + slot.div_id + "']:not([id$='-container'])", + ); + if (!el) return; + var actualDivId = el.id; + var b = bids[slot.id] || {}; + + var existingSlots = googletag.pubads().getSlots(); + var s = + existingSlots.find(function (gs) { + return gs.getSlotElementId() === actualDivId; + }) || null; + var tsOwned = false; + if (!s) { + // Use outer container div for TS's slot when publisher hasn't defined + // theirs yet — keeps both slots on separate divs so publisher's + // later defineSlot on the inner div doesn't conflict. + var containerEl = document.getElementById(actualDivId + "-container"); + var slotDivId = containerEl ? containerEl.id : actualDivId; + s = googletag.defineSlot(slot.gam_unit_path, slot.formats, slotDivId); + if (!s) return; + s.addService(googletag.pubads()); + tsOwned = true; + } + Object.entries(slot.targeting || {}).forEach(function (e) { s.setTargeting(e[0], e[1]); }); - var b = bids[slot.id] || {}; - ["hb_pb", "hb_bidder", "hb_adid"].forEach(function (k) { + [ + "hb_pb", + "hb_bidder", + "hb_adid", + "hb_cache_host", + "hb_cache_path", + ].forEach(function (k) { if (b[k]) s.setTargeting(k, b[k]); }); + // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); - divToSlotId[slot.div_id] = slot.id; - newSlots.push(s); + divToSlotId[actualDivId] = slot.id; + if (tsOwned) newSlots.push(s); + slotsToRefresh.push(s); }); - // Expose slot metadata on window so later calls (SPA navigation, - // the bundle's __tsAdInit) can destroy stale slots and the render - // listener can resolve slot IDs after navigation updates these maps. - window.__tsPrevGptSlots = newSlots; - window.__tsDivToSlotId = divToSlotId; - // Guard the one-time-per-page setup so a follow-up call (e.g. - // publisher's own init code or the bundle's `__tsAdInit` after - // it overwrites this stub) doesn't double-enable services. - if (!window.__tsServicesEnabled) { + ts.prevGptSlots = newSlots; + ts.divToSlotId = divToSlotId; + if (!ts.servicesEnabled) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); - window.__tsServicesEnabled = true; - googletag - .pubads() - .addEventListener("slotRenderEnded", function (ev) { - var divId = ev.slot.getSlotElementId(); - // Read from window so SPA navigation updates are picked up; - // early-return for slots not managed by Trusted Server. - var slotId = (window.__tsDivToSlotId || {})[divId]; - if (!slotId) return; - var b = (window.__ts_bids || {})[slotId] || {}; - // Prebid: verify the specific creative via hb_adid targeting. - // APS: no hb_adid — fire if any TS bidder is present and slot is non-empty. - var ourBidWon = - !ev.isEmpty && - (b.hb_adid - ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid - : !!b.hb_bidder); - if (ourBidWon) { - if (b.nurl) navigator.sendBeacon(b.nurl); - if (b.burl) navigator.sendBeacon(b.burl); - } - }); + ts.servicesEnabled = true; + googletag.pubads().addEventListener("slotRenderEnded", function (ev) { + var divId = ev.slot.getSlotElementId(); + var slotId = (ts.divToSlotId || {})[divId]; + if (!slotId) return; + var b = (ts.bids || {})[slotId] || {}; + var ourBidWon = + !ev.isEmpty && + (b.hb_adid + ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid + : !!b.hb_bidder); + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl); + if (b.burl) navigator.sendBeacon(b.burl); + } + }); } - if (newSlots.length > 0) { - googletag.pubads().refresh(newSlots); + if (slotsToRefresh.length > 0) { + googletag.pubads().refresh(slotsToRefresh); } }); }; diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index b74b234ca..1ab937baa 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -8,6 +8,7 @@ use fastly::http::{header, Method, StatusCode, Url}; use fastly::{Request, Response}; use serde::{Deserialize, Serialize}; use serde_json::Value as Json; +use url::Url as ParsedUrl; use validator::Validate; use crate::auction::provider::AuctionProvider; @@ -1374,6 +1375,49 @@ impl PrebidAuctionProvider { .collect() }); + // Extract PBS Cache coordinates from ext.prebid.cache.bids + let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + + let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + + let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + ParsedUrl::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + // path() returns "/" for root — only use if non-trivial + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { + None + } else { + Some(path) + }; + (host, path) + }) + .unwrap_or((None, None)); + + // Guard: if we extracted a cache UUID but couldn't extract the host, + // the bid will have hb_adid set but no endpoint to fetch from — creative will fail. + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{}'", + slot_id + ); + } + Ok(AuctionBid { slot_id, price: Some(price), // Prebid provides decoded prices @@ -1386,6 +1430,9 @@ impl PrebidAuctionProvider { nurl, burl, ad_id, + cache_id, + cache_host, + cache_path, metadata: std::collections::HashMap::new(), }) } @@ -4339,4 +4386,137 @@ set = { networkId = 42 } "should fail fast when a canonical rule has no matcher fields" ); } + + #[test] + fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { + let bid_json = serde_json::json!({ + "id": "bid-id-123", + "impid": "atf_sidebar_ad", + "price": 1.50, + "adm": "
ad
", + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "thetradedesk") + .expect("should parse bid"); + assert_eq!( + bid.cache_id.as_deref(), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should extract cacheId as cache_id" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("openads.adsrvr.org"), + "should extract host from cache URL" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should extract path from cache URL" + ); + } + + #[test] + fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { + let bid_json = serde_json::json!({ + "id": "bid-id-456", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250 + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert!(bid.cache_id.is_none(), "should be None when cache absent"); + assert!(bid.cache_host.is_none(), "should be None when cache absent"); + assert!(bid.cache_path.is_none(), "should be None when cache absent"); + } + + #[test] + fn parse_bid_handles_malformed_cache_url_gracefully() { + let bid_json = serde_json::json!({ + "id": "bid-id-789", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "not-a-valid-url", + "cacheId": "some-uuid" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid without panicking"); + assert_eq!( + bid.cache_id.as_deref(), + Some("some-uuid"), + "should still extract cacheId even if URL is malformed" + ); + assert!( + bid.cache_host.is_none(), + "should be None when URL parse fails" + ); + assert!( + bid.cache_path.is_none(), + "should be None when URL parse fails" + ); + } + + #[test] + fn parse_bid_preserves_ad_id_alongside_cache_id() { + let bid_json = serde_json::json!({ + "id": "bid-impression-id", + "impid": "atf_sidebar_ad", + "adid": "bidder-ad-id-abc", + "price": 1.0, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://cache.example.com/cache", + "cacheId": "cache-uuid-xyz" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.ad_id.as_deref(), + Some("bidder-ad-id-abc"), + "should keep ad_id from adid field" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid-xyz"), + "should extract cache UUID separately" + ); + } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c6ffa7761..12a368c47 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -428,7 +428,7 @@ pub struct OwnedProcessResponseParams { /// The streaming phase collects these and writes bids to `ad_bids_state` /// before processing the last body chunk, so `` injection sees live bids. pub(crate) dispatched_auction: Option, - /// Price granularity used to bucket bids when building `__ts_bids`. + /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, } @@ -516,6 +516,7 @@ pub async fn stream_publisher_body_async( &result.winning_bids, params.price_granularity, ¶ms.ad_bids_state, + settings.debug.inject_adm_for_testing, ); return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -611,13 +612,14 @@ pub(crate) fn write_bids_to_state( winning_bids: &std::collections::HashMap, price_granularity: PriceGranularity, ad_bids_state: &Arc>>, + inject_adm: bool, ) { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), winning_bids.keys().cloned().collect::>().join(", ") ); - let bid_map = build_bid_map(winning_bids, price_granularity); + let bid_map = build_bid_map(winning_bids, price_granularity, inject_adm); let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); } @@ -757,7 +759,12 @@ async fn one_behind_loop( "one_behind_loop: collect complete — {} winning bid(s)", result.winning_bids.len() ); - write_bids_to_state(&result.winning_bids, price_granularity, ad_bids_state); + write_bids_to_state( + &result.winning_bids, + price_granularity, + ad_bids_state, + settings.debug.inject_adm_for_testing, + ); if settings.debug.auction_html_comment { prepend_auction_debug_comment("stream", &result, ad_bids_state); @@ -853,7 +860,7 @@ pub async fn handle_publisher_request( integration_registry: &IntegrationRegistry, services: &RuntimeServices, orchestrator: &AuctionOrchestrator, - slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], mut req: Request, ) -> Result> { log::debug!("Proxying request to publisher_origin"); @@ -939,7 +946,7 @@ pub async fn handle_publisher_request( let is_bot = is_bot_user_agent(&req); let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { - crate::creative_opportunities::match_slots(&slots_file.slots, &request_path) + crate::creative_opportunities::match_slots(slots, &request_path) .into_iter() .cloned() .collect() @@ -1192,7 +1199,12 @@ pub async fn handle_publisher_request( "BufferedProcessed: auction collected — {} winning bid(s)", result.winning_bids.len() ); - write_bids_to_state(&result.winning_bids, price_granularity, &ad_bids_state); + write_bids_to_state( + &result.winning_bids, + price_granularity, + &ad_bids_state, + settings.debug.inject_adm_for_testing, + ); if settings.debug.auction_html_comment { prepend_auction_debug_comment("buffered", &result, &ad_bids_state); @@ -1311,6 +1323,7 @@ fn html_escape_for_script(s: &str) -> String { pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, + include_adm: bool, ) -> serde_json::Map { winning_bids .iter() @@ -1323,10 +1336,30 @@ pub(crate) fn build_bid_map( "hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone()), ); - if let Some(ref ad_id) = bid.ad_id { + // hb_adid: use PBS Cache UUID when present — the Prebid Universal Creative uses + // this as the cache lookup key, NOT the OpenRTB bid ID (bid.ad_id). Fall back to + // bid.ad_id for APS and other non-PBS providers. + let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); + if let Some(id) = hb_adid { obj.insert( "hb_adid".to_string(), - serde_json::Value::String(ad_id.clone()), + serde_json::Value::String(id.to_string()), + ); + } + + // Cache endpoint coordinates — only present for PBS bids with Prebid Cache enabled. + // The Prebid Universal Creative constructs: + // https://?uuid= + if let Some(ref host) = bid.cache_host { + obj.insert( + "hb_cache_host".to_string(), + serde_json::Value::String(host.clone()), + ); + } + if let Some(ref path) = bid.cache_path { + obj.insert( + "hb_cache_path".to_string(), + serde_json::Value::String(path.clone()), ); } if let Some(ref nurl) = bid.nurl { @@ -1335,13 +1368,40 @@ pub(crate) fn build_bid_map( if let Some(ref burl) = bid.burl { obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); } + // Include raw creative markup only for explicit debug injection. + // The pbRender bridge can use it while PBS Cache is unavailable. + if include_adm { + if let Some(ref adm) = bid.creative { + obj.insert("adm".to_string(), serde_json::Value::String(adm.clone())); + } + obj.insert( + "debug_bid".to_string(), + serde_json::json!({ + "slot_id": bid.slot_id, + "price": bid.price, + "currency": bid.currency, + "creative": bid.creative, + "adomain": bid.adomain, + "bidder": bid.bidder, + "width": bid.width, + "height": bid.height, + "nurl": bid.nurl, + "burl": bid.burl, + "ad_id": bid.ad_id, + "cache_id": bid.cache_id, + "cache_host": bid.cache_host, + "cache_path": bid.cache_path, + "metadata": bid.metadata, + }), + ); + } (slot_id.clone(), serde_json::Value::Object(obj)) }) }) .collect() } -/// Build the `__ts_bids` `` sequences inside the string. @@ -1350,7 +1410,7 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map should be infallible"); let escaped = html_escape_for_script(&json); format!( - "", + "", escaped ) } @@ -1363,7 +1423,7 @@ pub(crate) fn build_empty_bids_script() -> String { build_bids_script(&serde_json::Map::new()) } -/// Build the `__ts_ad_slots` `", + "", escaped ) } @@ -1479,7 +1539,7 @@ pub async fn handle_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, - slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], req: Request, ) -> Result> { let Some(co_config) = &settings.creative_opportunities else { @@ -1494,11 +1554,10 @@ pub async fn handle_page_bids( .map(|(_, v)| v.into_owned()) .unwrap_or_else(|| "/".to_string()); - let matched_slots: Vec<_> = - crate::creative_opportunities::match_slots(&slots_file.slots, &path_param) - .into_iter() - .cloned() - .collect(); + let matched_slots: Vec<_> = crate::creative_opportunities::match_slots(slots, &path_param) + .into_iter() + .cloned() + .collect(); let http_req = compat::from_fastly_headers_ref(&req); let request_info = @@ -1600,7 +1659,11 @@ pub async fn handle_page_bids( std::collections::HashMap::new() }; - let bid_map = build_bid_map(&winning_bids, co_config.price_granularity); + let bid_map = build_bid_map( + &winning_bids, + co_config.price_granularity, + settings.debug.inject_adm_for_testing, + ); let slots_json: Vec = matched_slots .iter() @@ -2582,6 +2645,7 @@ mod tests { gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, + slot: Vec::new(), } } @@ -2625,6 +2689,9 @@ mod tests { nurl: Some(nurl.to_string()), burl: Some(burl.to_string()), ad_id: Some(ad_id.to_string()), + cache_id: None, + cache_host: None, + cache_path: None, metadata: Default::default(), } } @@ -2635,11 +2702,15 @@ mod tests { let config = make_config(); let script = build_ad_slots_script(&slots, &config); assert!( - script.contains("window.__ts_ad_slots=JSON.parse"), - "should use JSON.parse" + script.contains("window.tsjs=window.tsjs||{}"), + "should initialise tsjs namespace" + ); + assert!( + script.contains(".adSlots=JSON.parse"), + "should use JSON.parse for adSlots" ); assert!(script.contains("atf_sidebar_ad"), "should include slot id"); - assert!(!script.contains("__ts_bids"), "must NOT contain bids"); + assert!(!script.contains("adInit"), "must NOT contain adInit"); assert!( !script.contains("__ts_request_id"), "must NOT contain request_id" @@ -2672,7 +2743,7 @@ mod tests { "https://ssp/bill", ), ); - let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); let obj = entry.as_object().expect("should be object"); assert_eq!( @@ -2688,7 +2759,7 @@ mod tests { assert_eq!( obj.get("hb_adid").and_then(|v| v.as_str()), Some("abc123"), - "should include ad_id" + "should fall back to ad_id when no cache_id present" ); assert_eq!( obj.get("nurl").and_then(|v| v.as_str()), @@ -2702,6 +2773,250 @@ mod tests { ); } + #[test] + fn client_bid_map_omits_adm_by_default() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + + assert!( + obj.get("adm").is_none(), + "should omit adm when debug injection is disabled" + ); + assert!( + obj.get("debug_bid").is_none(), + "should omit debug bid when debug injection is disabled" + ); + } + + #[test] + fn client_bid_map_includes_adm_when_debug_injection_enabled() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("adm").and_then(|v| v.as_str()), + Some("
Creative
"), + "should include adm when debug injection is enabled" + ); + } + + #[test] + fn client_bid_map_includes_debug_bid_when_debug_injection_enabled() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "mocktioneer", + "bid-ad-id", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + bid.adomain = Some(vec!["example.com".to_string()]); + bid.cache_id = Some("cache-uuid".to_string()); + bid.cache_host = Some("cache.example".to_string()); + bid.cache_path = Some("/cache".to_string()); + bid.metadata.insert( + "raw_field".to_string(), + serde_json::Value::String("raw-value".to_string()), + ); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + let debug_bid = obj + .get("debug_bid") + .and_then(|v| v.as_object()) + .expect("should include debug bid when debug injection is enabled"); + + assert_eq!( + debug_bid.get("slot_id").and_then(|v| v.as_str()), + Some("atf_sidebar_ad"), + "should expose original slot id" + ); + assert_eq!( + debug_bid.get("bidder").and_then(|v| v.as_str()), + Some("mocktioneer"), + "should expose original bidder" + ); + assert_eq!( + debug_bid.get("ad_id").and_then(|v| v.as_str()), + Some("bid-ad-id"), + "should expose original bid ad id" + ); + assert_eq!( + debug_bid.get("cache_id").and_then(|v| v.as_str()), + Some("cache-uuid"), + "should expose original PBS cache id" + ); + assert_eq!( + debug_bid.get("metadata").and_then(|v| v.get("raw_field")), + Some(&serde_json::Value::String("raw-value".to_string())), + "should expose provider metadata" + ); + } + + #[test] + fn bid_map_uses_cache_id_for_hb_adid_when_present() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), + cache_host: Some("openads.adsrvr.org".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should use cache_id for hb_adid, not ad_id" + ); + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("openads.adsrvr.org"), + "should emit hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "should emit hb_cache_path" + ); + } + + #[test] + fn bid_map_falls_back_to_ad_id_when_cache_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("aps-bid-token".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("aps-bid-token"), + "should fall back to ad_id when cache_id absent" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "should not emit hb_cache_host when absent" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "should not emit hb_cache_path when absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert!( + obj.get("hb_adid").is_none(), + "should omit hb_adid when no cache_id and no ad_id" + ); + } + #[test] fn bid_map_excludes_slot_when_price_is_none() { let mut winning_bids = HashMap::new(); @@ -2719,10 +3034,13 @@ mod tests { nurl: None, burl: None, ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: Default::default(), }, ); - let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); assert!( map.is_empty(), "slot with no price should be excluded from bid map" @@ -2789,9 +3107,7 @@ mod tests { mod page_bids_no_match_tests { use super::super::*; use crate::auction::AuctionOrchestrator; - use crate::creative_opportunities::{ - CreativeOpportunitiesFile, CreativeOpportunityFormat, CreativeOpportunitySlot, - }; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; use crate::platform::test_support::noop_services; use crate::test_support::tests::crate_test_settings_str; use fastly::http::Method; @@ -2805,24 +3121,22 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } - fn file_with_article_slot() -> CreativeOpportunitiesFile { - CreativeOpportunitiesFile { - slots: vec![CreativeOpportunitySlot { - id: "atf".to_string(), - gam_unit_path: None, - div_id: None, - page_patterns: vec!["/20**".to_string()], - formats: vec![CreativeOpportunityFormat { - width: 300, - height: 250, - media_type: crate::auction::types::MediaType::Banner, - }], - floor_price: Some(0.50), - targeting: Default::default(), - providers: Default::default(), - compiled_patterns: Vec::new(), + fn article_slot() -> Vec { + vec![CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, }], - } + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + }] } fn make_page_bids_request(path: &str) -> Request { @@ -2839,10 +3153,9 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = CreativeOpportunitiesFile { slots: vec![] }; let req = make_page_bids_request("/2024/01/my-article/"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &[], req) .await .expect("should return ok response"); @@ -2855,7 +3168,7 @@ mod tests { .expect("slots should be array") .len(), 0, - "empty slots file should produce zero injected slots" + "empty slots should produce zero injected slots" ); assert_eq!( body["bids"] @@ -2863,7 +3176,7 @@ mod tests { .expect("bids should be object") .len(), 0, - "empty slots file should produce zero bids" + "empty slots should produce zero bids" ); } @@ -2875,11 +3188,11 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = file_with_article_slot(); + let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) .await .expect("should return ok response"); @@ -2911,11 +3224,11 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = file_with_article_slot(); + let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("sec-purpose", "prefetch"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) .await .expect("should return ok response"); @@ -2946,10 +3259,10 @@ mod tests { let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let slots_file = file_with_article_slot(); // slot matches /20** only + let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match - let response = handle_page_bids(&settings, &orchestrator, &services, &slots_file, req) + let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) .await .expect("should return ok response"); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 386f0d54b..b221e0eac 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -416,6 +416,15 @@ pub struct DebugConfig { /// Never enable in production — visible in page source. #[serde(default)] pub auction_html_comment: bool, + + /// Include raw `adm` creative markup in `window.tsjs.bids` for GPT/GAM + /// debug rendering through the Prebid Universal Creative bridge. + /// + /// Use this to validate the server-side auction→GAM targeting→creative + /// rendering pipeline while PBS Cache is unavailable. Never enable in + /// production — injects raw HTML from SSPs. + #[serde(default)] + pub inject_adm_for_testing: bool, } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] @@ -522,14 +531,29 @@ impl Settings { /// # Errors /// /// Returns a configuration error if any cached runtime artifact cannot be prepared. - pub fn prepare_runtime(&self) -> Result<(), Report> { + pub fn prepare_runtime(&mut self) -> Result<(), Report> { for handler in &self.handlers { handler.prepare_runtime()?; } + if let Some(co) = &mut self.creative_opportunities { + co.compile_slots(); + } + Ok(()) } + /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + #[must_use] + pub fn creative_opportunity_slots( + &self, + ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { + self.creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]) + } + /// Resolve the first handler whose regex matches the request path. /// /// # Errors diff --git a/creative-opportunities.toml b/creative-opportunities.toml deleted file mode 100644 index da1ed23e7..000000000 --- a/creative-opportunities.toml +++ /dev/null @@ -1,47 +0,0 @@ -# Slot templates for server-side ad auction. -# Empty file = feature disabled (no auction fired, no globals injected). - -[[slot]] -id = "atf_sidebar_ad" -gam_unit_path = "/a/b/news" -div_id = "ad-atf_sidebar-0-_r_2_" -page_patterns = ["/20**", "/news/**"] -formats = [{ width = 300, height = 250 }] -floor_price = 0.50 - -[slot.targeting] -pos = "atf" -zone = "atfSidebar" - -[slot.providers.aps] -slot_id = "aps-slot-atf-sidebar" - -[[slot]] -id = "homepage_header_ad" -gam_unit_path = "/a/b/homepage" -div_id = "ad-header-0-_R_jpalubtak5lb_" -page_patterns = ["/"] -formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }, { width = 970, height = 250 }] -floor_price = 0.50 - -[slot.targeting] -pos = "atf" -zone = "header" - -[slot.providers.aps] -slot_id = "aps-slot-homepage-header" - -[[slot]] -id = "homepage_footer_ad" -gam_unit_path = "/a/b/homepage" -div_id = "ad-fixed_bottom-0-_R_klubtak5lb_" -page_patterns = ["/"] -formats = [{ width = 728, height = 90 }] -floor_price = 0.50 - -[slot.targeting] -pos = "btf" -zone = "fixedBottom" - -[slot.providers.aps] -slot_id = "aps-slot-homepage-footer" diff --git a/docs/superpowers/plans/2026-05-29-pr680-reviewer-findings.md b/docs/superpowers/plans/2026-05-29-pr680-reviewer-findings.md new file mode 100644 index 000000000..83866e3b8 --- /dev/null +++ b/docs/superpowers/plans/2026-05-29-pr680-reviewer-findings.md @@ -0,0 +1,630 @@ +# PR #680 Reviewer Findings Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Address the two reviewer-required findings from PR #680 plus low-effort cleanups: consolidate slot config into `trusted-server.toml`, consolidate `window.__ts*` globals under `window.tsjs`, and fix the TypeScript `formats` type cast and `ts_initial` hardcoded string. + +**Architecture:** Slot templates move from the standalone `creative-opportunities.toml` (embedded via `include_str!`) into the `[creative_opportunities]` section of `trusted-server.toml`, using the existing `vec_from_seq_or_map` deserializer pattern already used for `BID_PARAM_ZONE_OVERRIDES`. The window globals rename is a coordinated change across `gpt_bootstrap.js`, `index.ts`, and `publisher.rs` — all three must change together since they share a runtime contract. + +**Tech Stack:** Rust (serde, toml), TypeScript, vanilla JS, `cargo test --workspace`, `npx vitest run` + +--- + +## Context for all tasks + +- **Branch:** create `fix/pr680-review-findings` off `server-side-ad-templates-impl` before starting +- **Current codebase:** `crates/trusted-server-core/`, `crates/trusted-server-adapter-fastly/`, `crates/js/lib/` +- **CI gates:** `cargo fmt`, `cargo clippy --workspace --all-targets --all-features -- -D warnings`, `cargo test --workspace`, `npx vitest run`, `npm run format` +- **Error handling:** use `error-stack` (`Report`), not anyhow. Use `derive_more::Display`, not thiserror. +- **No `unwrap()` in production code** — use `expect("should ...")`. +- **Do not** add `println!` / `eprintln!` — use `log::` macros. + +--- + +## Task 1: Consolidate slot config into `trusted-server.toml` + +**What:** Delete `creative-opportunities.toml`. Move `[[slot]]` arrays into `trusted-server.toml` as `[[creative_opportunities.slot]]`. Wire the `vec_from_seq_or_map` deserializer so env var JSON blobs also work. Remove the `SLOTS_FILE` static and `include_str!` from `main.rs`. Update `build.rs` to validate slot IDs from settings instead of a separate file. + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/build.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` (function signatures) +- Modify: `trusted-server.toml` +- Delete: `creative-opportunities.toml` + +**Steps:** + +- [ ] **Step 1: Create the branch** + +```bash +git checkout -b fix/pr680-review-findings +``` + +- [ ] **Step 2: Add `Serialize` and `slot` field to structs** + +In `crates/trusted-server-core/src/creative_opportunities.rs`: + +1. Add `Serialize` to `CreativeOpportunitySlot` derive — it already has `#[serde(skip, default)]` on `compiled_patterns` so that field won't serialize. + +```rust +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CreativeOpportunitySlot { ... } +``` + +Also add `Serialize` to `CreativeOpportunityFormat`, `SlotProviders`, `ApsSlotParams` (any struct used inside `CreativeOpportunitySlot`). + +2. Add a `slot` field to `CreativeOpportunitiesConfig`: + +```rust +use crate::settings::vec_from_seq_or_map; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CreativeOpportunitiesConfig { + pub gam_network_id: String, + #[serde(default)] + pub auction_timeout_ms: Option, + #[serde(default = "PriceGranularity::dense")] + pub price_granularity: PriceGranularity, + /// Slot templates. Empty = feature disabled. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub slot: Vec, +} +``` + +Note: the field is named `slot` (not `slots`) to match the TOML key `[[creative_opportunities.slot]]`. + +- [ ] **Step 3: Delete `CreativeOpportunitiesFile`** + +Remove the `CreativeOpportunitiesFile` struct and its `impl` from `creative_opportunities.rs`. The `compile` logic moves to a free function or into `CreativeOpportunitiesConfig`: + +```rust +impl CreativeOpportunitiesConfig { + /// Pre-compile glob patterns for all slots. Call once after deserialization. + pub fn compile_slots(&mut self) { + for slot in &mut self.slot { + slot.compile_patterns(); + } + } +} +``` + +- [ ] **Step 4: Wire slot compilation into `Settings::prepare_runtime`** + +Glob pattern pre-compilation must happen once at startup, not per-request. `Settings::prepare_runtime` is already called after deserialization in both `from_toml_and_env` (build time) and `get_settings()` (runtime). Add slot compilation there: + +```rust +// In settings.rs, inside Settings::prepare_runtime +pub fn prepare_runtime(&mut self) -> Result<(), Report> { + for handler in &self.handlers { + handler.prepare_runtime()?; + } + // Pre-compile slot glob patterns for hot-path matching. + if let Some(co) = &mut self.creative_opportunities { + co.compile_slots(); + } + Ok(()) +} +``` + +Note: `prepare_runtime` must take `&mut self` for this change. Check current signature — if it takes `&self`, change it to `&mut self` and update call sites. + +Also add a helper method for call sites that need the slot slice: + +```rust +impl Settings { + /// Returns compiled creative opportunity slots, or empty slice if disabled. + pub fn creative_opportunity_slots(&self) -> &[CreativeOpportunitySlot] { + self.creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]) + } +} +``` + +- [ ] **Step 5: Update `build.rs` stub and slot validation** + +First update the `creative_opportunities` stub in `build.rs` to add the `slot` field — without this the settings parse will fail at build time when `trusted-server.toml` contains `[[creative_opportunities.slot]]` entries: + +```rust +mod creative_opportunities { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Clone, Deserialize, Serialize)] + pub struct CreativeOpportunitiesConfig { + pub gam_network_id: String, + #[serde(default)] + pub auction_timeout_ms: Option, + #[serde(default = "default_price_granularity")] + pub price_granularity: String, + // Use serde_json::Value to avoid pulling in full slot type in build context. + #[serde(default)] + pub slot: Vec, + } + + fn default_price_granularity() -> String { + "dense".to_string() + } +} +``` + +Then replace the separate-file validation block with reading slots from `Settings`: + +```rust +// After settings are parsed, validate slot IDs +let slot_id_re = regex::Regex::new(r"^[A-Za-z0-9_\-]+$").expect("should compile regex"); +if let Some(co) = &settings.creative_opportunities { + for slot in &co.slot { + if let Err(e) = trusted_server_core::creative_opportunities::validate_slot_id(&slot.id) { + panic!("trusted-server.toml [creative_opportunities.slot]: {e}"); + } + } + if !co.slot.is_empty() { + println!( + "cargo:warning=creative_opportunities: {} slot(s) validated", + co.slot.len() + ); + } +} +``` + +Remove: `CREATIVE_OPPORTUNITIES_PATH` const, the `co_path.exists()` block, and the `println!("cargo:rerun-if-changed={}", CREATIVE_OPPORTUNITIES_PATH)` line. + +Note: `build.rs` already pulls in `src/creative_opportunities.rs` as a module — make sure the module stub includes the new `Serialize` derive (it may need the serde `Serialize` import). + +- [ ] **Step 6: Update `main.rs` — remove `SLOTS_FILE` static** + +Remove: + +```rust +const CREATIVE_OPPORTUNITIES_TOML: &str = include_str!("../../../creative-opportunities.toml"); +static SLOTS_FILE: std::sync::LazyLock<...> = ...; +``` + +Replace `slots_file` parameter threading with deriving slots from `settings`: + +Where `slots_file` was passed as `&*SLOTS_FILE`, pass `settings.creative_opportunity_slots()` instead. This requires `settings` to be available at that call site (it is — `settings` is already in scope). + +Update function signatures in `main.rs` that reference `CreativeOpportunitiesFile` to accept `&[CreativeOpportunitySlot]` instead. + +- [ ] **Step 7: Update `publisher.rs` function signatures** + +Functions that take `&crate::creative_opportunities::CreativeOpportunitiesFile` change to `&[crate::creative_opportunities::CreativeOpportunitySlot]`: + +```rust +// Before +pub(crate) fn handle_page_bids( + ... + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + ... +) + +// After +pub(crate) fn handle_page_bids( + ... + slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + ... +) +``` + +Inside the function body, replace `slots_file.slots` with `slots`. + +Update all call sites and test helpers in `publisher.rs` that construct `CreativeOpportunitiesFile { slots: vec![...] }` to pass `&[slot]` directly. + +- [ ] **Step 8: Update `trusted-server.toml`** + +Move the slots from `creative-opportunities.toml` into `trusted-server.toml` under `[creative_opportunities]`. Use `[[creative_opportunities.slot]]` syntax. Use only example/fictional values per project convention (example.com domains, fictional IDs): + +```toml +[creative_opportunities] +gam_network_id = "88059007" +auction_timeout_ms = 1500 +price_granularity = "dense" + +[[creative_opportunities.slot]] +id = "atf_sidebar_ad" +gam_unit_path = "/a/b/news" +div_id = "div-ad-atf-sidebar" +page_patterns = ["/news/**"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-atf-sidebar" +``` + +- [ ] **Step 9: Delete `creative-opportunities.toml`** + +```bash +git rm creative-opportunities.toml +``` + +- [ ] **Step 10: Run tests** + +```bash +cargo test --workspace +``` + +Expected: all tests pass. Fix any compile errors from the signature changes. + +- [ ] **Step 11: Run clippy and fmt** + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +- [ ] **Step 12: Commit** + +```bash +git add -p +git commit -m "Move slot templates from creative-opportunities.toml into trusted-server.toml" +``` + +--- + +## Task 2: Consolidate `window.__ts*` globals under `window.tsjs` + +**What:** All `window.__ts*` globals become properties on a single `window._ts` namespace object. Changes must be coordinated across three files: `gpt_bootstrap.js`, `index.ts`, and `publisher.rs`. Tests in `index.test.ts` must be updated too. + +**Rename table:** + +| Old global | New property | Notes | +| ----------------------------- | ------------------------------ | ---------------------------- | +| `window.__ts_ad_slots` | `window.tsjs.adSlots` | Array, set at head-open | +| `window.__ts_bids` | `window.tsjs.bids` | Object, set before `` | +| `window.__tsAdInit` | `window.tsjs.adInit` | Function | +| `window.__tsPrevGptSlots` | `window.tsjs.prevGptSlots` | Array | +| `window.__tsServicesEnabled` | `window.tsjs.servicesEnabled` | Boolean | +| `window.__tsDivToSlotId` | `window.tsjs.divToSlotId` | Object | +| `window.__tsSpaHookInstalled` | `window.tsjs.spaHookInstalled` | Boolean | + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` +- Modify: `crates/js/lib/src/integrations/gpt/index.test.ts` +- Modify: `crates/js/lib/test/integrations/gpt/index.test.ts` (if exists) + +**Steps:** + +- [ ] **Step 1: Update `publisher.rs` injected scripts** + +`build_ad_slots_script` generates the `", escaped) + +// After — initialise _ts if absent, then set adSlots +format!("", escaped) +``` + +`build_bids_script` generates the script injected before ``. Change: + +```rust +// Before +format!( + "", + escaped +) + +// After +format!( + "", + escaped +) +``` + +Note: `{{}}` is the Rust format-string escape for a literal `{}`. + +Update any test assertions in `publisher.rs` that check for the old global names. + +- [ ] **Step 2: Update `gpt_bootstrap.js`** + +Replace all `window.__ts*` references. The bootstrap IIFE runs before the TS bundle, so it must initialise `window._ts` if absent: + +```js +;(function () { + if (typeof window === 'undefined') return + // Initialise namespace; adInit guard prevents double-install. + var ts = (window._ts = window._ts || {}) + if (ts.adInit) return + + ts.adInit = function () { + var slots = ts.adSlots || [] + var bids = ts.bids || {} + var divToSlotId = {} + googletag.cmd.push(function () { + var newSlots = [] + slots.forEach(function (slot) { + var s = googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!s) return + s.addService(googletag.pubads()) + Object.entries(slot.targeting || {}).forEach(function (e) { + s.setTargeting(e[0], e[1]) + }) + var b = bids[slot.id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (k) { + if (b[k]) s.setTargeting(k, b[k]) + }) + s.setTargeting('ts_initial', '1') + divToSlotId[slot.div_id] = slot.id + newSlots.push(s) + }) + ts.prevGptSlots = newSlots + ts.divToSlotId = divToSlotId + if (!ts.servicesEnabled) { + googletag.pubads().enableSingleRequest() + googletag.enableServices() + ts.servicesEnabled = true + googletag.pubads().addEventListener('slotRenderEnded', function (ev) { + var divId = ev.slot.getSlotElementId() + var slotId = (ts.divToSlotId || {})[divId] + if (!slotId) return + var b = (ts.bids || {})[slotId] || {} + var ourBidWon = + !ev.isEmpty && + (b.hb_adid + ? ev.slot.getTargeting('hb_adid')[0] === b.hb_adid + : !!b.hb_bidder) + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl) + if (b.burl) navigator.sendBeacon(b.burl) + } + }) + } + if (newSlots.length > 0) { + googletag.pubads().refresh(newSlots) + } + }) + } +})() +``` + +- [ ] **Step 3: Update `index.ts` — rename `TsWindow` type** + +Replace the `TsWindow` interface: + +```typescript +type TsNamespace = { + adSlots?: TsAdSlot[] + bids?: Record + adInit?: () => void + prevGptSlots?: GoogleTagSlot[] + servicesEnabled?: boolean + divToSlotId?: Record + spaHookInstalled?: boolean +} + +type TsWindow = Window & { + _ts?: TsNamespace +} +``` + +- [ ] **Step 4: Update `installTsAdInit` in `index.ts`** + +Update all properties to live under `window.tsjs`. Use `window.tsjs` directly: + +```typescript +export function installTsAdInit(): void { + const w = window as TsWindow + const ts = (w._ts = w._ts ?? {}) + ts.adInit = function () { + const slots = ts.adSlots ?? [] + const bids = ts.bids ?? {} + const g = (window as GptWindow).googletag + if (!g) return + + g.cmd?.push(() => { + if (ts.prevGptSlots && ts.prevGptSlots.length > 0) { + g.destroySlots?.(ts.prevGptSlots) + ts.prevGptSlots = [] + } + const newSlots: GoogleTagSlot[] = [] + const divToSlotId: Record = {} + + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ) + if (!gptSlot) return + gptSlot.addService(g.pubads!()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + gptSlot.setTargeting('ts_initial', '1') + divToSlotId[slot.div_id] = slot.id + newSlots.push(gptSlot) + }) + + ts.prevGptSlots = newSlots + ts.divToSlotId = divToSlotId + + if (!ts.servicesEnabled) { + g.pubads!().enableSingleRequest() + g.enableServices?.() + ts.servicesEnabled = true + g.pubads!().addEventListener?.( + 'slotRenderEnded', + (event: SlotRenderEndedEvent) => { + const divId: string = event.slot?.getSlotElementId?.() ?? '' + const slotId = (ts.divToSlotId ?? {})[divId] + if (!slotId) return + const bid = (ts.bids ?? {})[slotId] ?? {} + const ourBidWon = + !event.isEmpty && + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder) + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl) + if (bid.burl) navigator.sendBeacon(bid.burl) + } + } + ) + } + if (newSlots.length > 0) { + g.pubads!().refresh(newSlots) + } + }) + } +} +``` + +- [ ] **Step 5: Update `installSpaHook` in `index.ts`** + +Replace `__tsSpaHookInstalled` and `__ts_ad_slots`/`__ts_bids` reads: + +```typescript +export function installSpaHook(): void { + const win = window as TsWindow + const ts = (win._ts = win._ts ?? {}) + if (ts.spaHookInstalled) return + ts.spaHookInstalled = true + // ... rest of SPA hook logic uses ts.adSlots, ts.bids, ts.adInit +} +``` + +- [ ] **Step 6: Update tests in `index.test.ts`** + +Find all test assertions that reference `window.__ts_ad_slots`, `window.__ts_bids`, `window.__tsAdInit`, etc. and update to `window.tsjs.adSlots`, `window.tsjs.bids`, `window.tsjs.adInit` etc. + +Run tests first to see what fails: + +```bash +cd crates/js/lib && npx vitest run +``` + +Fix each failing assertion. + +- [ ] **Step 7: Run JS tests and format** + +```bash +cd crates/js/lib && npx vitest run +cd crates/js/lib && npm run format +``` + +Expected: all tests pass, no format errors. + +- [ ] **Step 8: Run Rust tests** + +```bash +cargo test --workspace +``` + +Update any test assertions in `publisher.rs` that check for old global names (e.g. `script.contains("window.__ts_ad_slots")`). + +- [ ] **Step 9: Run clippy and fmt** + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +- [ ] **Step 10: Commit** + +```bash +git commit -m "Namespace window globals under window._ts" +``` + +--- + +## Task 3: Fix `formats` type and extract `ts_initial` constant + +**What:** Two small TypeScript/JS cleanups. `TsAdSlot.formats` should be typed as `Array<[number, number]>` (tuple, not array-of-array) to match GPT's actual input. The string `'ts_initial'` is hardcoded in both `gpt_bootstrap.js` and `index.ts` — extract as a named constant in `index.ts` (no JS equivalent needed since the bootstrap is vanilla JS). + +**Files:** + +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` (comment only — JS can't share TS constants) + +**Steps:** + +- [ ] **Step 1: Fix `TsAdSlot.formats` type** + +In `index.ts`, change: + +```typescript +// Before +interface TsAdSlot { + ... + formats: Array; +} + +// After +interface TsAdSlot { + ... + formats: Array<[number, number]>; +} +``` + +Update the cast at the GPT `defineSlot` call site — `[number, number]` satisfies `number | number[]` so the cast can be removed or simplified: + +```typescript +// Before +slot.formats as Array + +// After — [number, number][] already satisfies Array +slot.formats +``` + +- [ ] **Step 2: Extract `ts_initial` constant in `index.ts`** + +Near the top of `index.ts`, add: + +```typescript +const TS_INITIAL_TARGETING_KEY = 'ts_initial' +``` + +Replace both occurrences of `'ts_initial'` in `installTsAdInit` with `TS_INITIAL_TARGETING_KEY`. + +Add a comment in `gpt_bootstrap.js` where `'ts_initial'` appears: + +```js +// Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts +s.setTargeting('ts_initial', '1') +``` + +- [ ] **Step 3: Run JS tests and format** + +```bash +cd crates/js/lib && npx vitest run +cd crates/js/lib && npm run format +``` + +- [ ] **Step 4: Commit** + +```bash +git commit -m "Fix TsAdSlot formats type and extract ts_initial constant" +``` + +--- + +## Final verification + +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` +- [ ] `cargo test --workspace` +- [ ] `cd crates/js/lib && npx vitest run` +- [ ] `cd crates/js/lib && npm run format` +- [ ] `cd docs && npm run format` diff --git a/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md b/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md new file mode 100644 index 000000000..7a3f34207 --- /dev/null +++ b/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md @@ -0,0 +1,760 @@ +# Prebid Creative Rendering Fix Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix `hb_adid` to carry the PBS Cache UUID (not the OpenRTB bid ID) so the Prebid Universal Creative in GAM can fetch and render the correct creative markup. + +**Architecture:** Three-file change: add `cache_id`/`cache_host`/`cache_path` fields to the shared `Bid` struct in `types.rs`, extract these from `ext.prebid.cache.bids` in `prebid.rs`'s `parse_bid`, then emit them as `hb_adid`/`hb_cache_host`/`hb_cache_path` in `publisher.rs`'s `build_bid_map`. `AuctionBid` in `prebid.rs` is a type alias for `Bid` (`use ... Bid as AuctionBid`), so only one struct needs the new fields. + +**Tech Stack:** Rust 2024, `serde`, `url` crate (already in workspace deps at v2.5.8), `cargo test --workspace` + +--- + +## Context for all tasks + +- **Branch:** `fix/server-side-ad-template-entrypoint` (already checked out) +- **Spec:** `docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md` +- **Error handling:** `error-stack` (`Report`), not anyhow. Use `expect("should ...")` not `unwrap()`. +- **No `println!`/`eprintln!`** — use `log::` macros. +- **All public items must have doc comments.** +- CI gates: `cargo fmt --all -- --check`, `cargo clippy --workspace --all-targets --all-features -- -D warnings`, `cargo test --workspace` + +--- + +## Task 1: Add cache fields to `Bid` struct and fix all construction sites + +**What:** Add three new `Option` fields to `Bid`. Since Rust struct literals are exhaustive, every place that constructs a `Bid { ... }` in the codebase will fail to compile until the new fields are added. Fix all of them with `None` defaults (except the APS provider which constructs a real `Bid` — also `None` since APS doesn't use PBS Cache). + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/types.rs:200` (after `ad_id` field) +- Modify (test helpers/literals — add `None` fields): + - `crates/trusted-server-core/src/auction/types.rs:314` (`make_bid` helper) + - `crates/trusted-server-core/src/auction/types.rs:445` (inline `Bid` literal) + - `crates/trusted-server-core/src/publisher.rs:2616` (`make_bid` helper) + - `crates/trusted-server-core/src/publisher.rs:2714` (inline `Bid` literal) + - `crates/trusted-server-core/src/auction/orchestrator.rs:1121,1138,1278,1325,1358` (test `Bid` literals) + - `crates/trusted-server-core/src/integrations/aps.rs:442` (production `Bid` construction) + +**Steps:** + +- [ ] **Step 1: Add three fields to `Bid` struct in `types.rs`** + + In `crates/trusted-server-core/src/auction/types.rs`, after line 200 (`pub ad_id: Option,`), add: + + ```rust + /// Prebid Cache UUID for this bid. + /// + /// Populated from `ext.prebid.cache.bids.cacheId` in the PBS response. + /// Used as `hb_adid` targeting value in `window._ts.bids`. `None` for + /// non-PBS providers (e.g., APS) and PBS bids without Prebid Cache enabled. + pub cache_id: Option, + /// Prebid Cache host (e.g., `"openads.adsrvr.org"`). + /// + /// Populated from the host of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_host` targeting value. `None` when cache is absent. + pub cache_host: Option, + /// Prebid Cache path (e.g., `"/cache"`). + /// + /// Populated from the path of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_path` targeting value. `None` when cache is absent. + pub cache_path: Option, + ``` + +- [ ] **Step 2: Verify compile fails as expected** + + ```bash + cargo check --package trusted-server-core 2>&1 | grep "missing field" + ``` + + Expected: multiple errors about missing `cache_id`, `cache_host`, `cache_path` in `Bid` struct literals. This confirms every construction site will be found. + +- [ ] **Step 3: Fix `make_bid` helper in `types.rs` (line ~314)** + + Add three `None` fields to the `Bid {}` literal inside the `make_bid` test helper: + + ```rust + fn make_bid(bidder: &str) -> Bid { + Bid { + slot_id: "slot-1".to_string(), + price: Some(1.0), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } + ``` + +- [ ] **Step 4: Fix inline `Bid` literal in `types.rs` (line ~445)** + + Find the `Bid {` literal around line 445 in the test section of `types.rs`. Add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 5: Fix `make_bid` helper in `publisher.rs` (line ~2616)** + + In the `make_bid` test helper function in `publisher.rs`, add to the `Bid {}` literal: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 6: Fix inline `Bid` literal in `publisher.rs` (line ~2714)** + + Find the `Bid {` literal around line 2714 in `publisher.rs` tests. Add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 7: Fix five `Bid` literals in `orchestrator.rs` (lines ~1121,1138,1278,1325,1358)** + + Add to each of the five `Bid {}` literals in the test section of `orchestrator.rs`: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 8: Fix APS production `Bid` construction in `aps.rs` (line ~442)** + + In `aps.rs`, inside `parse_aps_response` (or wherever the `Ok(Bid { ... })` is around line 442), add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + + APS does not use PBS Cache — these fields are intentionally `None` for APS bids. + +- [ ] **Step 9: Verify compile succeeds** + + ```bash + cargo check --package trusted-server-core 2>&1 | grep -E "^error" + ``` + + Expected: no output (clean compile). + +- [ ] **Step 10: Run tests to confirm nothing regressed** + + ```bash + cargo test --workspace 2>&1 | tail -5 + ``` + + Expected: all tests pass. + +- [ ] **Step 11: Run clippy and fmt** + + ```bash + cargo fmt --all + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: clean. + +- [ ] **Step 12: Commit** + + ```bash + git add crates/trusted-server-core/src/auction/types.rs \ + crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/auction/orchestrator.rs \ + crates/trusted-server-core/src/integrations/aps.rs + git commit -m "Add cache_id, cache_host, cache_path fields to Bid struct" + ``` + +--- + +## Task 2: Extract PBS Cache fields in `prebid.rs` `parse_bid` + tests + +**What:** After extracting `ad_id` in `parse_bid`, extract `ext.prebid.cache.bids.cacheId` as `cache_id` and split `ext.prebid.cache.bids.url` into `cache_host` + `cache_path`. Populate all three new fields on the returned `AuctionBid`. Add TDD tests first. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs:1362–1391` (extraction + struct literal) +- Test: `crates/trusted-server-core/src/integrations/prebid.rs` (test module near bottom) + +**Steps:** + +- [ ] **Step 1: Write the failing tests** + + Find the `#[cfg(test)]` module in `prebid.rs`. Add these tests (they will fail because extraction doesn't exist yet): + + ```rust + #[test] + fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { + // Real PBS response shape from auction_response.json + let bid_json = serde_json::json!({ + "id": "bid-id-123", + "impid": "atf_sidebar_ad", + "price": 1.50, + "adm": "
ad
", + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "thetradedesk") + .expect("should parse bid"); + assert_eq!( + bid.cache_id.as_deref(), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should extract cacheId as cache_id" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("openads.adsrvr.org"), + "should extract host from cache URL" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should extract path from cache URL" + ); + } + + #[test] + fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { + let bid_json = serde_json::json!({ + "id": "bid-id-456", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250 + // no ext.prebid.cache + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert!(bid.cache_id.is_none(), "should be None when cache absent"); + assert!(bid.cache_host.is_none(), "should be None when cache absent"); + assert!(bid.cache_path.is_none(), "should be None when cache absent"); + } + + #[test] + fn parse_bid_handles_malformed_cache_url_gracefully() { + let bid_json = serde_json::json!({ + "id": "bid-id-789", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "not-a-valid-url", + "cacheId": "some-uuid" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid without panicking"); + assert_eq!( + bid.cache_id.as_deref(), + Some("some-uuid"), + "should still extract cacheId even if URL is malformed" + ); + assert!(bid.cache_host.is_none(), "should be None when URL parse fails"); + assert!(bid.cache_path.is_none(), "should be None when URL parse fails"); + } + + #[test] + fn parse_bid_preserves_ad_id_alongside_cache_id() { + let bid_json = serde_json::json!({ + "id": "bid-impression-id", + "impid": "atf_sidebar_ad", + "adid": "bidder-ad-id-abc", + "price": 1.0, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://cache.example.com/cache", + "cacheId": "cache-uuid-xyz" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.ad_id.as_deref(), + Some("bidder-ad-id-abc"), + "should keep ad_id from adid field" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid-xyz"), + "should extract cache UUID separately" + ); + } + ``` + + Note: `base_config()` and `PrebidAuctionProvider::new()` are the standard test construction pattern used throughout the existing `prebid.rs` test module. `parse_bid` is a private method but is accessible from the `#[cfg(test)]` module in the same file. + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + cargo test --package trusted-server-core parse_bid_extracts_cache_id 2>&1 | tail -15 + ``` + + Expected: compile error (`no field 'cache_id' on type 'Bid'`) or test failure. Either confirms the extraction code is missing. + +- [ ] **Step 3: Add cache extraction to `parse_bid` in `prebid.rs`** + + In `parse_bid` (around line 1362), after the `ad_id` extraction block and before the `Ok(AuctionBid { ... })`, add: + + ```rust + // Extract PBS Cache coordinates from ext.prebid.cache.bids. + // The Prebid Universal Creative uses cacheId as hb_adid and the host/path + // to construct the fetch URL: https://?uuid= + let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + + let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + + let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + url::Url::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {e}")) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { + None + } else { + Some(path) + }; + (host, path) + }) + .unwrap_or((None, None)); + + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{slot_id}'" + ); + } + ``` + + Then add the three fields to the `Ok(AuctionBid { ... })` struct literal (around line 1377): + + ```rust + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain, + bidder: seat.to_string(), + width, + height, + nurl, + burl, + ad_id, + cache_id, + cache_host, + cache_path, + metadata: std::collections::HashMap::new(), + }) + ``` + +- [ ] **Step 4: Run tests to verify they pass** + + ```bash + cargo test --package trusted-server-core parse_bid 2>&1 | tail -20 + ``` + + Expected: all 4 new tests pass. + +- [ ] **Step 5: Run full test suite** + + ```bash + cargo test --workspace 2>&1 | tail -5 + ``` + + Expected: all tests pass. + +- [ ] **Step 6: Run clippy and fmt** + + ```bash + cargo fmt --all + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: clean. If clippy warns about the `log::debug!` return value being unused inside `map_err`, suppress with `let _ = ...` or restructure. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/prebid.rs + git commit -m "Extract PBS Cache UUID and endpoint from bid ext into Bid fields" + ``` + +--- + +## Task 3: Emit cache fields in `build_bid_map` + update tests + +**What:** Change `build_bid_map` to use `bid.cache_id` for `hb_adid` (falling back to `bid.ad_id` for APS/other providers), and emit `hb_cache_host`/`hb_cache_path` when present. Update the existing `bid_map_includes_nurl_and_burl` test (which currently passes `"abc123"` as `ad_id` and asserts `hb_adid = "abc123"`) to use a cache-based bid. Add new tests covering cache fields and fallback path. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs:1311–1342` (`build_bid_map`) +- Modify: `crates/trusted-server-core/src/publisher.rs:2608–2630` (`make_bid` helper — add cache params) +- Modify: `crates/trusted-server-core/src/publisher.rs:2666–2707` (existing `bid_map_includes_nurl_and_burl` test) +- Test: `crates/trusted-server-core/src/publisher.rs` (new tests in the existing test module) + +**Steps:** + +- [ ] **Step 1: Write new failing tests for cache field emission** + + Add these tests to the `#[cfg(test)]` module in `publisher.rs`, near the existing `bid_map_includes_nurl_and_burl` test: + + ```rust + #[test] + fn bid_map_uses_cache_id_for_hb_adid_when_present() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), + cache_host: Some("openads.adsrvr.org".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should use cache_id for hb_adid, not ad_id" + ); + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("openads.adsrvr.org"), + "should emit hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "should emit hb_cache_path" + ); + } + + #[test] + fn bid_map_falls_back_to_ad_id_when_cache_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "aps-amazon".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("aps-bid-token".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("aps-bid-token"), + "should fall back to ad_id when cache_id absent" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "should not emit hb_cache_host when absent" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "should not emit hb_cache_path when absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert!( + obj.get("hb_adid").is_none(), + "should omit hb_adid when no cache_id and no ad_id" + ); + } + ``` + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + cargo test --package trusted-server-core bid_map_uses_cache_id 2>&1 | tail -15 + ``` + + Expected: test fails — `hb_adid` returns `"bid-impression-id"` (the wrong value) instead of the cache UUID, and `hb_cache_host`/`hb_cache_path` are not emitted. + +- [ ] **Step 3: Update `build_bid_map` in `publisher.rs`** + + Replace the current `hb_adid` emission block (lines ~1326–1331) and the `nurl`/`burl` block with: + + ```rust + // hb_adid: PBS Cache UUID when present (Prebid Universal Creative uses this + // as the cache lookup key). Falls back to ad_id for APS and other non-PBS + // providers. Note: ad_id (OpenRTB bid ID) is NOT the same as the cache UUID. + let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); + if let Some(id) = hb_adid { + obj.insert( + "hb_adid".to_string(), + serde_json::Value::String(id.to_string()), + ); + } + + // Cache endpoint coordinates — only present for PBS bids with Prebid Cache. + // The Prebid Universal Creative constructs: + // https://?uuid= + if let Some(ref host) = bid.cache_host { + obj.insert( + "hb_cache_host".to_string(), + serde_json::Value::String(host.clone()), + ); + } + if let Some(ref path) = bid.cache_path { + obj.insert( + "hb_cache_path".to_string(), + serde_json::Value::String(path.clone()), + ); + } + + if let Some(ref nurl) = bid.nurl { + obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + } + if let Some(ref burl) = bid.burl { + obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + } + ``` + +- [ ] **Step 4: Update the existing `bid_map_includes_nurl_and_burl` test** + + The existing test at line ~2666 constructs a bid via `make_bid("atf_sidebar_ad", 1.50, "kargo", "abc123", ...)` and asserts `hb_adid = "abc123"`. Update `make_bid` to accept optional `cache_id`, `cache_host`, `cache_path`, OR create a separate variant. The simplest fix: update the assertion in the existing test to reflect the new priority logic. + + The test currently passes `ad_id = "abc123"` and `cache_id = None`. After the fix, `hb_adid` should still be `"abc123"` (fallback path). So the existing assertion is correct — just verify it still passes. No change needed to that test body. Just update `make_bid` to set the new fields to `None`: + + ```rust + fn make_bid( + slot_id: &str, + price: f64, + bidder: &str, + ad_id: &str, + nurl: &str, + burl: &str, + ) -> Bid { + Bid { + slot_id: slot_id.to_string(), + price: Some(price), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: Some(nurl.to_string()), + burl: Some(burl.to_string()), + ad_id: Some(ad_id.to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + } + } + ``` + + Also update the assertion comment at line ~2694 from `"should include ad_id"` to `"should fall back to ad_id when no cache_id"`. + +- [ ] **Step 5: Run all new tests** + + ```bash + cargo test --package trusted-server-core bid_map 2>&1 | tail -20 + ``` + + Expected: all `bid_map_*` tests pass, including both new and existing. + +- [ ] **Step 6: Add round-trip serialization test for `Bid`** + + Add this test to the `#[cfg(test)]` module in `types.rs`: + + ```rust + #[test] + fn bid_with_cache_fields_round_trips_through_json() { + let bid = Bid { + slot_id: "atf".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example.com".to_string()), + cache_path: Some("/pbc/v1/cache".to_string()), + metadata: HashMap::new(), + }; + let json = serde_json::to_string(&bid).expect("should serialize Bid"); + let restored: Bid = serde_json::from_str(&json).expect("should deserialize Bid"); + assert_eq!(restored.cache_id.as_deref(), Some("cache-uuid"), "should round-trip cache_id"); + assert_eq!(restored.cache_host.as_deref(), Some("cache.example.com"), "should round-trip cache_host"); + assert_eq!(restored.cache_path.as_deref(), Some("/pbc/v1/cache"), "should round-trip cache_path"); + } + ``` + + Run: + + ```bash + cargo test --package trusted-server-core bid_with_cache_fields_round_trips 2>&1 | tail -5 + ``` + + Expected: PASS. + +- [ ] **Step 7: Run full CI suite** + + ```bash + cargo test --workspace 2>&1 | tail -5 + cargo fmt --all -- --check + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: all pass, no warnings. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/auction/types.rs + git commit -m "Emit hb_adid from PBS Cache UUID and add hb_cache_host/hb_cache_path to bid map" + ``` + +--- + +## Final verification + +- [ ] Run `cargo test --workspace` — all pass +- [ ] Run `cargo clippy --workspace --all-targets --all-features -- -D warnings` — clean +- [ ] Run `cargo fmt --all -- --check` — clean +- [ ] In browser devtools after deploy: `window._ts.bids` shows `hb_cache_host`, `hb_cache_path`, and `hb_adid` matching the UUID in `ext.prebid.cache.bids.cacheId` from the raw PBS response + +--- + +## Rollout reminder (from spec §8) + +1. TS: this branch deployed +2. GAM: ad ops updates Prebid line item creatives to server-side cache-fetch variant (see spec §4.6) +3. PBS: Prebid Cache already enabled (confirmed from real response) +4. Verify in devtools diff --git a/docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md b/docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md new file mode 100644 index 000000000..a21ec4d28 --- /dev/null +++ b/docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md @@ -0,0 +1,345 @@ +# Prebid Creative Rendering Fix Design + +_Author · 2026-05-29_ + +--- + +## 1. Problem Statement + +The Trusted Server server-side auction returns winning bids from PBS, but ads never +render on the Prebid path because `hb_adid` carries the wrong value. + +The Prebid Universal Creative in GAM constructs the creative fetch URL as: + +``` +https://?uuid= +``` + +TS currently sets `hb_adid` from `bid.adid` or `bid.id` (the OpenRTB bid ID / +impression ID). PBS actually caches the creative markup and returns the cache UUID +in `ext.prebid.cache.bids.cacheId`. The Universal Creative needs the **cache UUID**, +not the bid ID. The cache host and path are also not forwarded today. + +**Effect:** GAM receives a wrong UUID, fetches nothing, and the slot renders empty. + +--- + +## 2. Root Cause — Two Extraction Gaps + +### Gap 1: Wrong `hb_adid` source + +`prebid.rs` extracts: + +```rust +let ad_id = bid_obj + .get("adid") + .or_else(|| bid_obj.get("id")) // ← falls back to impression ID + .and_then(|v| v.as_str()) + .map(String::from); +``` + +Real PBS response has (in `ext.prebid.cache.bids`): + +```json +{ + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" +} +``` + +`bid.id` = `"ad-header-0-_R_4uapbsnql8alb_"` — the impression ID, useless to the +creative renderer. + +### Gap 2: Cache host and path not forwarded + +`build_bid_map` in `publisher.rs` emits `hb_pb`, `hb_bidder`, `hb_adid`, `nurl`, +`burl`. It does not emit `hb_cache_host` or `hb_cache_path`. The Prebid Universal +Creative needs both to construct the fetch URL. + +--- + +## 3. Non-Goals + +- APS creative rendering — APS does not use PBS Cache. APS creative delivery is + Amazon-owned and not addressed here. +- APS win detection over-fire — separate known limitation, separate issue. +- Dual bootstrap sync risk — separate maintenance issue. +- Slim-Prebid bundle — out of scope for Phase 1. + +--- + +## 4. Design + +### 4.1 New Fields on `Bid` (types.rs) + +Add three fields to `Bid` to carry the PBS Cache coordinates extracted from the bid +response: + +```rust +/// Prebid Cache UUID for this bid. Populated from +/// `ext.prebid.cache.bids.cacheId` in the PBS response. +/// Used as `hb_adid` targeting value in `window.tsjs.bids`. +/// None for non-PBS providers (e.g., APS) and PBS bids without cache enabled. +pub cache_id: Option, + +/// Prebid Cache host (e.g., `"openads.adsrvr.org"`). Populated from +/// the host component of `ext.prebid.cache.bids.url`. +/// Used as `hb_cache_host` targeting value. +pub cache_host: Option, + +/// Prebid Cache path (e.g., `"/cache"`). Populated from +/// the path component of `ext.prebid.cache.bids.url`. +/// Used as `hb_cache_path` targeting value. +pub cache_path: Option, +``` + +### 4.2 Extraction in `prebid.rs` + +In `parse_bid_object`, after extracting `nurl`/`burl`, extract the cache fields from +`ext.prebid.cache.bids`: + +```rust +// Extract PBS Cache coordinates from ext.prebid.cache.bids +let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + +let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + +let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + url::Url::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + // path() returns "/" for root — only use if non-trivial + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { None } else { Some(path) }; + (host, path) + }) + .unwrap_or((None, None)); + +// Guard: if we extracted a cache UUID but couldn't extract the host, +// the bid will have hb_adid set but no endpoint to fetch from — creative will fail. +if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{}'", + slot_id + ); +} +``` + +Note: `url` crate is already a workspace dependency. If not, parse host/path manually +by splitting on the first `/` after the scheme. + +The `ad_id` field (from `bid.adid` / `bid.id`) is **kept** — it maps to the OpenRTB +`adid` / `id` field that APS and other non-PBS providers may use. The cache fields are +**in addition**, not replacing `ad_id`. + +Populate all three fields on `AuctionBid`: + +```rust +Ok(AuctionBid { + ..., + ad_id, + cache_id, + cache_host, + cache_path, + ... +}) +``` + +### 4.3 `build_bid_map` in `publisher.rs` + +Priority for `hb_adid`: use `cache_id` when present (PBS path), fall back to `ad_id` +(APS / other providers, backward compat): + +```rust +// hb_adid: use PBS Cache UUID when present — the Prebid Universal Creative uses +// this as the cache lookup key, NOT the OpenRTB bid ID (bid.ad_id). Fall back to +// bid.ad_id for APS and other non-PBS providers. +let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); +if let Some(id) = hb_adid { + obj.insert("hb_adid".to_string(), serde_json::Value::String(id.to_string())); +} + +// Cache coordinates — only present for PBS bids with Prebid Cache enabled +if let Some(ref host) = bid.cache_host { + obj.insert("hb_cache_host".to_string(), serde_json::Value::String(host.clone())); +} +if let Some(ref path) = bid.cache_path { + obj.insert("hb_cache_path".to_string(), serde_json::Value::String(path.clone())); +} +``` + +### 4.4 What `window.tsjs.bids` looks like after the fix + +```json +{ + "atf_sidebar_ad": { + "hb_pb": "0.01", + "hb_bidder": "thetradedesk", + "hb_adid": "f47447a0-b759-4f2f-9887-af458b79b570", + "hb_cache_host": "openads.adsrvr.org", + "hb_cache_path": "/cache", + "nurl": "https://...", + "burl": "https://..." + } +} +``` + +### 4.5 Win detection — no change required + +`slotRenderEnded` checks: + +```js +event.slot.getTargeting('hb_adid')[0] === bid.hb_adid +``` + +`adInit()` calls `setTargeting('hb_adid', cacheId)` with the cache UUID. +`event.slot.getTargeting('hb_adid')[0]` returns that same cache UUID. +`bid.hb_adid` is now also the cache UUID. +Match holds. No change to the win detection logic. + +### 4.6 GAM line item creative requirement (publisher action — not TS code) + +This is a **hard dependency outside the TS codebase**. The publisher must configure +GAM line items with a server-side compatible Prebid creative. The standard +client-side Universal Creative calls `pbjs.renderAd()` which requires Prebid.js to be +loaded — it will not be at first render (slim-Prebid loads post-`window.load`). + +The server-side compatible creative uses the `hb_cache_*` macros to fetch the markup +directly from PBS Cache: + +```html + +``` + +Alternatively, publishers using the Prebid Universal Creative package can use: + +```html + + +``` + +> **This creative configuration is a publisher/ad ops action, not a TS code change.** +> Document it in the integration guide and verify during onboarding. + +> **Cache TTL:** PBS Cache entries expire per the `bid.exp` field (default 300–3600s; +> the real response has `"exp": 3600`). Creative fetch must complete within this window. +> BFCache page restores after long idle sessions may hit expired cache entries — the +> creative will silently fail to render in that case. This is acceptable for Phase 1; +> the probability is low for typical session lengths. + +--- + +## 5. APS — Out of Scope + +APS does not use PBS Cache. APS bids will have `cache_id = None`, `cache_host = None`, +`cache_path = None`. The existing `ad_id` fallback path remains for APS. APS creative +rendering depends on Amazon's own GAM creative tag — separate from the Prebid path. + +APS win detection over-fires on the `!!bid.hb_bidder` fallback remain a known +limitation tracked separately. + +--- + +## 6. Files Changed + +| File | Change | +| ------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/auction/types.rs` | Add `cache_id`, `cache_host`, `cache_path` to `Bid` struct | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Extract `ext.prebid.cache.bids.{cacheId,url}` in `parse_bid_object`; update `AuctionBid` → `Bid` conversion to carry the three new fields | +| `crates/trusted-server-core/src/publisher.rs` | `build_bid_map`: use `cache_id` for `hb_adid`, emit `hb_cache_host`/`hb_cache_path` | + +> **Implementer note — `AuctionBid` → `Bid` conversion:** `prebid.rs` constructs an +> intermediate `AuctionBid` type that is later converted to the shared `Bid` type from +> `types.rs`. The new `cache_id`, `cache_host`, `cache_path` fields must be added to +> **both** types and the conversion must map them explicitly. Verify by grepping for +> where `AuctionBid` is constructed and where it is converted to `Bid`; if they are the +> same type (a type alias), only one struct needs the new fields. If they differ, both +> need updating or the fields will silently be `None` in `build_bid_map`. + +Test files: +| File | Change | +|---|---| +| `crates/trusted-server-core/src/integrations/prebid.rs` tests | Add test: PBS response with cache entry → correct `hb_adid`, `hb_cache_host`, `hb_cache_path` injected | +| `crates/trusted-server-core/src/publisher.rs` tests | Add test: `build_bid_map` emits cache fields when present; falls back to `ad_id` when absent | + +--- + +## 7. Testing + +**Unit tests:** + +1. `prebid.rs`: bid with `ext.prebid.cache.bids.cacheId` → `bid.cache_id = Some(uuid)`, `bid.cache_host = Some("openads.adsrvr.org")`, `bid.cache_path = Some("/cache")` +2. `prebid.rs`: bid without `ext.prebid.cache` → `bid.cache_id = None`, `bid.cache_host = None`, `bid.cache_path = None` +3. `prebid.rs`: bid with only `adid` (no cache) → `bid.ad_id = Some(...)`, `bid.cache_id = None` +4. `prebid.rs`: bid with malformed cache URL → `cache_host = None`, `cache_path = None`, no panic +5. `publisher.rs` `build_bid_map`: bid with `cache_id` → `hb_adid` uses `cache_id`, `hb_cache_host`/`hb_cache_path` emitted +6. `publisher.rs` `build_bid_map`: bid with no `cache_id` but has `ad_id` → `hb_adid` falls back to `ad_id`, no cache keys emitted +7. `publisher.rs` `build_bid_map`: APS bid (no `cache_id`, no `ad_id`) → no `hb_adid` emitted +8. `types.rs`: `Bid` with all three cache fields round-trips through `serde_json::to_string` / `from_str` + +> **Note for implementer:** `make_bid()` or equivalent `Bid` construction helpers in test modules +> must be updated to initialise `cache_id`, `cache_host`, `cache_path` to `None` +> (they will fail to compile otherwise once the fields are added to the struct). + +**Integration verification (manual):** + +After deploying, verify `window.tsjs.bids` in browser devtools shows `hb_cache_host` +and `hb_cache_path` present. Verify `hb_adid` matches the UUID in +`ext.prebid.cache.bids.cacheId` from the raw PBS response. + +--- + +## 8. Rollout Dependency Checklist + +Before this fix has end-to-end effect: + +- [ ] TS: this PR merged and deployed +- [ ] GAM: publisher ad ops updates all Prebid line item creatives to the server-side + cache-fetch variant (see §4.6) +- [ ] PBS: Prebid Cache enabled and populated (confirmed from real response — already + working) +- [ ] Verify: `window.tsjs.bids` shows correct cache UUID in `hb_adid` after deploy + +--- + +## 9. Known Remaining Gaps (not in scope) + +| Gap | Severity | Tracking | +| ----------------------------------------------------------------- | -------- | ------------------ | +| APS win detection over-fires nurl/burl | P1 | Separate issue | +| Dual bootstrap (`gpt_bootstrap.js` + `installTsAdInit`) sync risk | P2 | Separate issue | +| Slim-Prebid bundle not yet built | Phase 2 | §9.8 of design doc | diff --git a/trusted-server.toml b/trusted-server.toml index 899c8c895..c7b7ec96e 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -125,6 +125,7 @@ enabled = false script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true +# slim_prebid_url = "https://cdn.example.com/tsjs-prebid.min.js" # Consent forwarding configuration # Controls how Trusted Server interprets and forwards privacy consent signals. @@ -186,7 +187,7 @@ rewrite_script = true enabled = true providers = ["prebid", "aps"] mediator = "adserver_mock" -timeout_ms = 2000 +timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT_MS # Context keys the JS client is allowed to forward into auction requests. # Keys not in this list are silently dropped. An empty list blocks all keys. allowed_context_keys = ["permutive_segments"] @@ -195,7 +196,7 @@ allowed_context_keys = ["permutive_segments"] enabled = true pub_id = "test-pub" endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" -timeout_ms = 1000 +timeout_ms = 1000 # override per-publisher via TRUSTED_SERVER__INTEGRATIONS__APS__TIMEOUT_MS [integrations.google_tag_manager] enabled = false @@ -212,6 +213,10 @@ timeout_ms = 1000 # Inject before . # Visible in page source. Disable after investigation. # auction_html_comment = true +# +# Inject raw adm creative markup into window.tsjs.bids for GPT/GAM bridge +# debugging while PBS Cache is unavailable. NEVER enable in production. +# inject_adm_for_testing = true # Enable the JA4/TLS fingerprint debug endpoint at GET /_ts/debug/ja4. # Returns a plain-text response with the following fields (Fastly-observed values): # ja4 — JA4 TLS client fingerprint @@ -242,6 +247,53 @@ gam_network_id = "88059007" # drains in <50 ms but the auction runs to the limit. 500 ms is the recommended # default; raise only if your SSPs need more headroom and your analytics confirm # the DCL slip is acceptable. -auction_timeout_ms = 1500 +auction_timeout_ms = 1500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" +# Slot templates — override entire array via: +# TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT='[{"id":"...","gam_unit_path":"...",...}]' + +[[creative_opportunities.slot]] +id = "atf_sidebar_ad" +gam_unit_path = "/a/b/news" +div_id = "div-ad-atf-sidebar" +page_patterns = ["/20**", "/news/**"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-atf-sidebar" + +[[creative_opportunities.slot]] +id = "homepage_header_ad" +gam_unit_path = "/a/b/homepage" +div_id = "div-ad-homepage-header" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "atf" +zone = "header" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-homepage-header" + +[[creative_opportunities.slot]] +id = "homepage_footer_ad" +gam_unit_path = "/a/b/homepage" +div_id = "div-ad-homepage-footer" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }, { width = 768, height = 66 }] +floor_price = 0.50 + +[creative_opportunities.slot.targeting] +pos = "btf" +zone = "fixedBottom" + +[creative_opportunities.slot.providers.aps] +slot_id = "aps-slot-homepage-footer" From b77ebc4d1a50110066d805cb0d760d339cb62b7a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 16:55:00 +0530 Subject: [PATCH 074/195] Wire KV-enriched EID resolution into server-side auction paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both handle_publisher_request and handle_page_bids now run the full four-step EID pipeline (resolve_client_auction_eids → resolve_auction_eids → merge_auction_eids → gate_eids_by_consent) matching the client-side /auction endpoint. Previously both paths called parse_ts_eids_cookie, which read only the ts-eids browser cookie and skipped the KV identity graph lookup entirely. AuctionDispatch gains a registry field so the partner registry reaches handle_publisher_request without exceeding the seven-argument limit. handle_page_bids gains kv and registry parameters for the same reason. parse_ts_eids_cookie is moved to #[cfg(test)] as it is now test-only. --- .../trusted-server-adapter-fastly/src/main.rs | 49 +++++++------ .../src/auction/endpoints.rs | 6 +- crates/trusted-server-core/src/cookies.rs | 10 +-- .../src/creative_opportunities.rs | 8 ++- crates/trusted-server-core/src/publisher.rs | 69 ++++++++++++++----- 5 files changed, 97 insertions(+), 45 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 6f95373a1..7d8b98e48 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -328,6 +328,12 @@ async fn route_request( let path = req.get_path().to_string(); let method = req.get_method().clone(); + let registry_ref = if partner_registry.is_empty() { + None + } else { + Some(partner_registry) + }; + // Match known routes and handle them let (result, organic_route) = match (method, path.as_str()) { // Serve the tsjs library @@ -368,30 +374,32 @@ async fn route_request( } // Unified auction endpoint (returns creative HTML inline) - (Method::POST, "/auction") => { - let registry_ref = if partner_registry.is_empty() { - None - } else { - Some(partner_registry) - }; - ( - handle_auction( - settings, - orchestrator, - kv_graph.as_ref(), - registry_ref, - &ec_context, - runtime_services, - req, - ) - .await, - false, + (Method::POST, "/auction") => ( + handle_auction( + settings, + orchestrator, + kv_graph.as_ref(), + registry_ref, + &ec_context, + runtime_services, + req, ) - } + .await, + false, + ), // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path (Method::GET, "/__ts/page-bids") => ( - handle_page_bids(settings, orchestrator, runtime_services, slots, req).await, + handle_page_bids( + settings, + orchestrator, + runtime_services, + kv_graph.as_ref(), + registry_ref, + slots, + req, + ) + .await, false, ), @@ -443,6 +451,7 @@ async fn route_request( trusted_server_core::publisher::AuctionDispatch { orchestrator, slots, + registry: registry_ref, }, req, ) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 84d8b3f3b..f1c010de1 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -208,7 +208,7 @@ pub async fn handle_auction( /// Returns `None` when any prerequisite is missing (no KV store, no partner /// store, no EC, consent denied). On KV or partner-resolution errors, logs a /// warning and returns empty EIDs so the auction can proceed in degraded mode. -fn resolve_auction_eids( +pub(crate) fn resolve_auction_eids( kv: Option<&KvIdentityGraph>, registry: Option<&PartnerRegistry>, ec_context: &EcContext, @@ -251,7 +251,7 @@ fn extract_cookie_value(req: &Request, name: &str) -> Option { None } -fn resolve_client_auction_eids( +pub(crate) fn resolve_client_auction_eids( raw: Option<&JsonValue>, cookie_value: Option<&str>, ) -> Option> { @@ -347,7 +347,7 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { Some(Uid { id, atype, ext }) } -fn merge_auction_eids( +pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 302e35cea..2d558e314 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -3,17 +3,18 @@ //! This module provides functionality for parsing, stripping, and forwarding cookies //! used in the trusted server system. -use base64::{engine::general_purpose::STANDARD, Engine as _}; use cookie::{Cookie, CookieJar}; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; use http::Request; -use crate::constants::{ - COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_TS_EIDS, COOKIE_US_PRIVACY, -}; +#[cfg(test)] +use crate::constants::COOKIE_TS_EIDS; +use crate::constants::{COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_US_PRIVACY}; use crate::error::TrustedServerError; +#[cfg(test)] +use base64::{engine::general_purpose::STANDARD, Engine as _}; /// Cookie names carrying privacy consent signals. /// @@ -81,6 +82,7 @@ pub fn handle_request_cookies( /// Returns `None` if the cookie is absent, base64-malformed, JSON-malformed, /// or the decoded array is empty. Parse failures are logged at `debug` level /// so operators can diagnose JS SDK / server mismatches. +#[cfg(test)] #[must_use] pub(crate) fn parse_ts_eids_cookie(jar: Option<&CookieJar>) -> Option> { let value = jar?.get(COOKIE_TS_EIDS)?.value().to_owned(); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 2df61bb0c..a7b3d579a 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -457,7 +457,8 @@ mod tests { #[test] fn to_ad_slot_injects_trusted_server_when_prebid_bidders_empty() { let mut slot = make_slot("header", vec!["/"]); - slot.targeting.insert("zone".to_string(), "header".to_string()); + slot.targeting + .insert("zone".to_string(), "header".to_string()); slot.providers.prebid = Some(PrebidSlotParams { bidders: HashMap::new(), }); @@ -515,7 +516,10 @@ mod tests { .bidders .get("mocktioneer") .expect("should have mocktioneer bidder"); - assert_eq!(params.get("custom").and_then(serde_json::Value::as_bool), Some(true)); + assert_eq!( + params.get("custom").and_then(serde_json::Value::as_bool), + Some(true) + ); } #[test] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1bcd614c7..823977df5 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -18,15 +18,20 @@ use error_stack::{Report, ResultExt}; use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; +use crate::auction::endpoints::{ + merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, +}; use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; use crate::auction::types::{ AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::backend::BackendConfig; use crate::compat; -use crate::constants::HEADER_X_COMPRESS_HINT; -use crate::cookies::{handle_request_cookies, parse_ts_eids_cookie}; +use crate::consent::gate_eids_by_consent; +use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::cookies::handle_request_cookies; use crate::ec::kv::KvIdentityGraph; +use crate::ec::registry::PartnerRegistry; use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::http_util::{is_navigation_request, serve_static_with_etag, RequestInfo}; @@ -810,6 +815,8 @@ pub struct AuctionDispatch<'a> { pub orchestrator: &'a crate::auction::orchestrator::AuctionOrchestrator, /// Creative opportunity slot definitions matched against the request path. pub slots: &'a [crate::creative_opportunities::CreativeOpportunitySlot], + /// Partner registry for KV-backed EID resolution. `None` skips KV enrichment. + pub registry: Option<&'a PartnerRegistry>, } /// Proxies requests to the publisher's origin server. @@ -968,7 +975,19 @@ pub async fn handle_publisher_request( &request_info, req.get_header_str("user-agent"), ); - auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); + let ts_eids_value = cookie_jar + .as_ref() + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!("Server-side auction EIDs stripped by TCF consent gating"); + } let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); if client_ip.is_some() || geo.is_some() { let device = auction_request.device.get_or_insert(DeviceInfo { @@ -1456,6 +1475,8 @@ pub async fn handle_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, + kv: Option<&KvIdentityGraph>, + registry: Option<&PartnerRegistry>, slots: &[crate::creative_opportunities::CreativeOpportunitySlot], req: Request, ) -> Result> { @@ -1527,7 +1548,19 @@ pub async fn handle_page_bids( &request_info, req.get_header_str("user-agent"), ); - auction_request.user.eids = parse_ts_eids_cookie(cookie_jar.as_ref()); + let ts_eids_value = cookie_jar + .as_ref() + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let kv_eids = resolve_auction_eids(kv, registry, &ec_ctx); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); + } let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); if client_ip.is_some() || geo.is_some() { let device = auction_request.device.get_or_insert(DeviceInfo { @@ -3133,9 +3166,10 @@ mod tests { let services = noop_services(); let req = make_page_bids_request("/2024/01/my-article/"); - let response = handle_page_bids(&settings, &orchestrator, &services, &[], req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &[], req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); @@ -3170,9 +3204,10 @@ mod tests { let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); @@ -3206,9 +3241,10 @@ mod tests { let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("sec-purpose", "prefetch"); - let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); @@ -3240,9 +3276,10 @@ mod tests { let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match - let response = handle_page_bids(&settings, &orchestrator, &services, &slots, req) - .await - .expect("should return ok response"); + let response = + handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) + .await + .expect("should return ok response"); let body: serde_json::Value = serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); From 321fbafb3dc1123bba292ca5a125423d1c14b77c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 16:58:24 +0530 Subject: [PATCH 075/195] Remove dead build.rs from trusted-server-adapter-fastly The file only emitted a rerun-if-changed watch for creative-opportunities.toml, which was deleted when slot config was consolidated into trusted-server.toml. Config validation now runs entirely in trusted-server-core/build.rs. --- crates/trusted-server-adapter-fastly/build.rs | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 crates/trusted-server-adapter-fastly/build.rs diff --git a/crates/trusted-server-adapter-fastly/build.rs b/crates/trusted-server-adapter-fastly/build.rs deleted file mode 100644 index 0ad1f2dd9..000000000 --- a/crates/trusted-server-adapter-fastly/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=../../../creative-opportunities.toml"); -} From 459fe60179a89ca8057929f77a44d3cef17755b8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 17:04:19 +0530 Subject: [PATCH 076/195] Fix CI failures: update integration-tests lock file and prefer-const lint error --- crates/integration-tests/Cargo.lock | 237 +++++++++++--------- crates/js/lib/src/integrations/gpt/index.ts | 2 +- 2 files changed, 127 insertions(+), 112 deletions(-) diff --git a/crates/integration-tests/Cargo.lock b/crates/integration-tests/Cargo.lock index 9f80a0ef7..40fbe0039 100644 --- a/crates/integration-tests/Cargo.lock +++ b/crates/integration-tests/Cargo.lock @@ -201,9 +201,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "axum" @@ -274,9 +274,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" dependencies = [ "serde_core", ] @@ -316,7 +316,7 @@ checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" dependencies = [ "async-stream", "base64", - "bitflags 2.11.1", + "bitflags 2.13.0", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -387,9 +387,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -398,9 +398,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -423,9 +423,9 @@ checksum = "d8e6738dfb11354886f890621b4a34c0b177f75538023f7100b608ab9adbd66b" [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -441,9 +441,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.62" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "shlex", @@ -481,9 +481,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -530,9 +530,9 @@ checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "config" -version = "0.15.22" +version = "0.15.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68cfe19cd7d23ffde002c24ffa5cda73931913ef394d5eaaa32037dc940c0c" +checksum = "f316c6237b2d38be61949ecd15268a4c6ca32570079394a2444d9ce2c72a72d8" dependencies = [ "async-trait", "convert_case 0.6.0", @@ -903,9 +903,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -923,9 +923,9 @@ dependencies = [ [[package]] name = "docker_credential" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4564c274ebf369f501de192b02a0b81a5c4bda375abfe526aa70fc702fa6fa0" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ "base64", "serde", @@ -1043,9 +1043,9 @@ checksum = "7c6ba7d4eec39eaa9ab24d44a0e73a7949a1095a8b3f3abb11eddf27dbb56a53" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -1469,6 +1469,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -1520,6 +1526,15 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -1533,11 +1548,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -1584,9 +1599,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1629,9 +1644,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -2005,9 +2020,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "log", @@ -2018,9 +2033,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -2065,13 +2080,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2142,9 +2156,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lol_html" @@ -2152,7 +2166,7 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cssparser 0.36.0", "encoding_rs", @@ -2210,9 +2224,9 @@ checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "mime" @@ -2232,9 +2246,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2324,9 +2338,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-derive" @@ -2400,11 +2414,11 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.79" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "foreign-types", "libc", @@ -2431,9 +2445,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.115" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -2840,9 +2854,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2850,9 +2864,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools 0.14.0", @@ -2863,9 +2877,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -2972,7 +2986,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -3088,7 +3102,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4147b952f3f819eca0e99527022f7d6a8d05f111aeb0a62960c74eb283bec8fc" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "once_cell", "serde", "serde_derive", @@ -3147,7 +3161,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys", @@ -3171,9 +3185,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3305,7 +3319,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3328,7 +3342,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cssparser 0.34.0", "derive_more 0.99.20", "fxhash", @@ -3347,7 +3361,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cssparser 0.36.0", "derive_more 2.1.1", "log", @@ -3410,9 +3424,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -3455,9 +3469,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64", "bs58", @@ -3475,9 +3489,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -3520,9 +3534,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signature" @@ -3560,9 +3574,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3710,7 +3724,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4053,11 +4067,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "bytes", "futures-util", "http", @@ -4131,6 +4145,7 @@ dependencies = [ "fastly", "flate2", "futures", + "glob", "hex", "hmac", "http", @@ -4189,9 +4204,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -4217,9 +4232,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -4321,9 +4336,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -4417,9 +4432,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -4430,9 +4445,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -4440,9 +4455,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4450,9 +4465,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -4463,9 +4478,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] @@ -4498,7 +4513,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "hashbrown 0.15.5", "indexmap 2.14.0", "semver", @@ -4506,9 +4521,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", @@ -4526,9 +4541,9 @@ dependencies = [ [[package]] name = "which" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" +checksum = "c789537cf2f7f55be8e6192f92e464174ee55f91af622777f7f1ceb0dbccd03e" dependencies = [ "libc", ] @@ -4740,7 +4755,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -4758,7 +4773,7 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -4810,7 +4825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags 2.13.0", "indexmap 2.14.0", "log", "serde", @@ -4858,9 +4873,9 @@ dependencies = [ [[package]] name = "yaml-rust2" -version = "0.10.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" dependencies = [ "arraydeque", "encoding_rs", @@ -4869,9 +4884,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4892,18 +4907,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 4276bc7b8..1d11571bd 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -203,7 +203,7 @@ function injectAdmIntoSlot(divId: string, adm: string): void { try { // divId may be the container div (used by GPT slot) or the inner div. // Search both so we can find the GAM iframe wherever it was rendered. - let slotEl = document.getElementById(divId); + const slotEl = document.getElementById(divId); if (!slotEl) return; // Extract the first iframe src from the adm (e.g. mocktioneer creative From 66140220c932efe55cc15da4c1f88fef42dc6626 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 17:09:01 +0530 Subject: [PATCH 077/195] Update workspace Cargo.lock to resolve shared dependency version mismatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns log (0.4.29 → 0.4.32) and serde_json (1.0.149 → 1.0.150) with the versions already pulled into crates/integration-tests/Cargo.lock. --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fd679f6a..2d1ad743c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1582,9 +1582,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "log-fastly" @@ -2270,9 +2270,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", From e6f5fc890ff7875b3c8c19ed88c440b95b48b38b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 9 Jun 2026 17:33:59 +0530 Subject: [PATCH 078/195] Update spec to reflect consolidated slot config and current global namespace Replace all references to the deleted `creative-opportunities.toml` file with the `[creative_opportunities]` section in `trusted-server.toml`. Update all `window.__ts_*` global name references to the current `window.tsjs.*` namespace (tsjs.bids, tsjs.adSlots, tsjs.adInit). --- ...6-04-15-server-side-ad-templates-design.md | 144 +++++++++--------- 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md index 94fe1999a..bdf24ff9c 100644 --- a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -89,13 +89,14 @@ across every navigation in the user's clickstream rather than once per session. ## 4. Architecture -### 4.1 New File: `creative-opportunities.toml` +### 4.1 Slot configuration in `trusted-server.toml` -A new config file at the repo root, alongside `trusted-server.toml`. It holds all slot -templates: page pattern matching rules, ad formats, floor prices, GAM targeting -key-values, and per-provider bidder params. PBS bidder-level params (placement IDs, -account IDs) live in Prebid Server stored requests, keyed by slot ID. APS params are -specified inline per slot under `[slot.providers.aps]`. +Slot templates live in `trusted-server.toml` under `[[creative_opportunities.slot]]` +(consolidated from the original `creative-opportunities.toml`). Each entry holds page +pattern matching rules, ad formats, floor prices, GAM targeting key-values, and +per-provider bidder params. PBS bidder-level params (placement IDs, account IDs) live +in Prebid Server stored requests, keyed by slot ID. APS params are specified inline per +slot under `[slot.providers.aps]`. Loaded at build time via `include_str!()` and compiled into the WASM binary. Slot changes require a redeploy; this is intentional (fast reads, no KV overhead, no @@ -103,7 +104,7 @@ per-request cost). A migration path to KV-backed config is tracked in §9.5. `floor_price` is the publisher-owned hard floor per slot — the source of truth for the minimum acceptable bid price, enforced at the edge before bids reach the ad server. Any -bid below the floor is discarded at the orchestrator level before it enters `__ts_bids`. +bid below the floor is discarded at the orchestrator level before it enters `tsjs.bids`. SSPs may apply their own dynamic floors independently within their platforms; this floor is the publisher's baseline that supersedes all other floor logic by virtue of being enforced earliest in the pipeline. @@ -118,7 +119,7 @@ gam_network_id = "21765378893" # Optional. Defaults to [auction].timeout_ms if not set. # Recommended: 500ms (vs client-side 1000–1500ms) due to lower edge→PBS RTT. # This value is also the upper bound on the -close hold; once A_deadline -# fires, TS injects an empty __ts_bids and emits regardless. +# fires, TS injects an empty tsjs.bids and emits regardless. auction_timeout_ms = 500 # Granularity table for hb_pb price bucket strings. @@ -127,7 +128,7 @@ auction_timeout_ms = 500 price_granularity = "dense" ``` -#### `creative-opportunities.toml` schema +#### `[creative_opportunities]` schema ```toml [[slot]] @@ -278,10 +279,10 @@ request. Before firing, TS gates on: skip the auction. Avoids spending auction inventory on speculative navigations that may never paint. - **Method** — only `GET` requests trigger auctions. `HEAD` requests skip. -- **Slot match** — at least one slot in `creative-opportunities.toml` must match the +- **Slot match** — at least one slot in `[creative_opportunities]` (in `trusted-server.toml`) must match the request path. Empty match = no auction. -Skipped auctions emit no `__ts_bids` and let the page proceed unmodified by the ad +Skipped auctions emit no `tsjs.bids` and let the page proceed unmodified by the ad stack. Skipped requests still benefit from the EC cookie set / KV identity update paths that run independently of the auction. @@ -294,7 +295,7 @@ existing EC pipeline and is the load-bearing identity input to the auction (see Consent gating: - If consent is **absent or denied** (no TCF consent string, or purpose 1 not consented): - the auction is not fired. `__ts_bids` is omitted from the page. GPT falls back to its + the auction is not fired. `tsjs.bids` is omitted from the page. GPT falls back to its own auction. This is treated as a first-class edge case in §8. - **Mid-page consent revocation** is out of scope for Phase 1; bids already injected remain. Phase 2 will address consent event propagation. @@ -313,8 +314,7 @@ The orchestrator's existing behavior is unchanged: (`creative_opportunities.auction_timeout_ms`, falling back to `[auction].timeout_ms`) - Floor price filtering, bid unification, and winning bid selection are applied as today - PBS resolves bidder params from its stored requests by slot ID -- APS bidder params are read from `[slot.providers.aps]` in - `creative-opportunities.toml` +- APS bidder params are read from `[slot.providers.aps]` in `trusted-server.toml` #### The bounded `` hold @@ -344,7 +344,7 @@ In English: finished by the time we need it because we waited for origin too. - If origin drains before the auction completes: body close held until either auction completes or `A_deadline` fires. Hold is bounded by `A_deadline`. -- If `A_deadline` fires first: TS injects `__ts_bids = {}` (graceful no-bid fallback) +- If `A_deadline` fires first: TS injects `tsjs.bids = {}` (graceful no-bid fallback) and emits the close tag. GPT proceeds without bid targeting; GAM runs its own auction. This is the **soft inner deadline watchdog** — auction overrun never blocks the page past `A_deadline`. @@ -371,12 +371,12 @@ and resource load time, exactly the same as a page without TS in the path. TS injects two `, ContentType::Html)`. @@ -446,7 +446,7 @@ task, fallback to `{}` on watchdog) and calls > U+2029 are unicode-escaped to neutralize any markup that could break out of the > `", + b"", + ], + Arc::clone(&read_count), + ); + let mut processor = RecordingProcessor { + read_count: Arc::clone(&read_count), + body_close_processed_at: Arc::clone(&body_close_processed_at), + }; + let ad_bids_state = Arc::new(Mutex::new(None)); + let ctx = AuctionCollectCtx { + dispatched, + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + }; + let mut output = Vec::new(); + + body_close_hold_loop(reader, &mut output, &mut processor, ctx) + .await + .expect("should stream body with auction hold"); + + assert_eq!( + body_close_processed_at.load(Ordering::SeqCst), + 1, + "close-body tail should be processed as soon as it is found, before later chunks are read" + ); + assert_eq!( + std::str::from_utf8(&output).expect("should be utf8"), + "painted", + "post-body chunks should still stream in order" + ); + } + #[test] fn body_close_hold_buffer_holds_close_body_tail_in_single_chunk() { let mut hold = BodyCloseHoldBuffer::new(); diff --git a/trusted-server.toml b/trusted-server.toml index 73f225a18..e1c35e11d 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -215,7 +215,7 @@ rewrite_script = true [auction] enabled = true providers = ["prebid", "aps"] -mediator = "adserver_mock" +# mediator = "adserver_mock" timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT_MS # Context keys the JS client is allowed to forward into auction requests. # Keys not in this list are silently dropped. An empty list blocks all keys. From def951ab8f2b3c560a1694a570ca48b133738ab7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 10 Jun 2026 16:25:56 +0530 Subject: [PATCH 083/195] Add per-bidder Prebid nurl suppression and refresh metadata --- .env.example | 1 + .../js/lib/src/integrations/prebid/index.ts | 118 +++++++++++++++--- .../test/integrations/prebid/index.test.ts | 70 +++++++++++ .../src/integrations/prebid.rs | 66 +++++++++- docs/guide/configuration.md | 2 + docs/guide/integrations/prebid.md | 2 + ...6-04-15-server-side-ad-templates-design.md | 11 +- trusted-server.toml | 2 + 8 files changed, 247 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 1121ecd9b..cec5d91de 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,7 @@ TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_abc"}}}' # Preferred canonical env shape for future generic rules # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_abc"}}]' +# TRUSTED_SERVER__INTEGRATIONS__PREBID__SUPPRESS_NURL_BIDDERS=exampleBidder,anotherBidder # TRUSTED_SERVER__INTEGRATIONS__PREBID__AUTO_CONFIGURE=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__TEST_MODE=false diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index c395905ef..feb31e1a9 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -32,6 +32,7 @@ import './_user_ids.generated'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; +import type { AuctionSlot } from '../../core/types'; import { DEFAULT_PREBID_USER_ID_MODULES, PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; @@ -212,7 +213,13 @@ export function auctionBidsToPrebidBids(auctionBids: AuctionBid[], bidRequests: type PbjsConfig = Parameters[0]; type TrustedServerBid = { bidder?: string; params?: Record }; -type TrustedServerAdUnit = { code?: string; bids?: TrustedServerBid[] }; +type BannerSize = [number, number]; +type TrustedServerBanner = { sizes: BannerSize[]; name?: string }; +type TrustedServerAdUnit = { + code?: string; + mediaTypes?: { banner?: TrustedServerBanner }; + bids?: TrustedServerBid[]; +}; type TrustedServerBidRequest = { adUnitCode?: string; code?: string; @@ -232,6 +239,17 @@ type PrebidUserIdEid = { uids?: Array<{ id?: unknown; atype?: unknown; ext?: unknown }>; }; +type RefreshGptSlot = { + getSlotElementId?: () => string; + getTargeting?: (key: string) => string[]; + getSizes?: () => unknown[]; +}; + +const DEFAULT_REFRESH_SIZES: BannerSize[] = [ + [728, 90], + [300, 250], +]; + function sanitizeAuctionUid(uid: { id?: unknown; atype?: unknown; @@ -258,6 +276,63 @@ function isDefined(value: T | undefined): value is T { return value !== undefined; } +function isPositiveFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value > 0; +} + +function parseBannerSize(size: unknown): BannerSize | undefined { + if (Array.isArray(size) && isPositiveFiniteNumber(size[0]) && isPositiveFiniteNumber(size[1])) { + return [size[0], size[1]]; + } + + const gptSize = size as { getWidth?: () => unknown; getHeight?: () => unknown }; + const width = gptSize?.getWidth?.(); + const height = gptSize?.getHeight?.(); + if (isPositiveFiniteNumber(width) && isPositiveFiniteNumber(height)) { + return [width, height]; + } + + return undefined; +} + +function bannerSizesFromGptSlot(slot: RefreshGptSlot): BannerSize[] | undefined { + const sizes = slot.getSizes?.(); + if (!Array.isArray(sizes)) { + return undefined; + } + + const parsedSizes = sizes.map(parseBannerSize).filter(isDefined); + return parsedSizes.length > 0 ? parsedSizes : undefined; +} + +function bannerSizesFromInjectedSlot(slot: AuctionSlot | undefined): BannerSize[] | undefined { + const parsedSizes = slot?.formats?.map(parseBannerSize).filter(isDefined) ?? []; + return parsedSizes.length > 0 ? parsedSizes : undefined; +} + +function refreshSlotElementId(slot: RefreshGptSlot): string | undefined { + const elementId = slot.getSlotElementId?.(); + return elementId && elementId.length > 0 ? elementId : undefined; +} + +function findInjectedSlotForRefresh(slot: RefreshGptSlot): AuctionSlot | undefined { + const elementId = refreshSlotElementId(slot); + if (!elementId) { + return undefined; + } + + return window.tsjs?.adSlots?.find( + (adSlot) => + elementId === adSlot.div_id || + elementId === `${adSlot.div_id}-container` || + elementId.startsWith(adSlot.div_id) + ); +} + +function firstTargetingValue(values: string[] | undefined): string | undefined { + return values?.find((value) => value.length > 0); +} + function collectAuctionEids(): AuctionEid[] | undefined { if (typeof pbjs.getUserIdsAsEids !== 'function') { return undefined; @@ -524,13 +599,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { pubads.refresh = function (slots?: unknown[], opts?: unknown) { // For bare refresh() calls (no slots arg), get all registered slots from GPT // so we can filter out TS first-impression slots and auction the rest. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const targetSlots: any[] = slots ?? (pubads as any).getSlots?.() ?? []; + const targetSlots = ( + slots ?? + (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? + [] + ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); // Filter out TS first-impression slots — they don't need client-side refresh auctions. const nonTsSlots = targetSlots.filter( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (s: any) => !s.getTargeting?.('ts_initial')?.includes('1') + (slot) => !slot.getTargeting?.('ts_initial')?.includes('1') ); if (!nonTsSlots.length) { @@ -538,19 +615,24 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const adUnits = nonTsSlots.map((s: any) => ({ - code: s.getSlotElementId?.() ?? s, - mediaTypes: { - banner: { - sizes: [ - [728, 90], - [300, 250], - ] as [number, number][], - }, - }, - bids: [{ bidder: ADAPTER_CODE, params: { zone: 'refresh' } }], - })); + const adUnits = nonTsSlots.map((slot) => { + const injectedSlot = findInjectedSlotForRefresh(slot); + const zone = + injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)); + const banner: TrustedServerBanner = { + sizes: + bannerSizesFromInjectedSlot(injectedSlot) ?? + bannerSizesFromGptSlot(slot) ?? + DEFAULT_REFRESH_SIZES, + ...(zone ? { name: zone } : {}), + }; + + return { + code: refreshSlotElementId(slot) ?? 'refresh-slot', + mediaTypes: { banner }, + bids: [{ bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }], + }; + }); pbjs.requestBids({ adUnits, diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index f12345d25..c79bfd080 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -62,6 +62,7 @@ import { getInjectedConfig, auctionBidsToPrebidBids, installPrebidNpm, + installRefreshHandler, } from '../../../src/integrations/prebid/index'; import type { AuctionBid } from '../../../src/core/auction'; @@ -765,6 +766,75 @@ describe('prebid/installPrebidNpm with server-injected config', () => { }); }); +describe('prebid/installRefreshHandler', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + afterEach(() => { + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + it('builds refresh ad units from injected slot metadata', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage', pos: 'atf' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + mediaTypes: { + banner: { + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + }); +}); + describe('prebid/client-side bidders', () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 6c0ee0eaa..b757a1bf6 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -189,6 +189,14 @@ pub struct PrebidIntegrationConfig { /// client does not double-fire them via `sendBeacon`. Default: `false`. #[serde(default)] pub suppress_nurl: bool, + /// Bidder seats whose `nurl` and `burl` should be stripped before they reach + /// `window.tsjs.bids`. + /// + /// Use this when only specific PBS seats fire win/billing notifications + /// internally. The global [`suppress_nurl`](Self::suppress_nurl) switch still + /// suppresses every bidder when set. + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + pub suppress_nurl_bidders: Vec, } impl IntegrationConfig for PrebidIntegrationConfig { @@ -1341,6 +1349,15 @@ impl PrebidAuctionProvider { } } + fn should_suppress_bid_notifications(&self, bidder: &str) -> bool { + self.config.suppress_nurl + || self + .config + .suppress_nurl_bidders + .iter() + .any(|suppressed_bidder| suppressed_bidder == bidder) + } + /// Parse a single bid from `OpenRTB` response. fn parse_bid(&self, bid_obj: &Json, seat: &str) -> Result { let slot_id = bid_obj @@ -1370,7 +1387,8 @@ impl PrebidAuctionProvider { .and_then(|v| u32::try_from(v).ok()) .unwrap_or(0); - let nurl = if self.config.suppress_nurl { + let suppress_bid_notifications = self.should_suppress_bid_notifications(seat); + let nurl = if suppress_bid_notifications { None } else { bid_obj @@ -1379,7 +1397,7 @@ impl PrebidAuctionProvider { .map(std::string::ToString::to_string) }; - let burl = if self.config.suppress_nurl { + let burl = if suppress_bid_notifications { None } else { bid_obj @@ -1761,6 +1779,7 @@ mod tests { bid_param_override_rules: Vec::new(), consent_forwarding: ConsentForwardingMode::Both, suppress_nurl: false, + suppress_nurl_bidders: Vec::new(), } } @@ -4844,6 +4863,49 @@ set = { networkId = 42 } ); } + #[test] + fn parse_bid_strips_nurl_and_burl_for_configured_suppressed_bidder_only() { + let bid_json = serde_json::json!({ + "impid": "atf_sidebar_ad", + "price": 1.50, + "w": 300, + "h": 250, + "nurl": "https://ssp.example/win?id=abc123", + "burl": "https://ssp.example/bill?id=abc123" + }); + let config = PrebidIntegrationConfig { + suppress_nurl_bidders: vec!["appnexus".to_string()], + ..base_config() + }; + let provider = PrebidAuctionProvider::new(config); + + let suppressed_bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse suppressed bidder bid"); + let preserved_bid = provider + .parse_bid(&bid_json, "openx") + .expect("should parse unsuppressed bidder bid"); + + assert_eq!( + suppressed_bid.nurl, None, + "should strip nurl only for the configured bidder" + ); + assert_eq!( + suppressed_bid.burl, None, + "should strip burl only for the configured bidder" + ); + assert_eq!( + preserved_bid.nurl.as_deref(), + Some("https://ssp.example/win?id=abc123"), + "should preserve nurl for bidders not configured for suppression" + ); + assert_eq!( + preserved_bid.burl.as_deref(), + Some("https://ssp.example/bill?id=abc123"), + "should preserve burl for bidders not configured for suppression" + ); + } + #[test] fn parse_bid_preserves_ad_id_alongside_cache_id() { let bid_json = serde_json::json!({ diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9863de1c7..a7d93d1a8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -726,6 +726,8 @@ apply when the integration section exists in `trusted-server.toml`. | `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | +| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | +| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | | `debug` | Boolean | `false` | Enable debug mode (sets `ext.prebid.debug` and `returnallbidstatus`; surfaces debug metadata in responses) | | `test_mode` | Boolean | `false` | Set OpenRTB `test: 1` flag for non-billable test traffic (independent of `debug`) | | `debug_query_params` | String | `None` | Extra query params appended for debugging | diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 42e0d6b7a..1e8870051 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -58,6 +58,8 @@ set = { placementId = "_s2sHeaderPlacement" } | `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | +| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | +| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | | `debug` | Boolean | `false` | Enable Prebid debug mode (sets `ext.prebid.debug` and `ext.prebid.returnallbidstatus`; surfaces debug metadata in auction responses) | | `test_mode` | Boolean | `false` | Set the OpenRTB `test: 1` flag so bidders treat the auction as non-billable test traffic. Separate from `debug` to avoid suppressing real demand | | `debug_query_params` | String | `None` | Extra query params appended for debugging | diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md index bdf24ff9c..8617ef877 100644 --- a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -487,10 +487,11 @@ The `hb_adid` match confirms two things: that the slot was filled (`!event.isEmp **and** that **our** Prebid bid (not a direct deal or backfill) won the GAM line item match. Only then are SSP win/billing pixels fired. -**Per-bidder suppression** (`[integrations.].suppress_nurl`, default `false`) -is retained as an escape hatch in case a specific PBS deployment fires `nurl` -internally and wants to avoid double-firing. APS `burl` follows the same client-side -path. +**Per-bidder suppression** (`[integrations.prebid].suppress_nurl_bidders`, default +`[]`) is retained as an escape hatch in case a specific PBS seat fires `nurl` +internally and wants to avoid double-firing. `[integrations.prebid].suppress_nurl = +true` remains a deployment-wide compatibility switch. APS `burl` follows the same +client-side path. > **Operational note:** Client-side firing introduces a small (~50–200ms) delay in > win-pixel arrival vs server-side firing. SSPs accept this — it's identical to @@ -1047,7 +1048,7 @@ saving. synchronous bid read, `slotRenderEnded` nurl + burl firing, `ts_initial` sentinel; add lazy slim-Prebid loader scheduled for post-`window.load` - **`crates/trusted-server-core/src/integrations/prebid.rs`** — add - `suppress_nurl` per-bidder config (default `false`); **no server-side nurl firing + `suppress_nurl_bidders` per-bidder config (default `[]`); **no server-side nurl firing in the page-load path** (firing is client-side from `slotRenderEnded`) - **`trusted-server.toml`** — add `[creative_opportunities]` section - **`crates/trusted-server-core/src/settings.rs`** — add `CreativeOpportunitiesConfig` diff --git a/trusted-server.toml b/trusted-server.toml index e1c35e11d..efff74693 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -95,6 +95,8 @@ client_side_bidders = [] # Set to true if PBS is configured to fire win/billing notifications server-side # (ext.prebid.events.enabled), to prevent the client from double-firing nurl/burl. # suppress_nurl = false +# For per-bidder suppression, list PBS seats that fire win/billing internally. +# suppress_nurl_bidders = ["exampleBidder"] [integrations.nextjs] enabled = false From 80fe39220cbd2eed458ca3471331b183b51bd186 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 10 Jun 2026 19:19:30 +0530 Subject: [PATCH 084/195] Resolve server-side ad template review issues --- .env.example | 2 +- crates/js/lib/src/core/types.ts | 2 + crates/js/lib/src/integrations/gpt/index.ts | 39 +++++- .../js/lib/src/integrations/prebid/index.ts | 39 ++++-- .../lib/test/integrations/gpt/index.test.ts | 97 ++++++++++++++ .../test/integrations/prebid/index.test.ts | 80 ++++++++++++ crates/trusted-server-core/src/publisher.rs | 123 ++++++++++++++++-- docs/guide/auction-orchestration.md | 20 +-- docs/guide/configuration.md | 6 +- trusted-server.toml | 69 ++-------- 10 files changed, 374 insertions(+), 103 deletions(-) diff --git a/.env.example b/.env.example index cec5d91de..c2ac88e3a 100644 --- a/.env.example +++ b/.env.example @@ -37,7 +37,7 @@ TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=false # Prebid TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.com/openrtb2/auction +# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.example.com/openrtb2/auction # TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 # TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"bidder-name":{"param1":12345,"param2":"value"}}' diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 31f66d0a7..57a14f3ec 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -98,6 +98,8 @@ export interface TsjsApi { servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; + /** Slot-level GPT targeting keys TS applied on the previous route. */ + prevSlotTargetingKeys?: Record; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index b7a81bc98..21f1f120d 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -25,6 +25,14 @@ import { installGptGuard } from './script_guard'; */ const TS_INITIAL_TARGETING_KEY = 'ts_initial' as const; +const TS_BID_TARGETING_KEYS = [ + 'hb_pb', + 'hb_bidder', + 'hb_adid', + 'hb_cache_host', + 'hb_cache_path', +] as const; +const TS_BASE_TARGETING_KEYS = [...TS_BID_TARGETING_KEYS, TS_INITIAL_TARGETING_KEY] as const; // ------------------------------------------------------------------ // googletag type stubs (minimal surface needed by the shim) @@ -34,6 +42,7 @@ interface GoogleTagSlot { getAdUnitPath(): string; getSlotElementId(): string; setTargeting(key: string, value: string | string[]): GoogleTagSlot; + clearTargeting?(key?: string): GoogleTagSlot; addService(service: GoogleTagPubAdsService): GoogleTagSlot; getTargeting?(key: string): string[]; } @@ -82,6 +91,14 @@ function messageSourceBelongsToConfiguredSlot(source: MessageEventSource | null) ); } +function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { + if (typeof slot.clearTargeting !== 'function') return; + + for (const key of new Set(keys)) { + slot.clearTargeting(key); + } +} + interface GoogleTagPubAdsService { setTargeting(key: string, value: string | string[]): GoogleTagPubAdsService; getTargeting(key: string): string[]; @@ -333,6 +350,8 @@ export function installTsAdInit(): void { // All slots to refresh (TS-defined + publisher-owned reused). const slotsToRefresh: GoogleTagSlot[] = []; const divToSlotId: Record = {}; + const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; + const nextSlotTargetingKeys: Record = {}; slots.forEach((slot) => { // Resolve actual div ID: exact match first, then prefix query. @@ -363,19 +382,26 @@ export function installTsAdInit(): void { tsOwned = true; } + const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; + clearTargetingKeys(gptSlot, [ + ...TS_BASE_TARGETING_KEYS, + ...(prevSlotTargetingKeys[actualDivId] ?? []), + ...(prevSlotTargetingKeys[slotDivId2] ?? []), + ]); + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); - (['hb_pb', 'hb_bidder', 'hb_adid', 'hb_cache_host', 'hb_cache_path'] as const).forEach( - (key) => { - if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); - } - ); + TS_BID_TARGETING_KEYS.forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); + }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); // Map both inner div and container div → slot ID so slotRenderEnded // (which reports the GPT slot's div, i.e. slotDivId/container) can look up // the slot, while adm injection (which targets the inner div) also works. divToSlotId[actualDivId] = slot.id; - const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; if (slotDivId2 !== actualDivId) divToSlotId[slotDivId2] = slot.id; + const slotTargetingKeys = Object.keys(slot.targeting ?? {}); + nextSlotTargetingKeys[actualDivId] = slotTargetingKeys; + if (slotDivId2 !== actualDivId) nextSlotTargetingKeys[slotDivId2] = slotTargetingKeys; if (tsOwned) newSlots.push(gptSlot); slotsToRefresh.push(gptSlot); @@ -391,6 +417,7 @@ export function installTsAdInit(): void { ts.prevGptSlots = newSlots as unknown[]; // Replace (not merge) so destroyed slots from previous navigation don't linger. ts.divToSlotId = divToSlotId; + ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // enableSingleRequest and enableServices must only be called once per page load. if (!ts.servicesEnabled) { diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index feb31e1a9..faa6ec04a 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -39,6 +39,14 @@ import { DEFAULT_PREBID_USER_ID_MODULES, PREBID_USER_ID_MODULE_REGISTRY } from ' const ADAPTER_CODE = 'trustedServer'; const BIDDER_PARAMS_KEY = 'bidderParams'; const ZONE_KEY = 'zone'; +const TS_REFRESH_TARGETING_KEYS = [ + 'ts_initial', + 'hb_pb', + 'hb_bidder', + 'hb_adid', + 'hb_cache_host', + 'hb_cache_path', +] as const; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -242,6 +250,7 @@ type PrebidUserIdEid = { type RefreshGptSlot = { getSlotElementId?: () => string; getTargeting?: (key: string) => string[]; + clearTargeting?: (key?: string) => RefreshGptSlot; getSizes?: () => unknown[]; }; @@ -333,6 +342,14 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } +function clearRefreshTargeting(slot: RefreshGptSlot): void { + if (typeof slot.clearTargeting !== 'function') return; + + for (const key of TS_REFRESH_TARGETING_KEYS) { + slot.clearTargeting(key); + } +} + function collectAuctionEids(): AuctionEid[] | undefined { if (typeof pbjs.getUserIdsAsEids !== 'function') { return undefined; @@ -569,8 +586,9 @@ export function installPrebidNpm(config?: Partial): typeof pbjs * Wraps `googletag.pubads().refresh()` so that when the publisher's GPT * refresh policy fires (sticky anchor, viewability dwell, infinite scroll), * Prebid runs a fresh client-side auction for the refreshing slots before - * the GAM call. TS-owned first-impression slots (`ts_initial=1`) are excluded - * — they are managed server-side and should not re-auction client-side. + * the GAM call. TS-owned first-impression slots (`ts_initial=1`) are included + * on later publisher refreshes, but stale TS server-side targeting is cleared + * before fresh Prebid targeting is applied. * * Must be called after `installPrebidNpm()` and after GPT is loaded. * Idempotent: safe to call multiple times — wraps only once via a sentinel. @@ -598,24 +616,20 @@ export function installRefreshHandler(timeoutMs = 1500): void { const originalRefresh = pubads.refresh.bind(pubads); pubads.refresh = function (slots?: unknown[], opts?: unknown) { // For bare refresh() calls (no slots arg), get all registered slots from GPT - // so we can filter out TS first-impression slots and auction the rest. + // so we can auction the same concrete slot list and avoid stale targeting. const targetSlots = ( slots ?? (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? [] ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); - // Filter out TS first-impression slots — they don't need client-side refresh auctions. - const nonTsSlots = targetSlots.filter( - (slot) => !slot.getTargeting?.('ts_initial')?.includes('1') - ); - - if (!nonTsSlots.length) { - // All slots are TS-owned — pass through unchanged. + if (!targetSlots.length) { return originalRefresh(slots, opts); } - const adUnits = nonTsSlots.map((slot) => { + targetSlots.forEach(clearRefreshTargeting); + + const adUnits = targetSlots.map((slot) => { const injectedSlot = findInjectedSlotForRefresh(slot); const zone = injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)); @@ -638,8 +652,7 @@ export function installRefreshHandler(timeoutMs = 1500): void { adUnits, bidsBackHandler: () => { pbjs.setTargetingForGPTAsync?.(); - // Refresh only the non-TS slots (pass explicit list so TS slots are not re-refreshed). - originalRefresh(nonTsSlots, opts); + originalRefresh(targetSlots, opts); }, timeout: timeoutMs, }); diff --git a/crates/js/lib/test/integrations/gpt/index.test.ts b/crates/js/lib/test/integrations/gpt/index.test.ts index 839b121d6..08cedfc36 100644 --- a/crates/js/lib/test/integrations/gpt/index.test.ts +++ b/crates/js/lib/test/integrations/gpt/index.test.ts @@ -216,6 +216,103 @@ describe('GPT – installSlimPrebidLoader', () => { }); }); +describe('GPT – installTsAdInit', () => { + beforeEach(() => { + document.body.innerHTML = ''; + delete (window as any).tsjs; + delete (window as any).googletag; + }); + + afterEach(() => { + document.body.innerHTML = ''; + delete (window as any).tsjs; + delete (window as any).googletag; + }); + + it('clears stale TS-managed targeting before applying a new route to a reused GPT slot', async () => { + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + const slotTargeting = new Map([ + ['hb_pb', ['1.20']], + ['hb_bidder', ['kargo']], + ['hb_adid', ['old-ad']], + ['hb_cache_host', ['cache.example.com']], + ['hb_cache_path', ['/cache']], + ['ts_initial', ['1']], + ['pos', ['old-pos']], + ]); + const gptSlot: any = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), + setTargeting: vi.fn((key: string, value: string | string[]) => { + slotTargeting.set(key, Array.isArray(value) ? value : [value]); + return gptSlot; + }), + clearTargeting: vi.fn((key?: string) => { + if (key) { + slotTargeting.delete(key); + } else { + slotTargeting.clear(); + } + return gptSlot; + }), + }; + const pubads = { + getSlots: vi.fn(() => [gptSlot]), + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const cmd: Array<() => void> = []; + cmd.push = (...callbacks: Array<() => void>) => { + callbacks.forEach((callback) => callback()); + return cmd.length; + }; + + document.body.innerHTML = '
'; + (window as any).googletag = { + cmd, + pubads: () => pubads, + defineSlot: vi.fn(), + destroySlots: vi.fn(), + enableServices: vi.fn(), + }; + (window as any).tsjs = { + prevSlotTargetingKeys: { + 'div-ad-homepage-header': ['pos'], + }, + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + bids: {}, + }; + + installTsAdInit(); + (window as any).tsjs.adInit(); + + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(gptSlot.clearTargeting).toHaveBeenCalledWith('pos'); + expect(slotTargeting.get('hb_pb')).toBeUndefined(); + expect(slotTargeting.get('hb_bidder')).toBeUndefined(); + expect(slotTargeting.get('hb_adid')).toBeUndefined(); + expect(slotTargeting.get('hb_cache_host')).toBeUndefined(); + expect(slotTargeting.get('hb_cache_path')).toBeUndefined(); + expect(slotTargeting.get('pos')).toBeUndefined(); + expect(slotTargeting.get('zone')).toEqual(['homepage']); + expect(slotTargeting.get('ts_initial')).toEqual(['1']); + }); +}); + describe('GPT shim – runtime gating', () => { type GatedWindow = Window & { __tsjs_gpt_enabled?: boolean; diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index c79bfd080..18f8dd8cd 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -769,6 +769,7 @@ describe('prebid/installPrebidNpm with server-injected config', () => { describe('prebid/installRefreshHandler', () => { beforeEach(() => { vi.clearAllMocks(); + mockRequestBids.mockReset(); mockPbjs.requestBids = mockRequestBids; mockPbjs.adUnits = []; (window as any).tsjs = undefined; @@ -833,6 +834,85 @@ describe('prebid/installRefreshHandler', () => { }) ); }); + + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn((key: string) => { + if (key === 'ts_initial') return ['1']; + if (key === 'zone') return ['homepage']; + return []; + }), + getSizes: vi.fn(() => [ + { getWidth: () => 970, getHeight: () => 250 }, + { getWidth: () => 728, getHeight: () => 90 }, + ]), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + mediaTypes: { + banner: { + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(originalRefresh).not.toHaveBeenCalled(); + + const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; + bidsBackHandler(); + + expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); }); describe('prebid/client-side bidders', () => { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ef526cca0..1c4c7e02d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1034,10 +1034,7 @@ pub async fn handle_publisher_request( ); let consent_context = ec_context.consent().clone(); - let ec_id = ec_context - .ec_value() - .map(str::to_string) - .unwrap_or_default(); + let ec_id = ec_context.ec_value().filter(|_| ec_allowed); let cookie_jar = handle_request_cookies(&http_req)?; let geo = ec_context.geo_info().cloned(); @@ -1120,7 +1117,7 @@ pub async fn handle_publisher_request( }; let mut auction_request = build_auction_request( &slots_ctx, - &ec_id, + ec_id, &consent_context, &request_info, req.get_header_str("user-agent"), @@ -1129,7 +1126,11 @@ pub async fn handle_publisher_request( .as_ref() .and_then(|j| j.get(COOKIE_TS_EIDS)) .map(|c| c.value().to_owned()); - let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) + } else { + None + }; let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); @@ -1316,7 +1317,7 @@ pub(crate) struct MatchedSlotsContext<'a> { /// Build an [`AuctionRequest`] from matched creative opportunity slots. pub(crate) fn build_auction_request( slots_ctx: &MatchedSlotsContext<'_>, - ec_id: &str, + ec_id: Option<&str>, consent_context: &crate::consent::ConsentContext, request_info: &crate::http_util::RequestInfo, user_agent: Option<&str>, @@ -1330,15 +1331,20 @@ pub(crate) fn build_auction_request( "{}://{}{}", request_info.scheme, request_info.host, slots_ctx.request_path ); + let ec_id = ec_id.filter(|id| !id.is_empty()); + let request_id = ec_id.map_or_else( + || format!("ts-req-{}", uuid::Uuid::new_v4().simple()), + |id| format!("ts-{id}"), + ); AuctionRequest { - id: format!("ts-{}", ec_id), + id: request_id, slots, publisher: PublisherInfo { domain: request_info.host.clone(), page_url: Some(page_url.clone()), }, user: UserInfo { - id: Some(ec_id.to_string()), + id: ec_id.map(str::to_string), consent: Some(consent_context.clone()), eids: None, }, @@ -1477,7 +1483,7 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map should be infallible"); let escaped = html_escape_for_script(&json); format!( - "", + "", escaped ) } @@ -1592,7 +1598,7 @@ pub async fn handle_page_bids( EcContext::read_from_request(settings, &req).change_context(TrustedServerError::Proxy { message: "page-bids: failed to read EC context".to_string(), })?; - let ec_id = ec_ctx.ec_value().map(str::to_string).unwrap_or_default(); + let ec_id = ec_ctx.ec_value().filter(|_| ec_ctx.ec_allowed()); let consent_context = ec_ctx.consent().clone(); let geo = ec_ctx.geo_info().cloned(); let cookie_jar = handle_request_cookies(&http_req)?; @@ -1631,7 +1637,7 @@ pub async fn handle_page_bids( }; let mut auction_request = build_auction_request( &slots_ctx, - &ec_id, + ec_id, &consent_context, &request_info, req.get_header_str("user-agent"), @@ -1640,7 +1646,11 @@ pub async fn handle_page_bids( .as_ref() .and_then(|j| j.get(COOKIE_TS_EIDS)) .map(|c| c.value().to_owned()); - let client_eids = resolve_client_auction_eids(None, ts_eids_value.as_deref()); + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) + } else { + None + }; let kv_eids = resolve_auction_eids(kv, registry, &ec_ctx); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); @@ -2922,12 +2932,15 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - build_ad_slots_script, build_bid_map, build_bids_script, html_escape_for_script, + build_ad_slots_script, build_auction_request, build_bid_map, build_bids_script, + html_escape_for_script, MatchedSlotsContext, }; use crate::auction::types::{Bid, MediaType}; + use crate::consent::ConsentContext; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, }; + use crate::http_util::RequestInfo; use crate::price_bucket::PriceGranularity; use std::collections::HashMap; @@ -3350,6 +3363,88 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in bids script"); } + #[test] + fn bids_script_calls_ad_init_without_retry_timer() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + + let script = build_bids_script(&map); + + assert!( + script.contains("window.tsjs.adInit"), + "should hand off bids to adInit" + ); + assert!( + !script.contains("setTimeout"), + "should not retry adInit on a timer" + ); + assert!( + !script.contains("prevGptSlots"), + "should not use TS-owned slots as adInit success signal" + ); + } + + #[test] + fn auction_request_without_ec_id_omits_user_id_and_uses_non_ec_request_id() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + None, + &ConsentContext::default(), + &request_info, + Some("Mozilla/5.0"), + ); + + assert_eq!(request.user.id, None, "should not forward an EC user id"); + assert!( + request.id.starts_with("ts-req-"), + "should use a non-EC request id, got {}", + request.id + ); + } + + #[test] + fn auction_request_with_ec_id_sets_user_id_and_ec_request_id() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + Some("ec-abc"), + &ConsentContext::default(), + &request_info, + Some("Mozilla/5.0"), + ); + + assert_eq!( + request.user.id.as_deref(), + Some("ec-abc"), + "should forward EC id when identity consent allows it" + ); + assert_eq!( + request.id, "ts-ec-abc", + "should preserve existing EC-derived request id when present" + ); + } + #[test] fn html_escape_encodes_special_chars() { assert_eq!( diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 3a55bc3de..d75958812 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -373,8 +373,8 @@ This is why mediation is important when using APS: without a mediator, APS bids ```toml [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 800 ``` @@ -593,8 +593,8 @@ debug = false [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 800 [integrations.adserver_mock] @@ -629,12 +629,12 @@ price_floor = 0.50 #### `[integrations.aps]` -| Field | Type | Default | Description | -| ------------ | ------ | ------------------------------------------- | --------------------------- | -| `enabled` | bool | `false` | Enable APS provider | -| `pub_id` | string | — | APS publisher ID (required) | -| `endpoint` | string | `https://aax.amazon-adsystem.com/e/dtb/bid` | APS TAM endpoint | -| `timeout_ms` | u32 | `800` | Request timeout | +| Field | Type | Default | Description | +| ------------ | ------ | ----------------------------------- | --------------------------- | +| `enabled` | bool | `false` | Enable APS provider | +| `pub_id` | string | — | APS publisher ID (required) | +| `endpoint` | string | `https://aps.example.com/e/dtb/bid` | APS TAM endpoint | +| `timeout_ms` | u32 | `800` | Request timeout | #### `[integrations.adserver_mock]` diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index a7d93d1a8..aceec9fa6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -85,7 +85,7 @@ secret_store_id = "01GYYY" [integrations.prebid] enabled = true -server_url = "https://prebid-server.com/openrtb2/auction" +server_url = "https://prebid-server.example.com/openrtb2/auction" timeout_ms = 1200 bidders = ["kargo", "appnexus", "openx"] client_side_bidders = ["rubicon"] @@ -901,8 +901,8 @@ timeout_ms = 2000 [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" [integrations.prebid] enabled = true diff --git a/trusted-server.toml b/trusted-server.toml index efff74693..b8d3c50b6 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -57,8 +57,8 @@ config_store_id = "" # set config/secret store ids for k secret_store_id = "" [integrations.prebid] -enabled = true -server_url = "http://68.183.113.79:8000" +enabled = false +server_url = "https://prebid-server.example.com/openrtb2/auction" timeout_ms = 1000 bidders = ["kargo", "appnexus", "openx"] debug = false @@ -215,8 +215,8 @@ rewrite_script = true # ] [auction] -enabled = true -providers = ["prebid", "aps"] +enabled = false +providers = ["prebid"] # mediator = "adserver_mock" timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT_MS # Context keys the JS client is allowed to forward into auction requests. @@ -224,9 +224,9 @@ timeout_ms = 2000 # override per-publisher via TRUSTED_SERVER__AUCTION__TIMEOUT allowed_context_keys = ["permutive_segments"] [integrations.aps] -enabled = true -pub_id = "test-pub" -endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" +enabled = false +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 1000 # override per-publisher via TRUSTED_SERVER__INTEGRATIONS__APS__TIMEOUT_MS [integrations.google_tag_manager] @@ -235,8 +235,8 @@ container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" [integrations.adserver_mock] -enabled = true -endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" +enabled = false +endpoint = "https://mediator.example.com/adserver/mediate" timeout_ms = 1000 # Debug configuration (all flags default to false — do not enable in production) @@ -271,7 +271,7 @@ timeout_ms = 1000 permutive_segments = "permutive" [creative_opportunities] -gam_network_id = "88059007" +gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on # DOMContentLoaded and window.load. Worst case: a cache-hit page where origin @@ -281,50 +281,7 @@ gam_network_id = "88059007" auction_timeout_ms = 1500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" -# Slot templates — override entire array via: +# No slot templates are enabled in the checked-in default config. Add +# `[[creative_opportunities.slot]]` entries via private config or override the +# entire array via: # TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT='[{"id":"...","gam_unit_path":"...",...}]' - -[[creative_opportunities.slot]] -id = "atf_sidebar_ad" -gam_unit_path = "/a/b/news" -div_id = "div-ad-atf-sidebar" -page_patterns = ["/20**", "/news/**"] -formats = [{ width = 300, height = 250 }] -floor_price = 0.50 - -[creative_opportunities.slot.targeting] -pos = "atf" -zone = "atfSidebar" - -[creative_opportunities.slot.providers.aps] -slot_id = "aps-slot-atf-sidebar" - -[[creative_opportunities.slot]] -id = "homepage_header_ad" -gam_unit_path = "/a/b/homepage" -div_id = "div-ad-homepage-header" -page_patterns = ["/"] -formats = [{ width = 728, height = 90 }] -floor_price = 0.50 - -[creative_opportunities.slot.targeting] -pos = "atf" -zone = "header" - -[creative_opportunities.slot.providers.aps] -slot_id = "aps-slot-homepage-header" - -[[creative_opportunities.slot]] -id = "homepage_footer_ad" -gam_unit_path = "/a/b/homepage" -div_id = "div-ad-homepage-footer" -page_patterns = ["/"] -formats = [{ width = 728, height = 90 }, { width = 768, height = 66 }] -floor_price = 0.50 - -[creative_opportunities.slot.targeting] -pos = "btf" -zone = "fixedBottom" - -[creative_opportunities.slot.providers.aps] -slot_id = "aps-slot-homepage-footer" From d3d43bc9700a314c30daed037fb5a09a3e03548c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 12 Jun 2026 13:29:09 +0530 Subject: [PATCH 085/195] Resolve server-side ad template auction review findings - Fail closed on consent: add consent_allows_server_side_auction() helper requiring effective TCF/GPP Purpose 1 for GDPR and unknown jurisdictions (or any request carrying an EU TCF signal); used by both the publisher navigation auction and /__ts/page-bids - Pass the adapter's geo-aware EcContext into handle_page_bids so the jurisdiction decision sees real geo instead of always-unknown - Make [auction].enabled a real kill switch for the automatic publisher navigation ad stack and the /__ts/page-bids auction - Normalize Prebid server_url: use it as-is when it already ends with /openrtb2/auction, otherwise append the path (backward compatible) - Advertise the effective auction budget in provider payloads: PBS tmax and APS timeout now use the orchestrator-capped context.timeout_ms instead of raw provider config - Add a one-shot adInitRefreshInProgress bypass so slim-Prebid's refresh wrapper passes adInit()'s internal refresh straight to GPT instead of clearing server-side targeting with a duplicate client-side auction - Sweep stale TS targeting (hb_*, ts_initial, route keys) from all previously TS-touched GPT slots before applying a new SPA route - Map the GPT slot element ID (container div) in the inline bootstrap's divToSlotId so container-backed slots fire nurl/burl beacons --- crates/js/lib/src/core/types.ts | 7 + .../js/lib/src/integrations/gpt/index.test.ts | 92 ++++++ crates/js/lib/src/integrations/gpt/index.ts | 33 +- .../js/lib/src/integrations/prebid/index.ts | 9 + .../test/integrations/prebid/index.test.ts | 50 +++ .../trusted-server-adapter-fastly/src/main.rs | 9 +- crates/trusted-server-core/src/consent/mod.rs | 113 ++++++- .../src/integrations/aps.rs | 49 ++- .../src/integrations/gpt_bootstrap.js | 8 + .../src/integrations/prebid.rs | 96 +++++- crates/trusted-server-core/src/publisher.rs | 303 ++++++++++-------- 11 files changed, 615 insertions(+), 154 deletions(-) diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 57a14f3ec..4fb99f3b4 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -100,6 +100,13 @@ export interface TsjsApi { divToSlotId?: Record; /** Slot-level GPT targeting keys TS applied on the previous route. */ prevSlotTargetingKeys?: Record; + /** + * One-shot bypass for the slim-Prebid refresh wrapper: true only while + * adInit() runs its internal refresh of server-side-targeted slots, so the + * wrapper passes that refresh straight to GPT instead of starting a + * client-side auction that would clear the just-applied TS targeting. + */ + adInitRefreshInProgress?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/src/integrations/gpt/index.test.ts index 5573993b8..aaf2657b4 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/src/integrations/gpt/index.test.ts @@ -125,6 +125,98 @@ describe('installTsAdInit', () => { fetchSpy.mockRestore(); }); + it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + let flagDuringRefresh: boolean | undefined; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(() => { + flagDuringRefresh = (window as TestWindow).tsjs!.adInitRefreshInProgress; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(mockPubads.refresh).toHaveBeenCalled(); + expect(flagDuringRefresh).toBe(true); + expect((window as TestWindow).tsjs!.adInitRefreshInProgress).toBe(false); + }); + + it('clears stale TS targeting from previously touched slots when the new route has no TS slots', async () => { + const clearTargeting = vi.fn().mockReturnThis(); + const staleSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting, + getSlotElementId: vi.fn().mockReturnValue('div-old-route'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([staleSlot]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn(), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + // New route has no matching TS slots. + adSlots: [], + bids: {}, + // Previous route touched the publisher-owned slot on div-old-route. + divToSlotId: { 'div-old-route': 'old_slot' }, + prevSlotTargetingKeys: { 'div-old-route': ['pos'] }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('./index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('pos'); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect((window as TestWindow).tsjs!.divToSlotId).toEqual({}); + expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); + }); + it('keeps the GAM path when debug adm is present', async () => { const slotEl = document.getElementById('div-atf-sidebar')!; const mockSlot = { diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 21f1f120d..9054d4c15 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -353,6 +353,27 @@ export function installTsAdInit(): void { const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; const nextSlotTargetingKeys: Record = {}; + // Clear TS-managed targeting from every previously TS-touched GPT slot + // before applying the current route. Without this sweep, navigating to a + // route with no matching TS slots (or one where a previously touched + // publisher-owned slot is absent from the new slot list) leaves stale + // hb_* / ts_initial / route targeting that later publisher refreshes + // would reuse. + const prevTouchedDivIds = new Set([ + ...Object.keys(prevSlotTargetingKeys), + ...Object.keys(ts.divToSlotId ?? {}), + ]); + if (prevTouchedDivIds.size > 0) { + (g.pubads!().getSlots?.() ?? []).forEach((gptSlot: GoogleTagSlot) => { + const elementId = gptSlot.getSlotElementId(); + if (!prevTouchedDivIds.has(elementId)) return; + clearTargetingKeys(gptSlot, [ + ...TS_BASE_TARGETING_KEYS, + ...(prevSlotTargetingKeys[elementId] ?? []), + ]); + }); + } + slots.forEach((slot) => { // Resolve actual div ID: exact match first, then prefix query. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when @@ -451,7 +472,17 @@ export function installTsAdInit(): void { } if (slotsToRefresh.length > 0) { - g.pubads!().refresh(slotsToRefresh); + // One-shot bypass: this internal refresh delivers the just-applied + // server-side targeting to GAM. If slim-Prebid has wrapped refresh(), + // it must pass this call straight through — not clear the targeting + // and run a duplicate client-side auction. Later publisher-initiated + // refreshes of the same slots still go through the wrapper normally. + ts.adInitRefreshInProgress = true; + try { + g.pubads!().refresh(slotsToRefresh); + } finally { + ts.adInitRefreshInProgress = false; + } } }); }; diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index faa6ec04a..cd2ffd265 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -615,6 +615,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { const originalRefresh = pubads.refresh.bind(pubads); pubads.refresh = function (slots?: unknown[], opts?: unknown) { + // One-shot bypass for adInit()'s internal refresh: that refresh delivers + // freshly applied server-side targeting to GAM and must not be turned + // into a client-side auction (which would clear the TS targeting). + // Publisher-initiated refreshes of the same slots are not flagged and + // still run a fresh client-side auction below. + if (window.tsjs?.adInitRefreshInProgress) { + return originalRefresh(slots, opts); + } + // For bare refresh() calls (no slots arg), get all registered slots from GPT // so we can auction the same concrete slot list and avoid stale targeting. const targetSlots = ( diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 18f8dd8cd..31a922869 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -913,6 +913,56 @@ describe('prebid/installRefreshHandler', () => { expect(setTargetingForGPTAsync).toHaveBeenCalled(); expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); }); + + it('passes the adInit internal refresh straight to GPT without a client-side auction', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { adInitRefreshInProgress: true }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); + + it('runs a client-side auction for publisher refreshes after adInit completes', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { adInitRefreshInProgress: false }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); }); describe('prebid/client-side bidders', () => { diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 884f2c15e..9a0f65c81 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -392,11 +392,14 @@ async fn route_request( (Method::GET, "/__ts/page-bids") => ( handle_page_bids( settings, - orchestrator, runtime_services, kv_graph.as_ref(), - registry_ref, - slots, + trusted_server_core::publisher::AuctionDispatch { + orchestrator, + slots, + registry: registry_ref, + }, + &ec_context, req, ) .await, diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index cd73acc9e..fad04d6b9 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -291,6 +291,26 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { .or_else(|| ctx.gpp.as_ref().and_then(|g| g.eu_tcf.as_ref())) } +/// Returns whether a server-side auction may be dispatched for this request. +/// +/// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present +/// (`gdpr_applies`) **or** the request's geo jurisdiction is GDPR or unknown, +/// the effective TCF consent (standalone TC string or GPP EU TCF section) +/// must grant Purpose 1 (storage/access). Only requests from a known +/// non-GDPR jurisdiction with no EU TCF signal are freely allowed. +#[must_use] +pub fn consent_allows_server_side_auction(ctx: &ConsentContext) -> bool { + let requires_tcf_purpose1 = ctx.gdpr_applies + || matches!( + ctx.jurisdiction, + jurisdiction::Jurisdiction::Gdpr | jurisdiction::Jurisdiction::Unknown + ); + if !requires_tcf_purpose1 { + return true; + } + effective_tcf(ctx).is_some_and(|tcf| tcf.has_purpose_consent(1)) +} + /// Returns whether TCF consent allows EID transmission. #[must_use] fn allows_eid_transmission(tcf: &types::TcfConsent) -> bool { @@ -644,8 +664,8 @@ mod tests { use super::{ allows_ec_creation, apply_expiration_check, apply_tcf_conflict_resolution, - build_consent_context, build_context_from_signals, has_explicit_ec_withdrawal, - ConsentPipelineInput, + build_consent_context, build_context_from_signals, consent_allows_server_side_auction, + has_explicit_ec_withdrawal, ConsentPipelineInput, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -746,6 +766,95 @@ mod tests { } } + #[test] + fn auction_allowed_for_known_non_gdpr_jurisdiction_without_tcf_signal() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "known non-GDPR jurisdiction with no EU TCF signal should allow auction" + ); + } + + #[test] + fn auction_fails_closed_for_gdpr_jurisdiction_without_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "GDPR jurisdiction without a TCF signal should fail closed" + ); + } + + #[test] + fn auction_fails_closed_for_unknown_jurisdiction_without_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "unknown jurisdiction without a TCF signal should fail closed" + ); + } + + #[test] + fn auction_fails_closed_when_tcf_signal_present_without_purpose1() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gdpr_applies: true, + tcf: Some(TcfBuilder::new().with_storage(false).build()), + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "EU TCF signal without Purpose 1 should block auction even outside GDPR geo" + ); + } + + #[test] + fn auction_allowed_for_gdpr_jurisdiction_with_purpose1_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfBuilder::new().with_storage(true).build()), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "GDPR jurisdiction with Purpose 1 consent should allow auction" + ); + } + + #[test] + fn auction_allowed_with_purpose1_via_gpp_eu_tcf_section() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + gpp: Some(GppConsent { + version: 1, + section_ids: vec![2], + eu_tcf: Some(TcfBuilder::new().with_storage(true).build()), + us_sale_opt_out: None, + }), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "Purpose 1 granted via GPP EU TCF section should allow auction" + ); + } + #[test] fn missing_geo_keeps_unknown_jurisdiction_and_blocks_ec_creation() { let req = build_request(); diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 34e26ebea..b415e5c88 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -309,7 +309,15 @@ impl ApsAuctionProvider { /// creative-opportunity slot ID so the caller can remap bids in the response. /// Populates consent fields (GDPR, US Privacy, GPP) from the /// [`ConsentContext`](crate::consent::ConsentContext) attached to the request. - fn to_aps_request(&self, request: &AuctionRequest) -> (ApsBidRequest, HashMap) { + /// + /// `timeout_ms` is the effective auction budget for this provider (already + /// capped by the orchestrator) — advertised to APS so it never expects more + /// time than the edge will actually wait. + fn to_aps_request( + &self, + request: &AuctionRequest, + timeout_ms: u32, + ) -> (ApsBidRequest, HashMap) { let mut slot_id_map: HashMap = HashMap::new(); let slots: Vec = request .slots @@ -364,7 +372,7 @@ impl ApsAuctionProvider { slots, page_url: request.publisher.page_url.clone(), user_agent: request.device.as_ref().and_then(|d| d.user_agent.clone()), - timeout: Some(self.config.timeout_ms), + timeout: Some(timeout_ms), gdpr, us_privacy, gpp, @@ -539,7 +547,10 @@ impl AuctionProvider for ApsAuctionProvider { // Transform to APS format; store the APS-slot-ID → creative-slot-ID map so // parse_response can remap bids back to the creative opportunity slot ID. - let (aps_request, slot_id_map) = self.to_aps_request(request); + // `context.timeout_ms` is the effective budget the orchestrator granted + // this provider — the payload must advertise the same deadline the edge + // backend enforces below. + let (aps_request, slot_id_map) = self.to_aps_request(request, context.timeout_ms); *self .slot_id_map .lock() @@ -760,7 +771,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let auction_request = create_test_auction_request(); - let (aps_request, _slot_id_map) = provider.to_aps_request(&auction_request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&auction_request, 800); // Verify basic fields assert_eq!(aps_request.pub_id, "5128"); @@ -830,7 +841,7 @@ mod tests { context: HashMap::new(), }; - let (aps_request, slot_id_map) = provider.to_aps_request(&request); + let (aps_request, slot_id_map) = provider.to_aps_request(&request, 800); assert_eq!( aps_request.slots[0].slot_id, "aps-slot-atf-sidebar", "should send configured APS slot ID to APS" @@ -1069,6 +1080,28 @@ mod tests { assert!(!provider.supports_media_type(&MediaType::Native)); } + #[test] + fn aps_payload_timeout_uses_effective_auction_budget_not_provider_config() { + // Provider config says 1000ms but the auction budget grants only 500ms — + // the payload must advertise the tighter effective deadline. + let config = ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 1000, + }; + let provider = ApsAuctionProvider::new(config); + let request = create_test_auction_request(); + + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 500); + + assert_eq!( + aps_request.timeout, + Some(500), + "should advertise the effective auction budget, not the provider config timeout" + ); + } + #[test] fn test_aps_request_includes_consent_fields() { use crate::consent::ConsentContext; @@ -1091,7 +1124,7 @@ mod tests { ..Default::default() }); - let (aps_request, _slot_id_map) = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 800); // Verify GDPR consent let gdpr = aps_request.gdpr.expect("should have gdpr"); @@ -1120,7 +1153,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let request = create_test_auction_request(); // consent is None - let (aps_request, _slot_id_map) = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 800); assert!(aps_request.gdpr.is_none()); assert!(aps_request.us_privacy.is_none()); @@ -1147,7 +1180,7 @@ mod tests { ..Default::default() }); - let (aps_request, _slot_id_map) = provider.to_aps_request(&request); + let (aps_request, _slot_id_map) = provider.to_aps_request(&request, 800); let json = serde_json::to_value(&aps_request).expect("should serialize"); // GDPR fields present diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 0c7ea0dd2..cd4b05d42 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -75,7 +75,15 @@ }); // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); + // Map both the inner div and the GPT slot's element ID (the + // "-container" div when TS defined the slot there) so slotRenderEnded + // — which reports the GPT slot element ID — can find the slot for + // nurl/burl beacon firing. divToSlotId[actualDivId] = slot.id; + var slotElementId = s.getSlotElementId(); + if (slotElementId && slotElementId !== actualDivId) { + divToSlotId[slotElementId] = slot.id; + } if (tsOwned) newSlots.push(s); slotsToRefresh.push(s); }); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index b757a1bf6..d8e411aed 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -904,6 +904,21 @@ impl PrebidAuctionProvider { }) } + /// Returns the full Prebid Server `OpenRTB2` auction endpoint URL. + /// + /// Backward-compatible normalization: `server_url` may be configured as + /// either the PBS origin (path is appended here) or the full endpoint + /// already ending in `/openrtb2/auction` (used as-is, ignoring a trailing + /// slash) — both shapes produce the same request URL. + fn auction_endpoint_url(&self) -> String { + let base = self.config.server_url.trim_end_matches('/'); + if base.ends_with("/openrtb2/auction") { + base.to_string() + } else { + format!("{base}/openrtb2/auction") + } + } + /// Convert auction request to `OpenRTB` format with all enrichments. fn to_openrtb( &self, @@ -1168,7 +1183,12 @@ impl PrebidAuctionProvider { .get_header_str(header::REFERER) .map(std::string::ToString::to_string); - let tmax = to_openrtb_i32(self.config.timeout_ms, "tmax", "request"); + // Advertise the effective auction budget, not the raw provider config: + // the orchestrator caps `context.timeout_ms` to the remaining auction + // budget, and the edge backend stops waiting after that long. Telling + // PBS it has more time than the edge will wait turns partial bids into + // edge timeouts. + let tmax = to_openrtb_i32(context.timeout_ms, "tmax", "request"); OpenRtbRequest { id: Some(request.id.clone()), @@ -1622,8 +1642,8 @@ impl AuctionProvider for PrebidAuctionProvider { if log::log_enabled!(log::Level::Debug) { match serde_json::to_string_pretty(&openrtb) { Ok(json) => log::debug!( - "Prebid OpenRTB request to {}/openrtb2/auction:\n{}", - self.config.server_url, + "Prebid OpenRTB request to {}:\n{}", + self.auction_endpoint_url(), json ), Err(e) => { @@ -1633,10 +1653,7 @@ impl AuctionProvider for PrebidAuctionProvider { } // Create HTTP request - let mut pbs_req = Request::new( - Method::POST, - format!("{}/openrtb2/auction", self.config.server_url), - ); + let mut pbs_req = Request::new(Method::POST, self.auction_endpoint_url()); copy_request_headers( context.request, &mut pbs_req, @@ -3073,7 +3090,7 @@ server_url = "https://prebid.example" assert_eq!( openrtb.tmax, Some(1000), - "should set tmax from config timeout_ms" + "should set tmax from the effective auction context timeout" ); assert_eq!( openrtb.cur, @@ -3083,15 +3100,72 @@ server_url = "https://prebid.example" } #[test] - fn to_openrtb_omits_tmax_when_timeout_exceeds_i32_max() { + fn auction_endpoint_url_appends_path_to_base_origin() { + let provider = PrebidAuctionProvider::new(base_config()); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should append /openrtb2/auction to a base origin" + ); + } + + #[test] + fn auction_endpoint_url_does_not_double_append_full_endpoint() { + let mut config = base_config(); + config.server_url = "https://prebid.example/openrtb2/auction".to_string(); + let provider = PrebidAuctionProvider::new(config); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should use a full endpoint URL as-is" + ); + let mut config = base_config(); - config.timeout_ms = i32::MAX as u32 + 1; + config.server_url = "https://prebid.example/openrtb2/auction/".to_string(); + let provider = PrebidAuctionProvider::new(config); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should normalize a trailing slash on a full endpoint URL" + ); + } + + #[test] + fn to_openrtb_tmax_uses_effective_context_timeout_not_provider_config() { + // Provider config says 1000ms but the auction budget is only 500ms — + // PBS must be told the tighter effective deadline, otherwise the edge + // gives up before PBS responds. + let config = base_config(); + assert_eq!(config.timeout_ms, 1000, "should start from 1000ms config"); let provider = PrebidAuctionProvider::new(config); let auction_request = create_test_auction_request(); let settings = make_settings(); let request = Request::get("https://pub.example/auction"); - let context = create_test_auction_context(&settings, &request); + let context = shared_test_auction_context(&settings, &request, 500); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert_eq!( + openrtb.tmax, + Some(500), + "should set tmax from the effective auction context timeout, not provider config" + ); + } + + #[test] + fn to_openrtb_omits_tmax_when_timeout_exceeds_i32_max() { + let provider = PrebidAuctionProvider::new(base_config()); + let auction_request = create_test_auction_request(); + + let settings = make_settings(); + let request = Request::get("https://pub.example/auction"); + let context = shared_test_auction_context(&settings, &request, i32::MAX as u32 + 1); let openrtb = provider.to_openrtb( &auction_request, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1c4c7e02d..eb2c44174 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -26,7 +26,7 @@ use crate::auction::types::{ }; use crate::backend::BackendConfig; use crate::compat; -use crate::consent::gate_eids_by_consent; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::ec::kv::KvIdentityGraph; @@ -577,6 +577,9 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { /// Returns true only when the publisher request should run the full /// server-side ad stack: auction dispatch plus initial ad-slot injection. +/// +/// `auction_enabled` is the global `[auction].enabled` kill switch — when +/// false, no automatic server-side auction or ad-slot injection runs. pub(crate) fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, @@ -584,6 +587,7 @@ pub(crate) fn should_run_server_side_ad_stack( is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, + auction_enabled: bool, ) -> bool { is_get && is_navigation @@ -591,6 +595,7 @@ pub(crate) fn should_run_server_side_ad_stack( && !is_bot && has_matched_slots && consent_allows_auction + && auction_enabled } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. @@ -1067,13 +1072,10 @@ pub async fn handle_publisher_request( Vec::new() }; - // Non-GDPR regions (US, etc.) have no TCF string — auction is freely allowed. - // GDPR regions require TCF Purpose 1 (storage/access) before firing. - let consent_allows_auction = !consent_context.gdpr_applies - || consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Fail closed for GDPR-relevant traffic: GDPR/unknown jurisdictions and + // requests carrying an EU TCF signal require effective TCF Purpose 1 + // (storage/access) before firing. Known non-GDPR jurisdictions are free. + let consent_allows_auction = consent_allows_server_side_auction(&consent_context); let should_run_ad_stack = should_run_server_side_ad_stack( is_get, @@ -1082,6 +1084,7 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, + auction.orchestrator.is_enabled(), ); let should_run_auction = should_run_ad_stack; @@ -1567,11 +1570,10 @@ fn is_supported_content_encoding(encoding: &str) -> bool { /// Returns [`TrustedServerError`] if cookie parsing or EC ID generation fails. pub async fn handle_page_bids( settings: &Settings, - orchestrator: &AuctionOrchestrator, services: &RuntimeServices, kv: Option<&KvIdentityGraph>, - registry: Option<&PartnerRegistry>, - slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + auction: AuctionDispatch<'_>, + ec_context: &EcContext, req: Request, ) -> Result> { let Some(co_config) = &settings.creative_opportunities else { @@ -1586,28 +1588,23 @@ pub async fn handle_page_bids( .map(|(_, v)| v.into_owned()) .unwrap_or_else(|| "/".to_string()); - let matched_slots: Vec<_> = crate::creative_opportunities::match_slots(slots, &path_param) - .into_iter() - .cloned() - .collect(); + let matched_slots: Vec<_> = + crate::creative_opportunities::match_slots(auction.slots, &path_param) + .into_iter() + .cloned() + .collect(); let http_req = compat::from_fastly_headers_ref(&req); let request_info = crate::http_util::RequestInfo::from_request(&http_req, &services.client_info); - let ec_ctx = - EcContext::read_from_request(settings, &req).change_context(TrustedServerError::Proxy { - message: "page-bids: failed to read EC context".to_string(), - })?; - let ec_id = ec_ctx.ec_value().filter(|_| ec_ctx.ec_allowed()); - let consent_context = ec_ctx.consent().clone(); - let geo = ec_ctx.geo_info().cloned(); + let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); + let consent_context = ec_context.consent(); + let geo = ec_context.geo_info().cloned(); let cookie_jar = handle_request_cookies(&http_req)?; - let consent_allows_auction = !consent_context.gdpr_applies - || consent_context - .tcf - .as_ref() - .is_some_and(|tcf| tcf.has_purpose_consent(1)); + // Same fail-closed jurisdiction-aware gate the publisher navigation path + // uses — relies on the adapter's geo-aware EC context. + let consent_allows_auction = consent_allows_server_side_auction(consent_context); // Same bot / prefetch guards the publisher path uses — without them this // endpoint would fire real SSP auctions on Sec-Purpose=prefetch warm-up @@ -1615,7 +1612,10 @@ pub async fn handle_page_bids( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - if matched_slots.is_empty() { + let auction_enabled = auction.orchestrator.is_enabled(); + if !auction_enabled { + log::debug!("page-bids: [auction].enabled is false — skipping auction"); + } else if matched_slots.is_empty() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction", path_param @@ -1629,69 +1629,74 @@ pub async fn handle_page_bids( ); } - let winning_bids = - if !matched_slots.is_empty() && consent_allows_auction && !is_bot && !is_prefetch { - let slots_ctx = MatchedSlotsContext { - matched_slots: &matched_slots, - request_path: &path_param, - }; - let mut auction_request = build_auction_request( - &slots_ctx, - ec_id, - &consent_context, - &request_info, - req.get_header_str("user-agent"), - ); - let ts_eids_value = cookie_jar - .as_ref() - .and_then(|j| j.get(COOKIE_TS_EIDS)) - .map(|c| c.value().to_owned()); - let client_eids = if ec_id.is_some() { - resolve_client_auction_eids(None, ts_eids_value.as_deref()) - } else { - None - }; - let kv_eids = resolve_auction_eids(kv, registry, &ec_ctx); - let merged_eids = merge_auction_eids(client_eids, kv_eids); - let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); - if had_eids && auction_request.user.eids.is_none() { - log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); - } - let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } - let timeout_ms = co_config - .auction_timeout_ms - .unwrap_or(settings.auction.timeout_ms); - let auction_context = AuctionContext { - settings, - request: &req, - timeout_ms, - provider_responses: None, - services, - }; - match orchestrator - .run_auction(&auction_request, &auction_context) - .await - { - Ok(result) => result.winning_bids, - Err(e) => { - log::warn!("page-bids auction failed: {e:?}"); - std::collections::HashMap::new() - } - } + let winning_bids = if auction_enabled + && !matched_slots.is_empty() + && consent_allows_auction + && !is_bot + && !is_prefetch + { + let slots_ctx = MatchedSlotsContext { + matched_slots: &matched_slots, + request_path: &path_param, + }; + let mut auction_request = build_auction_request( + &slots_ctx, + ec_id, + consent_context, + &request_info, + req.get_header_str("user-agent"), + ); + let ts_eids_value = cookie_jar + .as_ref() + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) } else { - std::collections::HashMap::new() + None + }; + let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); + } + let client_ip = services.client_info.client_ip.map(|ip| ip.to_string()); + if client_ip.is_some() || geo.is_some() { + let device = auction_request.device.get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }); + device.ip = client_ip; + device.geo = geo.clone(); + } + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let auction_context = AuctionContext { + settings, + request: &req, + timeout_ms, + provider_responses: None, + services, }; + match auction + .orchestrator + .run_auction(&auction_request, &auction_context) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; let bid_map = build_bid_map( &winning_bids, @@ -1938,34 +1943,38 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true), + should_run_server_side_ad_stack(true, true, false, false, true, true, true), "GET, real navigation, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, true), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, true), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, true), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, true), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, true), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false), + !should_run_server_side_ad_stack(true, true, false, false, true, false, true), "requests without required consent should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + "disabled [auction].enabled kill switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -3501,12 +3510,46 @@ mod tests { fn settings_with_co() -> Settings { let toml = format!( - "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") + } + + fn settings_with_co_auction_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = false\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + async fn run_page_bids( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> serde_json::Value { + let services = noop_services(); + let ec_context = + EcContext::read_from_request(settings, &req).expect("should read EC context"); + let response = handle_page_bids( + settings, + &services, + None, + AuctionDispatch { + orchestrator, + slots, + registry: None, + }, + &ec_context, + req, + ) + .await + .expect("should return ok response"); + serde_json::from_slice(&response.into_body_bytes()).expect("should be json") + } + fn article_slot() -> Vec { vec![CreativeOpportunitySlot { id: "atf".to_string(), @@ -3538,16 +3581,9 @@ mod tests { // all server-side auction activity and injection. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let req = make_page_bids_request("/2024/01/my-article/"); - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &[], req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &[], req).await; assert_eq!( body["slots"] @@ -3574,18 +3610,11 @@ mod tests { // for them. Same gate the publisher path applies. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1)"); - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -3611,18 +3640,11 @@ mod tests { // SSP auctions — the user has not yet visited the page. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); req.set_header("sec-purpose", "prefetch"); - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -3647,17 +3669,10 @@ mod tests { // Slots exist but request path does not match — no auction, no injection. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let services = noop_services(); let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match - let response = - handle_page_bids(&settings, &orchestrator, &services, None, None, &slots, req) - .await - .expect("should return ok response"); - - let body: serde_json::Value = - serde_json::from_slice(&response.into_body_bytes()).expect("should be json"); + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -3676,5 +3691,35 @@ mod tests { "non-matching URL should produce zero bids" ); } + + #[tokio::test] + async fn disabled_auction_returns_slots_but_no_bids() { + // [auction].enabled = false is a global kill switch: slot definitions + // are still returned (HTML structure unchanged) but no server-side + // auction may be dispatched. + let settings = settings_with_co_auction_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "disabled auction should still return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled auction must not produce bids" + ); + } } } From 0f4dd86fefd0bcc164776a849963c0eb9516c0a3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 12 Jun 2026 13:55:28 +0530 Subject: [PATCH 086/195] Resolve beacon, validation, and orchestrator review findings - Dedupe win/billing beacons: fire each bid's nurl/burl at most once, keyed by slot + bid identity in shared tsjs state so the inline bootstrap and bundle listeners can never double-fire; unify the ourBidWon check (hb_adid confirmation with hb_bidder fallback for APS bids) across both implementations - Wire validate_slot_id into Settings::prepare_runtime so every load path (including env-injected slots on runtime-config adapters) rejects invalid slot IDs; build.rs settings stub gains a no-op - Normalize the client-controlled page-bids path parameter: strip query/fragment and force a leading slash before glob matching - Document the deliberate Cache-Control private, max-age=0 choice (BFCache eligibility per design spec section 4.7, not no-store) - Align orchestrator collect path with the parallel path: use parse_response_with_context for providers and the mediator, and add a defense-in-depth deadline check to the collect select-loop - Migrate adserver_mock off request-scoped Mutex state: the SSP bid index is rebuilt in parse_response_with_context from the context's provider responses; document why APS's slot_id_map cannot follow yet - Make platform_response_to_fastly infallible; drop the dead error arms - Remove redundant PriceGranularity::dense and MediaType::banner constructors in favor of Default-based serde field defaults - Clarify that the Prebid stored-request fallback cannot fire for the client /auction path (every ad unit carries a trustedServer entry) - Consolidate GPT JS suites under test/integrations/gpt/, replace the leaked module-scope addEventListener patch with a restored wrapper, and add installSpaAuctionHook coverage (pushState/replaceState/ popstate, stale-response guard, non-OK response, idempotence) --- crates/js/lib/src/core/types.ts | 6 + crates/js/lib/src/integrations/gpt/index.ts | 24 ++- .../integrations/gpt/ad_init.test.ts} | 95 ++++++---- .../test/integrations/gpt/spa_hook.test.ts | 167 ++++++++++++++++++ crates/trusted-server-core/build.rs | 8 + .../src/auction/orchestrator.rs | 157 ++++++++-------- .../trusted-server-core/src/auction/types.rs | 19 +- .../src/creative_opportunities.rs | 8 +- .../src/integrations/adserver_mock.rs | 147 ++++++++------- .../src/integrations/aps.rs | 5 + .../src/integrations/gpt_bootstrap.js | 15 +- .../src/integrations/prebid.rs | 7 + .../trusted-server-core/src/price_bucket.rs | 7 - crates/trusted-server-core/src/publisher.rs | 50 +++++- crates/trusted-server-core/src/settings.rs | 45 ++++- 15 files changed, 545 insertions(+), 215 deletions(-) rename crates/js/lib/{src/integrations/gpt/index.test.ts => test/integrations/gpt/ad_init.test.ts} (89%) create mode 100644 crates/js/lib/test/integrations/gpt/spa_hook.test.ts diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 4fb99f3b4..1bdf1057b 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -98,6 +98,12 @@ export interface TsjsApi { servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; + /** + * Win/billing beacons already fired, keyed by `slotId|bidIdentity`. + * Shared between the inline GPT bootstrap and the bundle listener so a + * bid's nurl/burl fire at most once even across GAM re-renders. + */ + firedBeacons?: Record; /** Slot-level GPT targeting keys TS applied on the previous route. */ prevSlotTargetingKeys?: Record; /** diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 9054d4c15..6d058a0c8 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -453,13 +453,27 @@ export function installTsAdInit(): void { // Read ts.bids live (not the snapshot above) so post-navigation bid data is used. const bid = (ts.bids ?? {})[slotId] ?? {}; // Compare hb_adid targeting to verify the specific creative won. + // APS bids carry no hb_adid — fall back to hb_bidder presence + // (same heuristic as the inline bootstrap) so APS wins still bill. const ourBidWon = !event.isEmpty && - !!bid.hb_adid && - event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid; - if (ourBidWon) { - if (bid.nurl) navigator.sendBeacon(bid.nurl); - if (bid.burl) navigator.sendBeacon(bid.burl); + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder); + if (ourBidWon && (bid.nurl || bid.burl)) { + // Fire win/billing beacons at most once per bid: GAM re-renders + // (publisher refreshes, repeated slotRenderEnded for the same + // line item) must not re-bill. New auctions produce new bid + // identities, so post-navigation bids still fire. Keyed in + // shared tsjs state so the inline-bootstrap listener and this + // one can never double-fire the same bid. + const beaconKey = `${slotId}|${bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''}`; + const fired = (ts.firedBeacons ??= {}); + if (!fired[beaconKey]) { + fired[beaconKey] = true; + if (bid.nurl) navigator.sendBeacon(bid.nurl); + if (bid.burl) navigator.sendBeacon(bid.burl); + } } // GAM interceptor (testing): when adm is present, replace the GAM creative. diff --git a/crates/js/lib/src/integrations/gpt/index.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts similarity index 89% rename from crates/js/lib/src/integrations/gpt/index.test.ts rename to crates/js/lib/test/integrations/gpt/ad_init.test.ts index aaf2657b4..147ecebf1 100644 --- a/crates/js/lib/src/integrations/gpt/index.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -1,23 +1,36 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; // Track every 'message' EventListener added to window across the entire test // file. This lets the installTsRenderBridge suite remove all accumulated -// handlers (registered by each vi.resetModules() + import('./index') in the -// installTsAdInit suite) before dispatching its own events. +// handlers (registered by each vi.resetModules() + module re-import in the +// installTsAdInit suite) before dispatching its own events. The spy is +// restored and remaining handlers are detached in the afterAll below so the +// patch never leaks past this file. const allMessageHandlers: EventListener[] = []; -const _origWindowAddEventListener = window.addEventListener.bind(window); -// eslint-disable-next-line @typescript-eslint/no-explicit-any -(window as any).addEventListener = function ( +const originalWindowAddEventListener = window.addEventListener.bind(window); +// Plain wrapper, deliberately not vi.spyOn: the render-bridge suite spies on +// window.addEventListener itself, and vi.spyOn on an already-spied method +// returns the same mock instance — its "original" would alias the inner +// implementation and recurse. +(window as { addEventListener: typeof window.addEventListener }).addEventListener = (( type: string, handler: EventListenerOrEventListenerObject, - options?: unknown -) { - if (type === 'message') { + options?: boolean | AddEventListenerOptions +) => { + if (type === 'message' && handler) { allMessageHandlers.push(handler as EventListener); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return _origWindowAddEventListener(type, handler as EventListener, options as any); -}; + return originalWindowAddEventListener(type, handler, options); +}) as typeof window.addEventListener; + +afterAll(() => { + for (const handler of allMessageHandlers) { + window.removeEventListener('message', handler); + } + allMessageHandlers.length = 0; + (window as { addEventListener: typeof window.addEventListener }).addEventListener = + originalWindowAddEventListener; +}); interface SlotRenderEvent { isEmpty: boolean; @@ -109,7 +122,7 @@ describe('installTsAdInit', () => { const fetchSpy = vi.spyOn(global, 'fetch'); - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -161,7 +174,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -201,7 +214,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -259,7 +272,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -317,7 +330,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -326,10 +339,17 @@ describe('installTsAdInit', () => { expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win'); expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + + // GAM re-rendering the same line item (same hb_adid) must not re-fire + // the same bid's win/billing beacons. + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); }); - it('does not fire beacons when a rendered bid has no hb_adid confirmation', async () => { + it('fires APS-style beacons once via hb_bidder fallback and dedupes repeat renders', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); let capturedListener: ((e: SlotRenderEvent) => void) | undefined; @@ -373,18 +393,27 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); expect(capturedListener).toBeDefined(); - capturedListener!({ isEmpty: false, slot: mockSlot }); - - expect(beaconSpy).not.toHaveBeenCalled(); + // Empty render never fires. capturedListener!({ isEmpty: true, slot: mockSlot }); expect(beaconSpy).not.toHaveBeenCalled(); + // First real render fires both beacons via the hb_bidder fallback + // (APS bids carry no hb_adid to confirm against). + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(beaconSpy).toHaveBeenCalledWith('https://aps/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://aps/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + + // Re-render of the same bid (publisher refresh) must not re-bill. + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); }); @@ -433,7 +462,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }); @@ -485,7 +514,7 @@ describe('installTsAdInit', () => { }, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -533,7 +562,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -580,7 +609,7 @@ describe('installTsAdInit', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -624,7 +653,7 @@ describe('installTsAdInit', () => { bids: {}, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); @@ -667,7 +696,7 @@ describe('installTsAdInit', () => { bids: {}, }; - const { installTsAdInit } = await import('./index'); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); expect(() => (window as TestWindow).tsjs!.adInit!()).not.toThrow(); @@ -746,7 +775,7 @@ describe('installTsRenderBridge', () => { origAdd(type, handler as EventListener, opts as any); } ); - await import('./index'); + await import('../../../src/integrations/gpt/index'); addSpy.mockRestore(); // Restore only addEventListener — fetchStub must stay stubbed expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); @@ -816,7 +845,7 @@ describe('installTsRenderBridge', () => { origAdd(type, handler as EventListener, opts as any); } ); - await import('./index'); + await import('../../../src/integrations/gpt/index'); addSpy.mockRestore(); expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); @@ -850,7 +879,7 @@ describe('installTsRenderBridge', () => { }); it('ignores message when adId does not match any TS bid', async () => { - await import('./index'); + await import('../../../src/integrations/gpt/index'); fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); window.dispatchEvent( @@ -865,7 +894,7 @@ describe('installTsRenderBridge', () => { }); it('ignores matching adId messages from outside configured slot iframes', async () => { - await import('./index'); + await import('../../../src/integrations/gpt/index'); fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); const foreignIframe = document.createElement('iframe'); @@ -890,7 +919,7 @@ describe('installTsRenderBridge', () => { }); it('ignores non-Prebid messages', async () => { - await import('./index'); + await import('../../../src/integrations/gpt/index'); window.dispatchEvent( new MessageEvent('message', { data: JSON.stringify({ message: 'Other' }) }) ); diff --git a/crates/js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts new file mode 100644 index 000000000..5a72c56e8 --- /dev/null +++ b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts @@ -0,0 +1,167 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +import type { TsjsApi } from '../../../src/core/types'; + +type TestWindow = Window & { + googletag?: unknown; + tsjs?: TsjsApi; +}; + +const originalPushState = history.pushState.bind(history); +const originalReplaceState = history.replaceState.bind(history); + +async function importGptModule() { + return import('../../../src/integrations/gpt/index'); +} + +/** Flush the microtask/timer queue so onNavigate's awaits settle. */ +async function flushAsync(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe('installSpaAuctionHook', () => { + let fetchStub: ReturnType; + + beforeEach(() => { + vi.resetModules(); + delete (window as TestWindow).tsjs; + // Restore unwrapped history methods so each module import wraps exactly + // once — without this, wrappers from prior imports accumulate. + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + fetchStub = vi.fn(); + vi.stubGlobal('fetch', fetchStub); + }); + + afterEach(() => { + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + // Reset jsdom location back to root for the next test. + originalReplaceState({}, '', '/'); + vi.unstubAllGlobals(); + }); + + it('fetches page-bids on pushState and applies slots/bids via adInit', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [{ id: 's1' }], bids: { s1: { hb_pb: '1.00' } } }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/next-page'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Fnext-page', + expect.objectContaining({ credentials: 'include' }) + ); + expect(ts.adSlots).toEqual([{ id: 's1' }]); + expect(ts.bids).toEqual({ s1: { hb_pb: '1.00' } }); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('does not fetch when pushState targets the current path', async () => { + await importGptModule(); + + history.pushState({}, '', '/'); + await flushAsync(); + + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('fetches on replaceState and popstate navigation', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + history.replaceState({}, '', '/replaced'); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Freplaced', + expect.objectContaining({ credentials: 'include' }) + ); + + window.dispatchEvent(new PopStateEvent('popstate')); + await flushAsync(); + expect(fetchStub).toHaveBeenLastCalledWith( + '/__ts/page-bids?path=%2Freplaced', + expect.objectContaining({ credentials: 'include' }) + ); + }); + + it('drops a stale response that resolves after a newer navigation started', async () => { + let resolveFirst: ((value: unknown) => void) | undefined; + fetchStub + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }) + ) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ slots: [{ id: 'newer' }], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/first'); + history.pushState({}, '', '/second'); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'newer' }]); + expect(adInit).toHaveBeenCalledTimes(1); + + // First navigation's response arrives late — it must not overwrite the + // newer route's slots or trigger another adInit. + resolveFirst!({ + ok: true, + json: async () => ({ slots: [{ id: 'stale' }], bids: {} }), + }); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'newer' }]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('leaves slots and bids untouched on a non-OK response', async () => { + fetchStub.mockResolvedValue({ ok: false, status: 500 }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.adSlots = [{ id: 'existing' } as never]; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/error-page'); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'existing' }]); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('is idempotent — repeated install calls do not double-fetch a navigation', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + // Module init already installed the hook; both calls must be no-ops. + installSpaAuctionHook(); + installSpaAuctionHook(); + + history.pushState({}, '', '/once'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledTimes(1); + }); +}); diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index a0cc07b30..fc5422af2 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -68,6 +68,14 @@ mod creative_opportunities { pub fn compile_slots(&mut self) {} } + /// Stub — the typed `slot` vec is always empty in the build context (see + /// `#[serde(skip)]` above), so `Settings::prepare_runtime` never reaches + /// this. Build-time slot-id validation happens in `main()` against + /// `slot_raw` instead. + pub fn validate_slot_id(_id: &str) -> Result<(), String> { + Ok(()) + } + fn default_price_granularity() -> String { "dense".to_string() } diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 8ccdca305..b77b110a9 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -813,37 +813,25 @@ impl AuctionOrchestrator { backend_to_provider.remove(&backend_name) { let response_time_ms = start_time.elapsed().as_millis() as u64; - match platform_response_to_fastly(platform_response) { - Ok(response) => { - match provider.parse_response(response, response_time_ms) { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids ({}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - log::warn!( - "Provider '{}' parse failed: {:?}", - provider_name, - e - ); - responses.push(AuctionResponse::error( - &provider_name, - response_time_ms, - )); - } - } + let response = platform_response_to_fastly(platform_response); + // Mirror run_providers_parallel: use the context-aware + // parse so providers behave identically on both paths. + match provider.parse_response_with_context( + response, + response_time_ms, + context, + ) { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); } Err(e) => { - log::warn!( - "Provider '{}' unsupported body: {:?}", - provider_name, - e - ); + log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); responses .push(AuctionResponse::error(&provider_name, response_time_ms)); } @@ -859,6 +847,19 @@ impl AuctionOrchestrator { log::warn!("A provider request failed during collection: {:?}", e); } } + + // Defense-in-depth deadline guard, mirroring run_providers_parallel. + // Dispatch already caps each backend's first_byte_timeout at the + // remaining auction budget, so this should not fire in practice — + // it protects against the two paths drifting apart. + if remaining_budget_ms(auction_start, timeout_ms) == 0 && !remaining.is_empty() { + log::warn!( + "Auction timeout ({}ms) reached during collection, dropping {} remaining request(s)", + timeout_ms, + remaining.len() + ); + break; + } } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { @@ -925,61 +926,47 @@ impl AuctionOrchestrator { ), }) { Ok(platform_resp) => { - match platform_response_to_fastly(platform_resp).change_context( - TrustedServerError::Auction { - message: format!( - "Mediator {} unsupported body", - mediator.provider_name() - ), - }, + let response = platform_response_to_fastly(platform_resp); + let response_time_ms = + mediator_start.elapsed().as_millis() as u64; + // Mirror run_parallel_mediation: use the + // context-aware parse so the mediator sees + // the collected provider responses. + match mediator.parse_response_with_context( + response, + response_time_ms, + &mediator_context, ) { - Ok(response) => { - let response_time_ms = - mediator_start.elapsed().as_millis() as u64; - match mediator - .parse_response(response, response_time_ms) - { - Ok(mediator_resp) => { - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - let winning = self - .apply_floor_prices(winning, &floor_prices); - (Some(mediator_resp), winning) - } - Err(e) => { - log::warn!( - "Mediator '{}' parse failed: {:?}", - mediator.provider_name(), - e - ); - let winning = self.select_winning_bids( - &responses, - &floor_prices, - ); - (None, winning) - } - } + Ok(mediator_resp) => { + let winning = mediator_resp + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + mediator.provider_name(), + bid.slot_id + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + let winning = + self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_resp), winning) } Err(e) => { - log::warn!("Mediator body error: {:?}", e); - ( - None, - self.select_winning_bids(&responses, &floor_prices), - ) + log::warn!( + "Mediator '{}' parse failed: {:?}", + mediator.provider_name(), + e + ); + let winning = + self.select_winning_bids(&responses, &floor_prices); + (None, winning) } } } @@ -1063,12 +1050,8 @@ impl OrchestrationResult { } } -fn platform_response_to_fastly( - platform_response: PlatformResponse, -) -> Result> { - Ok(crate::compat::to_fastly_response( - platform_response.response, - )) +fn platform_response_to_fastly(platform_response: PlatformResponse) -> fastly::Response { + crate::compat::to_fastly_response(platform_response.response) } #[cfg(test)] diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 418da884d..2560ed92a 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -52,22 +52,15 @@ pub struct AdFormat { } /// Media type enumeration. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum MediaType { + #[default] Banner, Video, Native, } -impl MediaType { - /// Returns the Banner media type. - #[must_use] - pub fn banner() -> Self { - Self::Banner - } -} - /// Publisher information. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PublisherInfo { @@ -492,8 +485,12 @@ mod tests { } #[test] - fn media_type_banner_fn_returns_banner() { - assert_eq!(MediaType::banner(), MediaType::Banner); + fn media_type_defaults_to_banner() { + assert_eq!( + MediaType::default(), + MediaType::Banner, + "should default to Banner for serde field defaults" + ); } #[test] diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index a7b3d579a..645ba423f 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -36,8 +36,8 @@ pub struct CreativeOpportunitiesConfig { /// When absent, falls back to `[auction].timeout_ms` from global config. #[serde(default)] pub auction_timeout_ms: Option, - /// Price granularity for header-bidding price bucketing. - #[serde(default = "PriceGranularity::dense")] + /// Price granularity for header-bidding price bucketing. Defaults to `Dense`. + #[serde(default)] pub price_granularity: PriceGranularity, /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). #[serde(default, deserialize_with = "vec_from_seq_or_map")] @@ -230,8 +230,8 @@ pub struct CreativeOpportunityFormat { pub width: u32, /// Creative height in pixels. pub height: u32, - /// Media type for this format. - #[serde(default = "MediaType::banner")] + /// Media type for this format. Defaults to `Banner`. + #[serde(default)] pub media_type: MediaType, } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 8dc18e286..f1fd5ab89 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -10,7 +10,7 @@ use fastly::Request; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as Json}; use std::collections::{BTreeMap, HashMap}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; use validator::Validate; @@ -88,28 +88,42 @@ impl IntegrationConfig for AdServerMockConfig { // Provider // ============================================================================ -/// Lookup index built from original SSP bids during `request_bids`, consumed -/// during `parse_response` to restore render/accounting fields that the mock +/// Lookup index built from the original SSP bids, used while parsing the +/// mediation response to restore render/accounting fields that the mock /// mediator endpoint does not echo back. /// /// Keyed by `(provider_name, slot_id, bidder_name)`. type BidIndex = HashMap<(String, String, String), Bid>; +/// Builds the SSP-bid lookup index from the orchestrator-provided +/// bidder responses on the auction context. +fn build_bid_index(bidder_responses: &[AuctionResponse]) -> BidIndex { + let mut index = BidIndex::new(); + for response in bidder_responses { + for bid in &response.bids { + index.insert( + ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), + ), + bid.clone(), + ); + } + } + index +} + /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, - /// Bridges SSP bid metadata from `request_bids` to `parse_response`. - bid_index: Mutex>, } impl AdServerMockProvider { /// Create a new mock ad server provider. #[must_use] pub fn new(config: AdServerMockConfig) -> Self { - Self { - config, - bid_index: Mutex::new(None), - } + Self { config } } /// Build the mediation endpoint URL, appending context values as query @@ -225,9 +239,10 @@ impl AdServerMockProvider { /// Parse `OpenRTB` response from mediation endpoint. /// Mediation returns decoded prices for all bids (including APS bids that were encoded). /// - /// `bid_index` is the SSP-bid lookup built in `request_bids`. The mock mediator - /// does not echo render/accounting fields back, so they are restored from the index - /// using `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` + /// `bid_index` is the SSP-bid lookup built from the auction context's + /// bidder responses. The mock mediator does not echo render/accounting + /// fields back, so they are restored from the index using + /// `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` /// field (`"{bidder}-creative"` format set during request construction). fn parse_mediation_response( &self, @@ -301,6 +316,45 @@ impl AdServerMockProvider { AuctionResponse::success("adserver_mock", all_bids, response_time_ms) } } + + /// Shared parse body for the context-aware and context-less trait methods. + /// + /// # Errors + /// + /// Returns an error when the mediation response body is not valid JSON. + fn parse_response_inner( + &self, + mut response: fastly::Response, + response_time_ms: u64, + bid_index: &BidIndex, + ) -> Result> { + if !response.get_status().is_success() { + log::warn!( + "AdServer Mock returned non-success: {}", + response.get_status() + ); + return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); + } + + let body_bytes = response.take_body_bytes(); + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { + message: "Failed to parse mediation response".to_string(), + })?; + + log::trace!("AdServer Mock response: {:?}", response_json); + + let auction_response = + self.parse_mediation_response(&response_json, response_time_ms, bid_index); + + log::info!( + "AdServer Mock returned {} bids in {}ms", + auction_response.bids.len(), + response_time_ms + ); + + Ok(auction_response) + } } impl AuctionProvider for AdServerMockProvider { @@ -322,23 +376,6 @@ impl AuctionProvider for AdServerMockProvider { bidder_responses.len() ); - // Build bid index so parse_response can restore nurl/burl/ad_id from - // the original SSP bids (the mock mediator does not echo these fields). - let mut index = BidIndex::new(); - for response in bidder_responses { - for bid in &response.bids { - index.insert( - ( - response.provider.clone(), - bid.slot_id.clone(), - bid.bidder.clone(), - ), - bid.clone(), - ); - } - } - *self.bid_index.lock().expect("should lock bid index") = Some(index); - // Build mediation request let mediation_req = self .build_mediation_request(request, bidder_responses) @@ -395,42 +432,28 @@ impl AuctionProvider for AdServerMockProvider { fn parse_response( &self, - mut response: fastly::Response, + response: fastly::Response, response_time_ms: u64, ) -> Result> { - if !response.get_status().is_success() { - log::warn!( - "AdServer Mock returned non-success: {}", - response.get_status() - ); - return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); - } - - let body_bytes = response.take_body_bytes(); - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { - message: "Failed to parse mediation response".to_string(), - })?; - - log::trace!("AdServer Mock response: {:?}", response_json); - - let bid_index = self - .bid_index - .lock() - .expect("should lock bid index") - .take() - .unwrap_or_default(); - - let auction_response = - self.parse_mediation_response(&response_json, response_time_ms, &bid_index); - - log::info!( - "AdServer Mock returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); + // No auction context available — nurl/burl/ad_id restoration from the + // original SSP bids is skipped. The orchestrator always calls + // [`parse_response_with_context`], so this path only serves callers + // outside the orchestration flow. + log::debug!("adserver_mock: parsing without context — SSP bid metadata unavailable"); + self.parse_response_inner(response, response_time_ms, &BidIndex::new()) + } - Ok(auction_response) + fn parse_response_with_context( + &self, + response: fastly::Response, + response_time_ms: u64, + context: &AuctionContext<'_>, + ) -> Result> { + // Rebuild the SSP-bid lookup from the orchestrator-provided bidder + // responses so nurl/burl/ad_id survive mediation. Request-scoped data + // travels on the context instead of provider-instance state. + let bid_index = build_bid_index(context.provider_responses.unwrap_or(&[])); + self.parse_response_inner(response, response_time_ms, &bid_index) } fn supports_media_type(&self, media_type: &MediaType) -> bool { diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index b415e5c88..ed8c73354 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -290,6 +290,11 @@ pub struct ApsAuctionProvider { // Written by request_bids before the async send; read by parse_response when the // response arrives. Safe because Fastly Compute runs each request in an isolated // single-threaded Wasm instance — the Mutex never contends in practice. + // + // Unlike adserver_mock's bid index (rebuilt in parse_response_with_context + // from context.provider_responses), this map derives from the AuctionRequest, + // which AuctionContext does not carry — migrating it off provider-instance + // state needs the request threaded through the context first. slot_id_map: std::sync::Mutex>, } diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index cd4b05d42..341b376a6 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -103,9 +103,18 @@ (b.hb_adid ? ev.slot.getTargeting("hb_adid")[0] === b.hb_adid : !!b.hb_bidder); - if (ourBidWon) { - if (b.nurl) navigator.sendBeacon(b.nurl); - if (b.burl) navigator.sendBeacon(b.burl); + if (ourBidWon && (b.nurl || b.burl)) { + // Fire each bid's win/billing beacons at most once — GAM can + // re-render the same line item on publisher refreshes. Keep the + // key format in sync with the bundle listener in index.ts; the + // map lives on tsjs so both listeners share dedupe state. + var beaconKey = slotId + "|" + (b.hb_adid || b.nurl || b.burl || ""); + var fired = (ts.firedBeacons = ts.firedBeacons || {}); + if (!fired[beaconKey]) { + fired[beaconKey] = true; + if (b.nurl) navigator.sendBeacon(b.nurl); + if (b.burl) navigator.sendBeacon(b.burl); + } } }); } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d8e411aed..658307a92 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -984,6 +984,13 @@ impl PrebidAuctionProvider { // When no inline PBS bidder params exist (e.g. creative-opportunity slots // whose PBS params live in stored requests), tell PBS to resolve bidder // config from the stored request keyed by this slot ID. + // + // This cannot fire for the client /auction path: the JS adapter + // injects a `trustedServer` entry into every ad unit, so `bidder` + // is only empty for server-side creative-opportunity slots with + // no inline provider params (or when `config.bidders` is empty, + // where PBS previously received an empty bidder map and returned + // no bids — a stored-request miss is the same no-bid outcome). let storedrequest = if bidder.is_empty() { Some(ImpStoredRequest { id: slot.id.clone(), diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs index cfdca9eb4..8fc4e50e6 100644 --- a/crates/trusted-server-core/src/price_bucket.rs +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -11,13 +11,6 @@ pub enum PriceGranularity { Auto, } -impl PriceGranularity { - #[must_use] - pub fn dense() -> Self { - Self::Dense - } -} - #[must_use] pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { // Reject NaN / Inf early so the `(x * 100.0).floor() as u64` cast below diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index eb2c44174..027903893 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1208,7 +1208,10 @@ pub async fn handle_publisher_request( }; // §4.7: assembled HTML responses must never be shared-cached — per-user bid data - // travels inline. Apply regardless of slot match or auction outcome (§8). + // travels inline. `private, max-age=0` is deliberate (not `no-store`): it keeps + // the page BFCache-eligible while restricting reuse to the same user's browser + // with revalidation; `Surrogate-Control` removal handles the Fastly shared + // cache. Apply regardless of slot match or auction outcome (§8). let origin_content_type = response .get_header(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) @@ -1559,6 +1562,20 @@ fn is_supported_content_encoding(encoding: &str) -> bool { matches!(encoding, "" | "identity" | "gzip" | "deflate" | "br") } +/// Normalizes the client-supplied `path` query parameter before glob matching. +/// +/// The SPA hook sends `location.pathname`, but the parameter is +/// client-controlled: strip any query string or fragment and force a leading +/// `/` so slot `page_patterns` always match against a canonical path shape. +fn normalize_page_bids_path(raw: &str) -> String { + let path = raw.split(['?', '#']).next().unwrap_or(""); + if path.starts_with('/') { + path.to_string() + } else { + format!("/{path}") + } +} + /// Handle `GET /__ts/page-bids?path=` — server-side auction for SPA navigation. /// /// Matches creative opportunity slots for the given path, runs a server-side @@ -1585,7 +1602,7 @@ pub async fn handle_page_bids( .get_url() .query_pairs() .find(|(k, _)| k == "path") - .map(|(_, v)| v.into_owned()) + .map(|(_, v)| normalize_page_bids_path(&v)) .unwrap_or_else(|| "/".to_string()); let matched_slots: Vec<_> = @@ -3692,6 +3709,35 @@ mod tests { ); } + #[test] + fn normalize_page_bids_path_strips_query_fragment_and_forces_leading_slash() { + assert_eq!( + normalize_page_bids_path("/2024/01/article/"), + "/2024/01/article/", + "canonical path should pass through unchanged" + ); + assert_eq!( + normalize_page_bids_path("/2024/01/article/?utm_source=x"), + "/2024/01/article/", + "query string should be stripped before glob matching" + ); + assert_eq!( + normalize_page_bids_path("/2024/01/article/#section"), + "/2024/01/article/", + "fragment should be stripped before glob matching" + ); + assert_eq!( + normalize_page_bids_path("2024/01/article/"), + "/2024/01/article/", + "missing leading slash should be added" + ); + assert_eq!( + normalize_page_bids_path(""), + "/", + "empty path should normalize to root" + ); + } + #[tokio::test] async fn disabled_auction_returns_slots_but_no_bids() { // [auction].enabled = false is a global kill switch: slot definitions diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 642ee4366..dc59bdfa9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -776,7 +776,8 @@ impl Settings { /// /// # Errors /// - /// Returns a configuration error if any cached runtime artifact cannot be prepared. + /// Returns a configuration error if any cached runtime artifact cannot be + /// prepared, or if a creative opportunity slot has an invalid ID. pub fn prepare_runtime(&mut self) -> Result<(), Report> { for handler in &self.handlers { handler.prepare_runtime()?; @@ -784,6 +785,16 @@ impl Settings { if let Some(co) = &mut self.creative_opportunities { co.compile_slots(); + // Slot IDs flow into injected HTML/JS and provider payloads, and + // can arrive via TRUSTED_SERVER__ env overrides that bypass any + // static config review — validate them on every load path. + for slot in &co.slot { + crate::creative_opportunities::validate_slot_id(&slot.id).map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity slot id: {err}"), + }) + })?; + } } Ok(()) @@ -2707,6 +2718,38 @@ auction_timeout_ms = 500 assert_eq!(co.auction_timeout_ms, Some(500)); } + #[test] + fn settings_rejects_invalid_creative_opportunity_slot_id() { + let toml = r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" + +[[creative_opportunities.slot]] +id = "xss"#; + let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "Text/HTML; Charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: state, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let mut output = Vec::new(); + + stream_publisher_body( + Body::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should process mixed-case HTML content type"); + + let html = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + html.contains(".adSlots=JSON.parse"), + "mixed-case HTML must use the HTML processor and inject ad slots. Got: {html}" + ); + assert!( + html.contains(".bids=JSON.parse"), + "mixed-case HTML must use the HTML processor and inject bids. Got: {html}" + ); + } + /// Mid-stream decoder failure must surface as an error. The adapter /// relies on this: once headers are committed, it logs and drops the /// `StreamingBody` so the client sees a truncated response. If a decode From 8fb30b3ce7a5c2f4485882397e32f3fb6b0919ad Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sun, 14 Jun 2026 10:43:51 +0530 Subject: [PATCH 096/195] Bind render bridge to source slot and fix refresh parity Resolve three #680 review findings on the server-side ad runtime: - Render bridge now requires the requesting iframe's slot to own the resolved hb_adid before responding or firing win/billing beacons. Previously an iframe under slot A could request slot B's adId and receive slot B's creative while firing slot B's beacons. - Refresh ad units now include configured client-side bidders by merging matching pbjs.adUnits bid entries, so native Prebid demand is not dropped on refresh/scroll impressions. - Inline GPT bootstrap wraps its internal refresh with the adInitRefreshInProgress sentinel, mirroring the TS adInit so a pre-installed slim-Prebid refresh wrapper does not clear TS targeting. Add regression tests for the two-slot render-bridge mismatch and the client-side bidder refresh merge. --- crates/js/lib/src/integrations/gpt/index.ts | 16 +++-- .../js/lib/src/integrations/prebid/index.ts | 38 +++++++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 45 ++++++++++++++ .../test/integrations/prebid/index.test.ts | 58 +++++++++++++++++++ .../src/integrations/gpt_bootstrap.js | 12 +++- 5 files changed, 161 insertions(+), 8 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index eae2fc881..8d138a9bf 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -80,15 +80,15 @@ function candidateSlotRoots(divId: string): HTMLElement[] { return roots; } -function messageSourceBelongsToConfiguredSlot(source: MessageEventSource | null): boolean { - if (!source) return false; +function slotIdForMessageSource(source: MessageEventSource | null): string | undefined { + if (!source) return undefined; const slots = window.tsjs?.adSlots ?? []; - return slots.some((slot) => + return slots.find((slot) => candidateSlotRoots(slot.div_id).some((root) => Array.from(root.querySelectorAll('iframe')).some((iframe) => iframe.contentWindow === source) ) - ); + )?.id; } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -672,7 +672,8 @@ export function installTsRenderBridge(): void { const port = e.ports?.[0]; if (!port) return; - if (!messageSourceBelongsToConfiguredSlot(e.source)) return; + const sourceSlotId = slotIdForMessageSource(e.source); + if (!sourceSlotId) return; // Build reverse map adId → slotId from live window.tsjs.bids. const bids = window.tsjs?.bids ?? {}; @@ -689,6 +690,11 @@ export function installTsRenderBridge(): void { // Not a TS bid — let Prebid.js handle it. if (!slotId || !matchedBid) return; + // The requesting iframe's slot must own the resolved adId. Without this an + // iframe under slot A could request slot B's hb_adid and receive slot B's + // creative/dimensions while firing slot B's win/billing beacons. + if (slotId !== sourceSlotId) return; + const slot = window.tsjs?.adSlots?.find((s) => s.id === slotId); const [width, height] = slot?.formats?.[0] ?? [728, 90]; diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index d42c7d265..61b546e5b 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -342,6 +342,36 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } +/** + * Collect the configured client-side bidder entries for a refreshing slot. + * + * Synthetic refresh ad units carry only the `trustedServer` bid. The + * `requestBids` shim preserves a client-side bidder only when its bid entry is + * already present on the ad unit, so without re-attaching them here publishers + * that split demand between server-side and native Prebid adapters would lose + * all client-side demand on refresh/scroll impressions. Bids are sourced from + * the matching `pbjs.adUnits` entry (by ad unit code) so the publisher's + * configured params are preserved. + */ +function clientSideBidsForRefresh( + code: string +): Array<{ bidder: string; params: Record }> { + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); + if (clientSideBidders.size === 0) return []; + + const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; + const match = adUnits.find((unit) => unit.code === code); + if (!match?.bids) return []; + + const bids: Array<{ bidder: string; params: Record }> = []; + for (const bid of match.bids) { + if (bid?.bidder && clientSideBidders.has(bid.bidder)) { + bids.push({ bidder: bid.bidder, params: bid.params ?? {} }); + } + } + return bids; +} + function clearRefreshTargeting(slot: RefreshGptSlot): void { if (typeof slot.clearTargeting !== 'function') return; @@ -663,10 +693,14 @@ export function installRefreshHandler(timeoutMs = 1500): void { ...(zone ? { name: zone } : {}), }; + const code = refreshSlotElementId(slot) ?? 'refresh-slot'; return { - code: refreshSlotElementId(slot) ?? 'refresh-slot', + code, mediaTypes: { banner }, - bids: [{ bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }], + bids: [ + { bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }, + ...clientSideBidsForRefresh(code), + ], }; }); diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index 3a69c5a42..73d3be9c3 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -932,6 +932,51 @@ describe('installTsRenderBridge', () => { foreignIframe.remove(); }); + it('ignores a request whose source slot does not own the resolved adId', async () => { + // Two configured slots; slot A's iframe requests slot B's hb_adid. The + // bridge must not return slot B's creative or fire slot B's beacons. + (window as TestWindow).tsjs.bids.homepage_footer = { + hb_adid: 'footer-uuid', + hb_bidder: 'kargo', + hb_pb: '2.00', + hb_cache_host: 'openads.example.com', + hb_cache_path: '/cache', + nurl: 'https://ssp.example/footer-win', + burl: 'https://ssp.example/footer-bill', + }; + (window as TestWindow).tsjs.adSlots.push({ + id: 'homepage_footer', + formats: [[300, 250]] as [number, number][], + gam_unit_path: '/a/b/footer', + div_id: 'div-footer', + targeting: {}, + }); + + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + await import('../../../src/integrations/gpt/index'); + fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); + + // Source iframe lives under slot A (div-header). + const source = createTrustedSlotIframe(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + + window.dispatchEvent( + new MessageEvent('message', { + // adId belongs to slot B (homepage_footer), not slot A's iframe. + data: JSON.stringify({ message: 'Prebid Request', adId: 'footer-uuid' }), + ports: [fakePort as MessagePort], + source, + }) + ); + + await new Promise((r) => setTimeout(r, 50)); + expect(fetchStub).not.toHaveBeenCalled(); + expect(portMessages).toHaveLength(0); + expect(beaconSpy).not.toHaveBeenCalled(); + document.getElementById('div-footer')?.remove(); + }); + it('ignores non-Prebid messages', async () => { await import('../../../src/integrations/gpt/index'); window.dispatchEvent( diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 5e28f2a25..2ca650e18 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -864,6 +864,64 @@ describe('prebid/installRefreshHandler', () => { ); }); + it('includes configured client-side bidders in refresh ad units', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + // Original publisher ad unit carries a client-side rubicon bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: {} }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: { zone: 'homepage' } }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + ], + }), + ], + }) + ); + + delete (window as any).__tsjs_prebid; + mockPbjs.adUnits = []; + }); + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); const clearTargeting = vi.fn(); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 74c2dfdd1..ecd186668 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -95,7 +95,17 @@ ts.servicesEnabled = true; } if (slotsToRefresh.length > 0) { - googletag.pubads().refresh(slotsToRefresh); + // One-shot bypass: this internal refresh delivers the just-applied + // server-side targeting to GAM. If slim-Prebid has already wrapped + // refresh(), it must pass this call straight through — not clear the + // targeting and run a duplicate client-side auction. Mirrors the + // bundle's adInit() in crates/js/lib/src/integrations/gpt/index.ts. + ts.adInitRefreshInProgress = true; + try { + googletag.pubads().refresh(slotsToRefresh); + } finally { + ts.adInitRefreshInProgress = false; + } } }); }; From ceeae6fd89ece5cae3d40b88c4cb829b249aa597 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 13:33:56 +0530 Subject: [PATCH 097/195] Address server-side ad review comments --- crates/js/lib/src/integrations/gpt/index.ts | 43 ++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 121 ++++++++++++++++++ crates/trusted-server-core/build.rs | 6 + .../src/creative_opportunities.rs | 72 +++++++++++ .../src/integrations/gpt.rs | 26 ++++ .../src/integrations/gpt_bootstrap.js | 21 ++- crates/trusted-server-core/src/settings.rs | 119 +++++++++++++++-- 7 files changed, 385 insertions(+), 23 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 8d138a9bf..9fd9ca5c3 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -313,13 +313,46 @@ function injectAdmIntoSlot(divId: string, adm: string): void { function fireWinBillingBeacons(slotId: string, bid: AuctionBidData): void { if (!slotId || (!bid.nurl && !bid.burl)) return; - const beaconKey = `${slotId}|${bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''}`; const fired = (window.tsjs!.firedBeacons ??= {}); - if (fired[beaconKey]) return; + const bidIdentity = bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''; + const urls = [ + ['nurl', bid.nurl], + ['burl', bid.burl], + ] as const; - fired[beaconKey] = true; - if (bid.nurl) navigator.sendBeacon(bid.nurl); - if (bid.burl) navigator.sendBeacon(bid.burl); + for (const [kind, url] of urls) { + if (!url) continue; + + const beaconKey = `${slotId}|${bidIdentity}|${kind}|${url}`; + if (fired[beaconKey]) continue; + + if (queueWinBillingBeacon(url)) { + fired[beaconKey] = true; + } + } +} + +function queueWinBillingBeacon(url: string): boolean { + if (typeof navigator !== 'undefined' && typeof navigator.sendBeacon === 'function') { + try { + if (navigator.sendBeacon(url)) { + return true; + } + } catch (err) { + log.warn('[tsjs-gpt] win/billing sendBeacon failed', err); + } + } + + if (typeof fetch === 'function') { + try { + void fetch(url, { method: 'POST', keepalive: true, mode: 'no-cors' }); + return true; + } catch (err) { + log.warn('[tsjs-gpt] win/billing fetch fallback failed', err); + } + } + + return false; } // ------------------------------------------------------------------ diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index 73d3be9c3..f7bb53e9d 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -707,6 +707,13 @@ describe('installTsRenderBridge', () => { fetchStub = vi.fn(); vi.stubGlobal('fetch', fetchStub); + if (typeof navigator.sendBeacon !== 'function') { + Object.defineProperty(navigator, 'sendBeacon', { + value: vi.fn().mockReturnValue(true), + writable: true, + configurable: true, + }); + } (window as TestWindow).tsjs = { bids: { @@ -747,6 +754,25 @@ describe('installTsRenderBridge', () => { return iframe.contentWindow!; } + async function captureBridgeListener(): Promise<(e: MessageEvent) => unknown> { + let bridgeListener: ((e: MessageEvent) => unknown) | undefined; + const origAdd = window.addEventListener.bind(window); + const addSpy = vi + .spyOn(window, 'addEventListener') + .mockImplementation( + (type: string, handler: EventListenerOrEventListenerObject, opts?: unknown) => { + if (type === 'message') bridgeListener = handler as (e: MessageEvent) => unknown; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + origAdd(type, handler as EventListener, opts as any); + } + ); + await import('../../../src/integrations/gpt/index'); + addSpy.mockRestore(); + + expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); + return bridgeListener!; + } + it('calls stopImmediatePropagation and fetches PBS Cache for a TS bid', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const mockAd = '
Test Creative
'; @@ -892,6 +918,101 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('falls back to keepalive fetch when sendBeacon is unavailable', async () => { + const originalSendBeacon = navigator.sendBeacon; + Object.defineProperty(navigator, 'sendBeacon', { + value: undefined, + writable: true, + configurable: true, + }); + + try { + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: 'debug-no-beacon', + hb_bidder: 'mocktioneer', + hb_pb: '0.20', + nurl: 'https://debug.example/win', + burl: 'https://debug.example/bill', + adm: '
Debug Creative
', + }; + + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + + expect(() => + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'debug-no-beacon' }), + ports: [fakePort], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ) + ).not.toThrow(); + + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/win', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/bill', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + } finally { + Object.defineProperty(navigator, 'sendBeacon', { + value: originalSendBeacon, + writable: true, + configurable: true, + }); + } + }); + + it('falls back to keepalive fetch when sendBeacon rejects the payload', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(false); + (window as TestWindow).tsjs.bids.homepage_header = { + hb_adid: 'debug-rejected-beacon', + hb_bidder: 'mocktioneer', + hb_pb: '0.20', + nurl: 'https://debug.example/win', + burl: 'https://debug.example/bill', + adm: '
Debug Creative
', + }; + + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + const event = Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'debug-rejected-beacon' }), + ports: [fakePort], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(event); + + expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/bill'); + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/win', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + expect(fetchStub).toHaveBeenCalledWith('https://debug.example/bill', { + method: 'POST', + keepalive: true, + mode: 'no-cors', + }); + + bridgeListener(event); + expect(fetchStub).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + it('ignores message when adId does not match any TS bid', async () => { await import('../../../src/integrations/gpt/index'); fetchStub.mockResolvedValue({ ok: true, text: () => Promise.resolve('') } as Response); diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index a4fe174ce..1787d5063 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -76,6 +76,12 @@ mod creative_opportunities { impl CreativeOpportunitiesConfig { /// No-op stub — pattern compilation only runs at runtime. pub fn compile_slots(&mut self) {} + + /// No-op stub — full slot-shape validation runs at runtime against + /// the real creative opportunity types. + pub fn validate_runtime(&self) -> Result<(), String> { + Ok(()) + } } /// Stub — the typed `slot` vec is always empty in the build context (see diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 645ba423f..67728bd28 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -51,6 +51,20 @@ impl CreativeOpportunitiesConfig { slot.compile_patterns(); } } + + /// Validate all slot definitions after runtime preparation. + /// + /// # Errors + /// + /// Returns an error string when a slot has an invalid identifier, page + /// pattern set, format list, dimensions, or resolved GAM unit path. + pub fn validate_runtime(&self) -> Result<(), String> { + for slot in &self.slot { + slot.validate_runtime(&self.gam_network_id)?; + } + + Ok(()) + } } /// A single ad placement opportunity on the publisher's site. @@ -94,6 +108,54 @@ pub struct CreativeOpportunitySlot { } impl CreativeOpportunitySlot { + /// Validate the slot shape after [`compile_patterns`](Self::compile_patterns) has run. + /// + /// # Errors + /// + /// Returns an error string when required slot fields are empty, invalid, + /// or semantically unusable at runtime. + pub fn validate_runtime(&self, gam_network_id: &str) -> Result<(), String> { + validate_slot_id(&self.id)?; + + if self.page_patterns.is_empty() { + return Err(format!( + "slot `{}` must include at least one page pattern", + self.id + )); + } + + if self.compiled_patterns.is_empty() { + return Err(format!( + "slot `{}` must include at least one valid page pattern", + self.id + )); + } + + if self.formats.is_empty() { + return Err(format!( + "slot `{}` must include at least one format", + self.id + )); + } + + for format in &self.formats { + format.validate_runtime(&self.id)?; + } + + if self + .resolved_gam_unit_path(gam_network_id) + .trim() + .is_empty() + { + return Err(format!( + "slot `{}` resolved GAM unit path must not be empty", + self.id + )); + } + + Ok(()) + } + /// Returns `true` if `path` matches any of this slot's [`page_patterns`](Self::page_patterns). /// /// Patterns use glob syntax (e.g., `"/20**"` matches any path starting with `/20`, @@ -236,6 +298,16 @@ pub struct CreativeOpportunityFormat { } impl CreativeOpportunityFormat { + fn validate_runtime(&self, slot_id: &str) -> Result<(), String> { + if self.width == 0 || self.height == 0 { + return Err(format!( + "slot `{slot_id}` format must have positive width and height" + )); + } + + Ok(()) + } + fn to_ad_format(&self) -> AdFormat { AdFormat { media_type: self.media_type.clone(), diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index dedb830f9..0a847651d 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1118,6 +1118,32 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_uses_css_safe_div_prefix_lookup() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("querySelectorAll(\"[id]\")"), + "bootstrap should scan ID-bearing elements instead of interpolating div_id into CSS" + ); + assert!( + combined.contains(".startsWith(slot.div_id)"), + "bootstrap should match metacharacter-containing div_id prefixes with startsWith" + ); + assert!( + !combined.contains("[id^='\" + slot.div_id"), + "bootstrap must not build a CSS attribute selector from raw div_id" + ); + } + #[test] fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { let config = test_config(); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index ecd186668..90eb2181b 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -31,14 +31,23 @@ // All slots to refresh (TS-defined + publisher-owned reused). var slotsToRefresh = []; slots.forEach(function (slot) { - // Resolve actual div ID: exact match first, then prefix query. + // Resolve actual div ID: exact match first, then safe prefix scan. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when // the suffix is dynamically generated by the framework at render time. - var el = - document.getElementById(slot.div_id) || - document.querySelector( - "[id^='" + slot.div_id + "']:not([id$='-container'])", - ); + var el = document.getElementById(slot.div_id); + if (!el) { + var idElements = document.querySelectorAll("[id]"); + for (var i = 0; i < idElements.length; i++) { + var candidate = idElements[i]; + if ( + candidate.id.startsWith(slot.div_id) && + !candidate.id.endsWith("-container") + ) { + el = candidate; + break; + } + } + } if (!el) return; var actualDivId = el.id; var b = bids[slot.id] || {}; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 286ab4234..24c933552 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1786,7 +1786,7 @@ impl Settings { /// /// Returns a configuration error if any cached runtime artifact cannot be /// prepared, if any handler path regex does not compile, or if a creative - /// opportunity slot has an invalid ID. + /// opportunity slot is invalid. pub fn prepare_runtime(&mut self) -> Result<(), Report> { self.image_optimizer.prepare_runtime()?; self.proxy.prepare_runtime()?; @@ -1798,16 +1798,14 @@ impl Settings { if let Some(co) = &mut self.creative_opportunities { co.compile_slots(); - // Slot IDs flow into injected HTML/JS and provider payloads, and - // can arrive via TRUSTED_SERVER__ env overrides that bypass any - // static config review — validate them on every load path. - for slot in &co.slot { - crate::creative_opportunities::validate_slot_id(&slot.id).map_err(|err| { - Report::new(TrustedServerError::Configuration { - message: format!("Invalid creative opportunity slot id: {err}"), - }) - })?; - } + // Slots flow into injected HTML/JS, provider payloads, and GPT + // calls. Env/private config can bypass static review, so validate + // the full runtime shape on every load path. + co.validate_runtime().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity slot config: {err}"), + }) + })?; } Ok(()) @@ -4681,11 +4679,108 @@ formats = [{ width = 300, height = 250 }] "#; let err = Settings::from_toml(toml).expect_err("should reject invalid slot id"); assert!( - format!("{err:?}").contains("Invalid creative opportunity slot id"), + format!("{err:?}").contains("Invalid creative opportunity slot config"), "error should mention the invalid slot id, got: {err:?}" ); } + fn creative_opportunity_settings_toml(slot_body: &str) -> String { + format!( + r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" + +[[creative_opportunities.slot]] +{slot_body} +"# + ) + } + + fn assert_creative_opportunity_slot_config_rejected(slot_body: &str, expected: &str) { + let toml = creative_opportunity_settings_toml(slot_body); + let err = Settings::from_toml(&toml) + .expect_err("should reject malformed creative opportunity slot"); + assert!( + format!("{err:?}").contains(expected), + "error should contain {expected:?}, got: {err:?}" + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_without_page_patterns() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = [] +formats = [{ width = 300, height = 250 }] +"#, + "must include at least one page pattern", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_without_valid_page_patterns() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = ["["] +formats = [{ width = 300, height = 250 }] +"#, + "must include at least one valid page pattern", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_without_formats() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = ["/"] +formats = [] +"#, + "must include at least one format", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_with_zero_dimensions() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +page_patterns = ["/"] +formats = [{ width = 0, height = 250 }] +"#, + "must have positive width and height", + ); + } + + #[test] + fn settings_rejects_creative_opportunity_slot_with_empty_gam_unit_path() { + assert_creative_opportunity_slot_config_rejected( + r#" +id = "atf" +gam_unit_path = "" +page_patterns = ["/"] +formats = [{ width = 300, height = 250 }] +"#, + "resolved GAM unit path must not be empty", + ); + } + #[test] fn admin_endpoints_match_fastly_router() { let router_source = include_str!("../../trusted-server-adapter-fastly/src/main.rs"); From 911a6456b44cf9d6b62f39c2cd067d21c7f70415 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 20:43:26 +0530 Subject: [PATCH 098/195] Restore publisher platform-http-client test on the merged signature Re-add publisher_request_uses_platform_http_client_with_http_types, dropped during the main merge because it called the pre-feature 4-arg handle_publisher_request. A run_publisher_proxy test helper supplies the no-auction EC/AuctionDispatch wiring so the test body stays a plain (settings, registry, services, req) proxy call. --- crates/trusted-server-core/src/publisher.rs | 73 ++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1e75197d6..3c89b923d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1931,7 +1931,9 @@ mod tests { use super::*; use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; - use crate::platform::test_support::noop_services; + use crate::platform::test_support::{ + build_services_with_http_client, noop_services, StubHttpClient, + }; use crate::test_support::tests::create_test_settings; use edgezero_core::body::Body as EdgeBody; use http::{header, Method, Request as HttpRequest, StatusCode}; @@ -2162,6 +2164,75 @@ mod tests { ); } + /// Drive `handle_publisher_request` with no creative opportunities — a plain + /// proxy with no server-side auction. Hides the auction/EC wiring so callers + /// read like a simple `(settings, registry, services, req)` proxy. + async fn run_publisher_proxy( + settings: &Settings, + integration_registry: &IntegrationRegistry, + services: &RuntimeServices, + req: Request, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let fastly_req = crate::compat::to_fastly_request_ref(&req); + let mut ec_context = + EcContext::read_from_request(settings, &fastly_req).expect("should read EC context"); + handle_publisher_request( + settings, + integration_registry, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request") + } + + #[tokio::test] + async fn publisher_request_uses_platform_http_client_with_http_types() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, ®istry, &services, req).await { + PublisherResponse::Buffered(r) => r, + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + response + } + PublisherResponse::Stream { response, .. } => response, + }; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + String::from_utf8(response.into_body().into_bytes().to_vec()) + .expect("response body should be valid UTF-8"), + "origin response" + ); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "should proxy through the platform http client" + ); + } + #[test] fn test_content_type_detection() { let test_cases = vec![ From 89281f0edfc24475b3e8978e69c2526522442733 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 21:51:47 +0530 Subject: [PATCH 099/195] Gate POST /auction behind the server-side auction consent check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publisher-navigation and /__ts/page-bids paths fail closed for GDPR or unknown jurisdictions that lack effective TCF Purpose 1, but POST /auction proceeded straight to run_auction after only stripping EC IDs/EIDs — still dispatching PBS/APS calls and forwarding request-derived signals (UA/IP/geo, and cookies under some Prebid consent-forwarding modes) for traffic the gate says must not run a server-side auction. Apply consent_allows_server_side_auction before resolving EIDs or contacting providers; when it denies, return an empty no-bid OpenRTB response without invoking run_auction. Add a regression test that registers a panic-on-bid provider and proves a GDPR/unknown request lacking Purpose 1 returns no bids without contacting any provider. Route the orchestration-failure /auction tests through a non-regulated geo so they still exercise the provider path. --- .../src/route_tests.rs | 28 +++- .../src/auction/endpoints.rs | 150 +++++++++++++++++- 2 files changed, 176 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 7e2aa23f7..c616223bc 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -378,6 +378,23 @@ fn us_california_geo() -> GeoInfo { } } +/// Geo resolving to a non-regulated jurisdiction, so the server-side auction +/// consent gate (which fails closed for GDPR/unknown jurisdictions without TCF +/// Purpose 1) allows the auction to proceed. Used by `/auction` route tests +/// that exercise orchestration behavior rather than consent. +fn non_regulated_geo() -> GeoInfo { + GeoInfo { + city: "Example City".to_string(), + country: "AU".to_string(), + continent: "OC".to_string(), + latitude: -33.8, + longitude: 151.2, + metro_code: 0, + region: Some("NSW".to_string()), + asn: None, + } +} + fn valid_ec_id() -> String { format!("{}.Abc123", "a".repeat(64)) } @@ -594,7 +611,16 @@ fn route_auction_with_stack( let req = Request::post("https://test.com/auction") .with_header(header::CONTENT_TYPE, "application/json") .with_body(body.into()); - let services = test_runtime_services(&req); + // Resolve to a non-regulated jurisdiction so the server-side auction consent + // gate allows the auction; these tests assert orchestration behavior, not + // consent gating (covered separately in endpoints.rs). + let services = test_runtime_services_with_secret_http_client_and_geo( + &req, + Arc::new(NoopBackend), + Arc::new(NoopSecretStore), + Arc::new(NoopHttpClient) as Arc, + Arc::new(FixedGeo(non_regulated_geo())), + ); let route_result = futures::executor::block_on(route_request( settings, diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f72954212..5ed59aae5 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1,12 +1,15 @@ //! HTTP endpoint handlers for auction requests. +use std::collections::HashMap; + use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{header, Request, Response, StatusCode}; use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; -use crate::consent::gate_eids_by_consent; +use crate::auction::orchestrator::OrchestrationResult; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::COOKIE_TS_EIDS; use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::kv::KvIdentityGraph; @@ -163,6 +166,43 @@ pub async fn handle_auction( }; let consent_context = ec_context.consent().clone(); + // Server-side auction consent gate. The publisher-navigation and + // `/__ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that + // lack effective TCF Purpose 1. `/auction` is the programmatic entry point + // for the same server-side auction, so it must gate identically: returning + // a no-bid response here prevents outbound PBS/APS calls and the forwarding + // of request-derived signals (UA/IP/geo, and cookies under some Prebid + // consent-forwarding modes) for traffic that must not run an auction. + if !consent_allows_server_side_auction(&consent_context) { + log::info!( + "/auction: server-side auction consent gate denied; returning no-bid response without contacting providers" + ); + // Build the request shape locally (no outbound calls, no geo lookup, no + // EID resolution) so the no-bid OpenRTB response echoes the request id. + let auction_request = convert_tsjs_to_auction_request( + &body, + settings, + services, + &http_req, + consent_context, + ec_id, + None, + )?; + let empty_result = OrchestrationResult { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + }; + return convert_to_openrtb_response( + &empty_result, + settings, + &auction_request, + ec_context.ec_allowed(), + ); + } + // Parse client-provided EIDs from the current request body. When the // current request does not include them, fall back to the persisted // `ts-eids` cookie so later requests can still forward the browser's @@ -444,12 +484,19 @@ pub(crate) fn merge_auction_eids( #[cfg(test)] mod tests { use super::*; + use crate::auction::config::AuctionConfig; + use crate::auction::provider::AuctionProvider; + use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; use crate::openrtb::Uid; + use crate::platform::test_support::noop_services; + use crate::platform::{PlatformPendingRequest, PlatformResponse}; + use crate::test_support::tests::create_test_settings; use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine as _; use serde_json::json; + use std::sync::Arc; fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { EcContext::new_for_test( @@ -461,6 +508,107 @@ mod tests { ) } + /// Provider that fails the test if it is ever contacted. Used to prove the + /// `/auction` consent gate short-circuits before any outbound bid request. + struct PanicOnBidProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for PanicOnBidProvider { + fn provider_name(&self) -> &'static str { + "panic_provider" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + panic!("provider must not be contacted when the consent gate denies the auction"); + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("provider must not parse a response when the auction is gated off"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _timeout_ms: u32) -> Option { + Some("panic-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { + // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run + // a server-side auction. The /auction endpoint must short-circuit to a + // no-bid response before dispatching to any provider — matching the + // publisher-navigation and /__ts/page-bids paths. + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: true, + providers: vec!["panic_provider".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let services = noop_services(); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + + let body = json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("gated auction should still return a valid response"); + + assert_eq!( + response.status(), + StatusCode::OK, + "gated auction should return a 200 no-bid response" + ); + let body_bytes = response.into_body().into_bytes(); + let parsed: JsonValue = + serde_json::from_slice(&body_bytes).expect("response body should be valid JSON"); + let seatbid_empty = match parsed.get("seatbid").and_then(JsonValue::as_array) { + Some(seatbid) => seatbid.is_empty(), + None => true, + }; + assert!( + seatbid_empty, + "gated auction must return no bids, got: {parsed}" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_kv() { let registry = PartnerRegistry::empty(); From 4d4fb1b238902b74ede109d14166ca0ba2cc430e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 15 Jun 2026 21:51:47 +0530 Subject: [PATCH 100/195] Validate creative-opportunity slots at build time build.rs deserialized slots into a stub whose validate_runtime was a no-op and only checked slot-id syntax, so an invalid trusted-server.toml or TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT override (empty page_patterns, empty formats, zero dimensions, empty resolved GAM unit path) passed CI and got embedded, then failed at request time as a configuration error. Extract the validation into creative_slot_build_check, shared by build.rs (via #[path]) and the crate test build (via #[cfg(test)] mod) so the rules run under cargo test. It mirrors CreativeOpportunitySlot::validate_runtime and runs against the merged config (base TOML plus TRUSTED_SERVER__* env overrides) before the config is serialized and embedded, so an invalid slot fails the build and is never persisted. --- crates/trusted-server-core/build.rs | 53 ++--- .../src/creative_slot_build_check.rs | 201 ++++++++++++++++++ crates/trusted-server-core/src/lib.rs | 4 + 3 files changed, 232 insertions(+), 26 deletions(-) create mode 100644 crates/trusted-server-core/src/creative_slot_build_check.rs diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index 1787d5063..cee32e259 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -100,6 +100,10 @@ mod creative_opportunities { #[path = "src/settings.rs"] mod settings; +#[path = "src/creative_slot_build_check.rs"] +mod creative_slot_build_check; + +use creative_slot_build_check::validate_creative_slot; use std::fs; use std::path::Path; @@ -118,38 +122,24 @@ fn main() { let toml_content = fs::read_to_string(init_config_path) .unwrap_or_else(|_| panic!("Failed to read {init_config_path:?}")); - // Merge base TOML with environment variable overrides and write output. + // Merge base TOML with environment variable overrides. // Panics if admin endpoints are not covered by a handler. let settings = settings::Settings::from_toml_and_env(&toml_content) .expect("Failed to parse settings at build time"); - let merged_toml = - toml::to_string_pretty(&settings).expect("Failed to serialize settings to TOML"); - - // Only write when content changes to avoid unnecessary recompilation. - let dest_path = Path::new(TRUSTED_SERVER_OUTPUT_CONFIG_PATH); - let current = fs::read_to_string(dest_path).unwrap_or_default(); - if current != merged_toml { - fs::write(dest_path, merged_toml) - .unwrap_or_else(|_| panic!("Failed to write {dest_path:?}")); - } - - // Validate slot IDs from [creative_opportunities.slot] in trusted-server.toml - let slot_id_re = regex::Regex::new(r"^[A-Za-z0-9_\-]+$").expect("should compile regex"); + // Validate [creative_opportunities.slot] entries from the *merged* config + // (base trusted-server.toml plus any TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT + // env overrides) before it is serialized and embedded. This mirrors the + // runtime validator (CreativeOpportunitySlot::validate_runtime) — the build + // context uses a stub whose validate_runtime is a no-op, so without this an + // invalid slot would pass CI and surface as a request-time configuration + // error / service outage. The validator is shared with the crate (see + // `creative_slot_build_check`) so it stays under test. Running it before the + // write also means a rejected config is never persisted to the embedded file. if let Some(co) = &settings.creative_opportunities { for slot in &co.slot_raw { - if let Some(id) = slot.get("id").and_then(|v| v.as_str()) { - if !slot_id_re.is_match(id) { - panic!( - "trusted-server.toml [creative_opportunities.slot]: slot id '{}' is invalid; \ - only [A-Za-z0-9_-] allowed", - id - ); - } - } else { - panic!( - "trusted-server.toml [creative_opportunities.slot]: a slot entry is missing the required 'id' field" - ); + if let Err(err) = validate_creative_slot(slot, &co.gam_network_id) { + panic!("trusted-server.toml [creative_opportunities.slot]: {err}"); } } if !co.slot_raw.is_empty() { @@ -159,4 +149,15 @@ fn main() { ); } } + + let merged_toml = + toml::to_string_pretty(&settings).expect("Failed to serialize settings to TOML"); + + // Only write when content changes to avoid unnecessary recompilation. + let dest_path = Path::new(TRUSTED_SERVER_OUTPUT_CONFIG_PATH); + let current = fs::read_to_string(dest_path).unwrap_or_default(); + if current != merged_toml { + fs::write(dest_path, merged_toml) + .unwrap_or_else(|_| panic!("Failed to write {dest_path:?}")); + } } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs new file mode 100644 index 000000000..55d17f918 --- /dev/null +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -0,0 +1,201 @@ +//! Build-time validation for creative-opportunity slot definitions. +//! +//! This module is compiled in two contexts: +//! - by `build.rs` (via `#[path]`), which runs it against the raw slot JSON +//! merged from `trusted-server.toml` and `TRUSTED_SERVER__*` env overrides +//! before the config is embedded into the binary; +//! - by the crate's test build (via `#[cfg(test)] mod`), so the rules below are +//! exercised under `cargo test`. +//! +//! It mirrors the runtime validator +//! (`CreativeOpportunitySlot::validate_runtime`) so an invalid slot fails the +//! build instead of surfacing as a request-time configuration error. It reads +//! raw JSON (not the typed runtime struct) because the typed slot vec is +//! intentionally empty in the build context, keeping `build.rs` free of the +//! full runtime dependency graph. + +/// Returns `true` when `id` is non-empty and only `[A-Za-z0-9_-]`. +fn is_valid_slot_id(id: &str) -> bool { + !id.is_empty() + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') +} + +/// Validate a single raw creative-opportunity slot. +/// +/// Mirrors the runtime checks in `CreativeOpportunitySlot::validate_runtime`: +/// a syntactically safe non-empty id, at least one non-empty page pattern, at +/// least one format with positive dimensions, and a non-empty resolved GAM unit +/// path. Returns an error string describing the first problem found. +/// +/// # Errors +/// +/// Returns an error string when the slot is missing required fields, has an +/// invalid id, has no usable page pattern or format, has a zero-dimension +/// format, or resolves to an empty GAM unit path. +pub(crate) fn validate_creative_slot( + slot: &serde_json::Value, + gam_network_id: &str, +) -> Result<(), String> { + let id = match slot.get("id").and_then(serde_json::Value::as_str) { + Some(id) => id, + None => return Err("a slot entry is missing the required 'id' field".to_string()), + }; + if id.is_empty() { + return Err("slot id must not be empty".to_string()); + } + if !is_valid_slot_id(id) { + return Err(format!( + "slot id '{id}' is invalid; only [A-Za-z0-9_-] allowed" + )); + } + + // At least one non-empty page pattern. + let has_valid_pattern = slot + .get("page_patterns") + .and_then(serde_json::Value::as_array) + .is_some_and(|patterns| { + patterns + .iter() + .any(|p| p.as_str().is_some_and(|s| !s.trim().is_empty())) + }); + if !has_valid_pattern { + return Err(format!( + "slot `{id}` must include at least one non-empty page pattern" + )); + } + + // At least one format, each with positive width and height. + match slot.get("formats").and_then(serde_json::Value::as_array) { + Some(formats) if !formats.is_empty() => { + for format in formats { + let width = format.get("width").and_then(serde_json::Value::as_u64); + let height = format.get("height").and_then(serde_json::Value::as_u64); + if !matches!((width, height), (Some(w), Some(h)) if w > 0 && h > 0) { + return Err(format!( + "slot `{id}` format must have positive width and height" + )); + } + } + } + _ => { + return Err(format!("slot `{id}` must include at least one format")); + } + } + + // Resolved GAM unit path must not be empty. An explicit override is used + // when present; otherwise it is derived as `//`. + let resolved_gam_unit_path = match slot + .get("gam_unit_path") + .and_then(serde_json::Value::as_str) + { + Some(path) => path.to_string(), + None => format!("/{gam_network_id}/{id}"), + }; + if resolved_gam_unit_path.trim().is_empty() { + return Err(format!( + "slot `{id}` resolved GAM unit path must not be empty" + )); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::validate_creative_slot; + use serde_json::json; + + #[test] + fn accepts_a_well_formed_slot() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }] + }); + assert!(validate_creative_slot(&slot, "123456789").is_ok()); + } + + #[test] + fn accepts_explicit_gam_unit_path_override() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/"], + "formats": [{ "width": 300, "height": 250 }], + "gam_unit_path": "/123456789/publisher/atf" + }); + assert!(validate_creative_slot(&slot, "123456789").is_ok()); + } + + #[test] + fn rejects_empty_formats() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("empty formats must fail at build time"); + assert!(err.contains("at least one format"), "got: {err}"); + } + + #[test] + fn rejects_zero_dimension_format() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 0, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("zero dimensions must fail at build time"); + assert!(err.contains("positive width and height"), "got: {err}"); + } + + #[test] + fn rejects_empty_page_patterns() { + let slot = json!({ + "id": "atf", + "page_patterns": [], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("empty page patterns must fail at build time"); + assert!(err.contains("page pattern"), "got: {err}"); + } + + #[test] + fn rejects_blank_page_pattern_strings() { + let slot = json!({ + "id": "atf", + "page_patterns": [" "], + "formats": [{ "width": 300, "height": 250 }] + }); + assert!(validate_creative_slot(&slot, "123456789").is_err()); + } + + #[test] + fn rejects_blank_gam_unit_path_override() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "gam_unit_path": " " + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("blank GAM unit path must fail at build time"); + assert!(err.contains("GAM unit path"), "got: {err}"); + } + + #[test] + fn rejects_missing_id() { + let slot = json!({ "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); + assert!(validate_creative_slot(&slot, "net").is_err()); + } + + #[test] + fn rejects_invalid_id_characters() { + let slot = json!({ "id": "a b", "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); + assert!(validate_creative_slot(&slot, "net").is_err()); + } +} diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index de71e010c..23ea63045 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -42,6 +42,10 @@ pub mod constants; pub mod cookies; pub mod creative; pub mod creative_opportunities; +// Build-time slot validation, shared with `build.rs` via `#[path]`. Compiled +// here only under test so its rules stay exercised by `cargo test`. +#[cfg(test)] +mod creative_slot_build_check; pub mod ec; pub(crate) mod edge_cookie; pub mod error; From c99bac8b8763c20f7e7045285bae2e30df494db8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 101/195] Restore mediated render/accounting fields on the synchronous auction path run_parallel_mediation parsed the mediator response through parse_response, which (for adserver_mock) drops nurl/burl/ad_id and PBS cache fields restored only in parse_response_with_context. The synchronous mediation path used by POST /auction and /__ts/page-bids could therefore return mediated cache bids without hb_adid / cache metadata, breaking creative rendering and win/billing beacons even though the dispatched collect path preserves them. Call parse_response_with_context with the mediator context (which carries the collected SSP responses), matching the dispatched collect path. Add a regression test proving a mediated bid keeps its restored nurl/ad_id through run_auction. --- .../src/auction/orchestrator.rs | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index ba06e6c74..dc3bb5e83 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -266,8 +266,14 @@ impl AuctionOrchestrator { })?; let response_time_ms = start_time.elapsed().as_millis() as u64; + // Use the context-aware parse so mediators (e.g. adserver_mock) can + // restore nurl/burl/ad_id and PBS cache fields from the collected SSP + // responses. The dispatched collect path already does this; the + // synchronous mediation path used by POST /auction and + // /__ts/page-bids must match or mediated cache bids lose the metadata + // needed for creative rendering and win/billing beacons. let mediator_resp = mediator - .parse_response(platform_resp, response_time_ms) + .parse_response_with_context(platform_resp, response_time_ms, &mediator_context) .await .change_context(TrustedServerError::Auction { message: format!("Mediator {} parse failed", mediator.provider_name()), @@ -1211,6 +1217,159 @@ mod tests { } } + /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring + /// `adserver_mock`), while its context-free parse does not. Lets a test prove + /// the synchronous mediation path calls `parse_response_with_context`. + struct CacheRestoringMediator; + + fn mediated_bid(nurl: Option) -> Bid { + Bid { + slot_id: "header-banner".to_string(), + price: Some(2.5), + currency: "USD".to_string(), + creative: Some("
ad
".to_string()), + adomain: None, + bidder: "mediator".to_string(), + width: 728, + height: 90, + nurl: nurl.clone(), + burl: nurl, + ad_id: Some("creative-123".to_string()), + cache_id: Some("cache-abc".to_string()), + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for CacheRestoringMediator { + fn provider_name(&self) -> &'static str { + "mediator" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let req = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/mediate") + .body(edgezero_core::body::Body::empty()) + .expect("should build mediator request"), + "mediator-backend", + ); + context + .services + .http_client() + .send_async(req) + .await + .change_context(TrustedServerError::Auction { + message: "mediator launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + // Context-free path: cannot restore SSP-only render/accounting fields. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(None)], + response_time_ms, + )) + } + + async fn parse_response_with_context( + &self, + _response: PlatformResponse, + response_time_ms: u64, + _context: &AuctionContext<'_>, + ) -> Result> { + // Context-aware path: restores nurl/ad_id from the collected SSP bids. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(Some("https://nurl.example/win".to_string()))], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 2000 + } + + fn backend_name(&self, _timeout_ms: u32) -> Option { + Some("mediator-backend".to_string()) + } + } + + #[tokio::test] + async fn mediated_bid_preserves_restored_fields_through_run_auction() { + // run_parallel_mediation must parse the mediator response via + // parse_response_with_context so cache/nurl fields restored from SSP + // responses survive the synchronous mediation path (POST /auction, + // /__ts/page-bids), matching the dispatched collect path. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // bidder send_async + stub.push_response(200, b"{}".to_vec()); // mediator send_async + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + mediator: Some("mediator".to_string()), + timeout_ms: 2000, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider { + name: "bidder", + backend: "bidder-backend", + })); + orchestrator.register_provider(Arc::new(CacheRestoringMediator)); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("mediated auction should complete"); + + let bid = result + .winning_bids + .get("header-banner") + .expect("mediator should produce a winning bid for the slot"); + assert_eq!( + bid.nurl.as_deref(), + Some("https://nurl.example/win"), + "synchronous mediation must restore nurl via parse_response_with_context" + ); + assert_eq!( + bid.ad_id.as_deref(), + Some("creative-123"), + "mediated bid must keep its restored ad_id" + ); + } + fn create_test_auction_request() -> AuctionRequest { AuctionRequest { id: "test-auction-123".to_string(), From b9d2d06483ca642465c0c892ee1e1a32884f3379 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 102/195] Display TS-defined GPT slots instead of only refreshing them In the fallback path where Trusted Server defines a GPT slot itself, the code called defineSlot().addService() then refresh(), but never googletag.display() for the new slot. GPT requires a display() call to register/render a slot, so TS-owned first-impression slots no-op ("defineSlot was called without a matching display call") and miss impressions. Reused publisher-owned slots are unaffected because the publisher already displayed them. Track TS-defined slot element IDs separately, display() them once after services are enabled, and keep refresh() for reused publisher-owned slots only. Mirror the change in the inline gpt_bootstrap.js. Add Vitest coverage for the TS-owned display path and keep the refresh-bypass test on a reused slot. --- crates/js/lib/src/integrations/gpt/index.ts | 32 ++++++++--- .../lib/test/integrations/gpt/ad_init.test.ts | 53 ++++++++++++++++++- .../src/integrations/gpt_bootstrap.js | 23 ++++++-- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 9fd9ca5c3..2effbc593 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -392,8 +392,14 @@ export function installTsAdInit(): void { // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. const newSlots: GoogleTagSlot[] = []; - // All slots to refresh (TS-defined + publisher-owned reused). + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. const slotsToRefresh: GoogleTagSlot[] = []; + // Element IDs of slots TS defined itself this call. GPT requires a + // display() call to register/render a freshly-defined slot; refresh() + // alone no-ops for a slot that was never displayed, so these are + // display()ed instead of refreshed. + const slotsToDisplay: string[] = []; const divToSlotId: Record = {}; const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; const nextSlotTargetingKeys: Record = {}; @@ -468,8 +474,12 @@ export function installTsAdInit(): void { const slotTargetingKeys = Object.keys(slot.targeting ?? {}); nextSlotTargetingKeys[actualDivId] = slotTargetingKeys; if (slotDivId2 !== actualDivId) nextSlotTargetingKeys[slotDivId2] = slotTargetingKeys; - if (tsOwned) newSlots.push(gptSlot); - slotsToRefresh.push(gptSlot); + if (tsOwned) { + newSlots.push(gptSlot); + slotsToDisplay.push(slotDivId2); + } else { + slotsToRefresh.push(gptSlot); + } // APS: signal to apstag that bids are ready so Amazon's GAM creative // can render. apstag must already be initialised on the page (which it @@ -507,12 +517,20 @@ export function installTsAdInit(): void { }); } + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot — without it the slot no-ops ("defineSlot was + // called without a matching display call") and misses its impression. + // Must run after enableServices(); on SPA navigation services are already + // enabled, so this runs unconditionally for any newly-defined slots. + slotsToDisplay.forEach((divId) => g.display?.(divId)); + if (slotsToRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied - // server-side targeting to GAM. If slim-Prebid has wrapped refresh(), - // it must pass this call straight through — not clear the targeting - // and run a duplicate client-side auction. Later publisher-initiated - // refreshes of the same slots still go through the wrapper normally. + // server-side targeting to GAM for reused publisher-owned slots. If + // slim-Prebid has wrapped refresh(), it must pass this call straight + // through — not clear the targeting and run a duplicate client-side + // auction. Later publisher-initiated refreshes of the same slots still + // go through the wrapper normally. ts.adInitRefreshInProgress = true; try { g.pubads!().refresh(slotsToRefresh); diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index f7bb53e9d..43551644a 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -138,6 +138,55 @@ describe('installTsAdInit', () => { fetchSpy.mockRestore(); }); + it('displays TS-defined slots and does not include them in refresh', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + // Publisher has not defined this slot, so TS defines (owns) it. + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const defineSlotMock = vi.fn().mockReturnValue(mockSlot); + const displayMock = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: defineSlotMock, + display: displayMock, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(defineSlotMock).toHaveBeenCalled(); + // GPT requires display() to register/render a freshly-defined slot. + expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); + // TS-owned slots are displayed, not refreshed (refresh() no-ops for a slot + // that was never displayed). + expect(mockPubads.refresh).not.toHaveBeenCalled(); + }); + it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -148,7 +197,9 @@ describe('installTsAdInit', () => { let flagDuringRefresh: boolean | undefined; const mockPubads = { enableSingleRequest: vi.fn(), - getSlots: vi.fn().mockReturnValue([]), + // Publisher-owned slot reused by TS, so it goes through refresh() (which + // carries the bypass flag) rather than display(). + getSlots: vi.fn().mockReturnValue([mockSlot]), addEventListener: vi.fn(), refresh: vi.fn(() => { flagDuringRefresh = (window as TestWindow).tsjs!.adInitRefreshInProgress; diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 90eb2181b..46cfe0fd3 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -28,8 +28,13 @@ // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. var newSlots = []; - // All slots to refresh (TS-defined + publisher-owned reused). + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. var slotsToRefresh = []; + // Element IDs of slots TS defined itself. GPT requires display() to + // register/render a freshly-defined slot; refresh() alone no-ops for a + // slot that was never displayed, so these are display()ed instead. + var slotsToDisplay = []; slots.forEach(function (slot) { // Resolve actual div ID: exact match first, then safe prefix scan. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when @@ -93,8 +98,13 @@ if (slotElementId && slotElementId !== actualDivId) { divToSlotId[slotElementId] = slot.id; } - if (tsOwned) newSlots.push(s); - slotsToRefresh.push(s); + if (tsOwned) { + newSlots.push(s); + var displayId = s.getSlotElementId() || actualDivId; + slotsToDisplay.push(displayId); + } else { + slotsToRefresh.push(s); + } }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; @@ -103,6 +113,13 @@ googletag.enableServices(); ts.servicesEnabled = true; } + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot; without it the slot no-ops and misses its + // impression. Runs after enableServices(); on SPA navigation services are + // already enabled, so this runs unconditionally for new slots. + slotsToDisplay.forEach(function (divId) { + googletag.display(divId); + }); if (slotsToRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied // server-side targeting to GAM. If slim-Prebid has already wrapped From fa1410011563ac9648c2a21b6ec1cad8a0a76b37 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 103/195] Validate creative-opportunity glob patterns at build time The build-time validator only checked for a non-empty page_patterns string, so a config like page_patterns = ["["] passed the release build and then failed settings load at runtime when compile_patterns rejected the slot. Compile each pattern with the same glob::Pattern::new + ** -> * normalization contract used by the runtime compile_patterns, requiring at least one pattern that compiles. Adds glob as a build-dependency and tests for an uncompilable pattern and the recursive ** case. --- crates/trusted-server-core/Cargo.toml | 1 + .../src/creative_slot_build_check.rs | 48 +++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index 6e2cbd82f..b86b48dbd 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -55,6 +55,7 @@ edgezero-core = { workspace = true } config = { workspace = true } derive_more = { workspace = true } error-stack = { workspace = true } +glob = { workspace = true } http = { workspace = true } log = { workspace = true } regex = { workspace = true } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 55d17f918..9970e0d28 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -22,6 +22,17 @@ fn is_valid_slot_id(id: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') } +/// Returns `true` when `pattern` compiles as a glob, mirroring the runtime +/// `CreativeOpportunitySlot::compile_patterns` contract: try `glob::Pattern::new` +/// directly, then fall back to the `**` -> `*` normalization. A pattern that +/// fails both is dropped at runtime, leaving the slot unmatchable, so the build +/// must reject it too. +fn pattern_compiles(pattern: &str) -> bool { + glob::Pattern::new(pattern) + .or_else(|_| glob::Pattern::new(&pattern.replace("**", "*"))) + .is_ok() +} + /// Validate a single raw creative-opportunity slot. /// /// Mirrors the runtime checks in `CreativeOpportunitySlot::validate_runtime`: @@ -51,18 +62,22 @@ pub(crate) fn validate_creative_slot( )); } - // At least one non-empty page pattern. + // At least one page pattern that is non-empty and compiles as a glob. + // Runtime preparation drops uncompilable patterns and rejects the slot when + // none remain, so a private/env config like `page_patterns = ["["]` would + // otherwise pass the build and fail settings load on the deployed service. let has_valid_pattern = slot .get("page_patterns") .and_then(serde_json::Value::as_array) .is_some_and(|patterns| { patterns .iter() - .any(|p| p.as_str().is_some_and(|s| !s.trim().is_empty())) + .filter_map(serde_json::Value::as_str) + .any(|s| !s.trim().is_empty() && pattern_compiles(s)) }); if !has_valid_pattern { return Err(format!( - "slot `{id}` must include at least one non-empty page pattern" + "slot `{id}` must include at least one valid page pattern" )); } @@ -174,6 +189,33 @@ mod tests { assert!(validate_creative_slot(&slot, "123456789").is_err()); } + #[test] + fn rejects_uncompilable_glob_pattern() { + // `[` is an unterminated character class; it fails to compile both + // directly and after the ** -> * normalization, so the slot would be + // unmatchable at runtime. + let slot = json!({ + "id": "atf", + "page_patterns": ["["], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("uncompilable glob pattern must fail at build time"); + assert!(err.contains("valid page pattern"), "got: {err}"); + } + + #[test] + fn accepts_recursive_glob_pattern() { + // `/20**` fails direct glob compilation but compiles after the + // ** -> * normalization, matching runtime behavior. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }] + }); + assert!(validate_creative_slot(&slot, "123456789").is_ok()); + } + #[test] fn rejects_blank_gam_unit_path_override() { let slot = json!({ From 8f13d5f808676aedb204787a0383fe3a3a1ef869 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 104/195] Match Cache-Control privacy directives case-insensitively finalize_response checked for lowercase "private"/"no-store" substrings, but Cache-Control directives are case-insensitive (RFC 9111). A Cache-Control: No-Store on a Set-Cookie response was treated as cacheable and downgraded to the weaker private, max-age=0, and a Cache-Control: Private did not block operator response_headers from re-enabling shared caching. Lowercase the header value before matching. Add mixed-case No-Store / Private tests. --- .../trusted-server-adapter-fastly/src/main.rs | 4 ++ .../src/route_tests.rs | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index c71ea9cde..7d81c3aa3 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -807,10 +807,13 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: // net covers ordinary navigations whose sole per-user payload is the cookie. // Skip when the response is already uncacheable so we don't clobber a // stricter directive (e.g. `no-store`). + // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match + // against a lowercased copy — `No-Store` / `Private` must count. let already_uncacheable = response .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) .is_some_and(|v| v.contains("private") || v.contains("no-store")); if !already_uncacheable && response.headers().contains_key(header::SET_COOKIE) { response.headers_mut().insert( @@ -829,6 +832,7 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) .is_some_and(|v| v.contains("private")); for (key, value) in &settings.response_headers { diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index c616223bc..9b987a843 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -1152,6 +1152,58 @@ fn finalize_response_leaves_stricter_no_store_untouched() { ); } +#[test] +fn finalize_response_treats_mixed_case_no_store_as_uncacheable() { + // Cache-Control directives are case-insensitive: `No-Store` on a Set-Cookie + // response must be recognized as already-uncacheable and left untouched, not + // downgraded to the weaker `private, max-age=0`. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "No-Store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("No-Store"), + "mixed-case No-Store must be treated as uncacheable and preserved" + ); +} + +#[test] +fn finalize_response_mixed_case_private_blocks_operator_surrogate_reenable() { + // A mixed-case `Private` directive must still mark the response private so + // operator response_headers cannot re-enable shared caching. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "Private, max-age=0") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable caching for a mixed-case Private response" + ); +} + #[test] fn finalize_response_cookie_net_blocks_operator_surrogate_reenable() { // Operator response_headers must not re-add surrogate caching once the From f997c66ee6c5f94c7833fc4618ab4e78a682afca Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 09:26:48 +0530 Subject: [PATCH 105/195] Align checked-in creative auction timeout with its 500ms guidance The comment recommends a 500ms default because the value bounds the DOMContentLoaded/window.load slip, but the checked-in value was 1500ms, so a first rollout that enables slots while inheriting the default would impose a 1.5s close-body hold on cache-hit pages. Set the sample default to 500ms. --- trusted-server.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trusted-server.toml b/trusted-server.toml index 0bd461b43..dc64d468a 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -357,7 +357,7 @@ gam_network_id = "123456789" # drains in <50 ms but the auction runs to the limit. 500 ms is the recommended # default; raise only if your SSPs need more headroom and your analytics confirm # the DCL slip is acceptable. -auction_timeout_ms = 1500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS +auction_timeout_ms = 500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" # No slot templates are enabled in the checked-in default config. Add From 3a5c4b4b06668c737f36634bce1d572048c3578e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 106/195] Correct float-truncation under-bucketing in price_bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many two-decimal CPMs are not exactly representable in binary floating point: 0.29 * 100.0 is 28.999…, so flooring truncated it to 28 ("0.28"), and 1.15 became "1.14". These values feed hb_pb targeting keys, so the auction reported a cent low. Convert CPM to whole cents through a helper that nudges values sitting an ULP below a cent boundary up before flooring, leaving genuinely sub-cent values (0.015 -> "0.01") untouched. Adds a float-boundary regression test. --- .../trusted-server-core/src/price_bucket.rs | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs index 8fc4e50e6..30b7430de 100644 --- a/crates/trusted-server-core/src/price_bucket.rs +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -11,30 +11,40 @@ pub enum PriceGranularity { Auto, } +/// Convert a CPM in dollars to whole cents, flooring to the cent. +/// +/// Multiplying by 100 and flooring directly under-buckets common CPMs because +/// many two-decimal values are not exactly representable in binary floating +/// point: `0.29 * 100.0` is `28.999…`, which would truncate to `28` ("0.28"). +/// A tiny epsilon corrects values sitting an ULP below a cent boundary without +/// promoting genuinely sub-cent values — `0.015` (`1.4999…`) still floors to +/// `1` ("0.01"), while `0.29` correctly yields `29`. +fn cpm_to_cents(cpm: f64) -> u64 { + const CENT_EPSILON: f64 = 1e-6; + (cpm * 100.0 + CENT_EPSILON).floor() as u64 +} + #[must_use] pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { - // Reject NaN / Inf early so the `(x * 100.0).floor() as u64` cast below - // can never see a non-finite value (the cast's behaviour for NaN/Inf is - // implementation-defined in Rust and "saturate to 0" only by convention). + // Reject NaN / Inf early so the cast in `cpm_to_cents` can never see a + // non-finite value (the cast's behaviour for NaN/Inf is implementation- + // defined in Rust and "saturate to 0" only by convention). if !cpm.is_finite() || cpm <= 0.0 { return "0.00".to_string(); } match granularity { PriceGranularity::Low => { - let capped = cpm.min(5.0); - let cents = (capped * 100.0).floor() as u64; + let cents = cpm_to_cents(cpm.min(5.0)); let bucketed_cents = (cents / 50) * 50; format!("{:.2}", bucketed_cents as f64 / 100.0) } PriceGranularity::Medium => { - let capped = cpm.min(20.0); - let cents = (capped * 100.0).floor() as u64; + let cents = cpm_to_cents(cpm.min(20.0)); let bucketed_cents = (cents / 10) * 10; format!("{:.2}", bucketed_cents as f64 / 100.0) } PriceGranularity::High => { - let capped = cpm.min(20.0); - let cents = (capped * 100.0).floor() as u64; + let cents = cpm_to_cents(cpm.min(20.0)); format!("{:.2}", cents as f64 / 100.0) } PriceGranularity::Dense | PriceGranularity::Auto => dense_bucket(cpm), @@ -46,17 +56,14 @@ fn dense_bucket(cpm: f64) -> String { return "20.00".to_string(); } if cpm >= 8.0 { - let cents = (cpm * 100.0).floor() as u64; - let bucketed_cents = (cents / 50) * 50; + let bucketed_cents = (cpm_to_cents(cpm) / 50) * 50; return format!("{:.2}", bucketed_cents as f64 / 100.0); } if cpm >= 3.0 { - let cents = (cpm * 100.0).floor() as u64; - let bucketed_cents = (cents / 5) * 5; + let bucketed_cents = (cpm_to_cents(cpm) / 5) * 5; return format!("{:.2}", bucketed_cents as f64 / 100.0); } - let cents = (cpm * 100.0).floor() as u64; - format!("{:.2}", cents as f64 / 100.0) + format!("{:.2}", cpm_to_cents(cpm) as f64 / 100.0) } #[cfg(test)] @@ -122,6 +129,19 @@ mod tests { ); } + #[test] + fn float_boundary_cpms_are_not_under_bucketed() { + // These two-decimal CPMs are not exactly representable in binary float + // (`0.29 * 100.0 == 28.999…`); a naive floor truncates them a cent low. + assert_eq!(price_bucket(0.29, PriceGranularity::Dense), "0.29"); + assert_eq!(price_bucket(1.15, PriceGranularity::Dense), "1.15"); + assert_eq!(price_bucket(0.29, PriceGranularity::High), "0.29"); + assert_eq!(price_bucket(1.15, PriceGranularity::High), "1.15"); + // Genuinely sub-cent values must still floor, not round up. + assert_eq!(price_bucket(0.289, PriceGranularity::High), "0.28"); + assert_eq!(price_bucket(0.015, PriceGranularity::Dense), "0.01"); + } + #[test] fn non_finite_cpm_returns_zero_bucket() { for granularity in [ From 0f241cad79370d5c0ac4435f3680c39e41a4e924 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 107/195] Cap synchronous mediator timeout to its configured budget run_parallel_mediation gave the mediator the full remaining auction budget, while the dispatched collect path bounds it by remaining.min(mediator.timeout_ms()). Apply the same cap for symmetry between the two paths. --- crates/trusted-server-core/src/auction/orchestrator.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index dc3bb5e83..48aebaa97 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -243,7 +243,9 @@ impl AuctionOrchestrator { let mediator_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: remaining_ms, + // Bound by both the remaining auction budget and the mediator's + // own configured timeout, matching the dispatched collect path. + timeout_ms: remaining_ms.min(mediator.timeout_ms()), provider_responses: Some(&provider_responses), services: context.services, }; From 64ecc74b08edce9173b8733701236a3883736a71 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 108/195] Warn when a dispatched auction is dropped on non-streaming routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit should_run_auction is decided from request signals before the origin content-type/status/encoding is known. A navigation that dispatched SSP bid requests but then routes to PassThrough (2xx non-HTML) or BufferedUnmodified (non-2xx, unsupported encoding, empty host) dropped the DispatchedAuction without collecting it — wasted SSP quota with no visibility. Log a warning on those arms when an auction was dispatched. --- crates/trusted-server-core/src/publisher.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 3c89b923d..2bcb7a751 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1346,6 +1346,17 @@ pub async fn handle_publisher_request( content_type, status, ); + if dispatched_auction.is_some() { + // should_run_auction is decided from request signals before the + // origin content-type is known. A pass-through (2xx non-HTML) + // response has no `` to inject bids into, so the dispatched + // SSP requests are wasted — surface it for quota observability. + log::warn!( + "Server-side auction dispatched but response routed to pass-through (Content-Type: '{}', status: {}); in-flight SSP bid requests will not be collected", + content_type, + status, + ); + } let (parts, body) = response.into_parts(); let response = Response::from_parts(parts, EdgeBody::empty()); Ok(PublisherResponse::PassThrough { response, body }) @@ -1368,6 +1379,16 @@ pub async fn handle_publisher_request( status, ); } + if dispatched_auction.is_some() { + // Same wasted-dispatch case as the pass-through arm: an + // unprocessable/non-2xx response can't carry injected bids, so + // the in-flight SSP requests are left uncollected. + log::warn!( + "Server-side auction dispatched but response routed to buffered-unmodified (Content-Type: '{}', status: {}); in-flight SSP bid requests will not be collected", + content_type, + status, + ); + } Ok(PublisherResponse::Buffered(response)) } ResponseRoute::Stream => { From ac0add3b9f7278c315981e902dd7a1c470386fd1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 109/195] Test env-injected creative-opportunity slot-id rejection Lock in that a TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT override with an invalid id is rejected through from_toml_and_env, complementing the existing TOML-path test and exercising the same validation the build-time check uses. --- crates/trusted-server-core/src/settings.rs | 46 ++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 04172a9dc..bfbdc329a 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4697,6 +4697,52 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_rejects_env_injected_invalid_creative_opportunity_slot_id() { + // A TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT override must go through + // the same runtime slot validation as a TOML-defined slot, so an invalid + // id injected via env is rejected by from_toml_and_env (the build-time + // path uses the same validation against the merged config). + let toml = r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" +"#; + let slot_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}SLOT", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + temp_env::with_var( + slot_key, + Some( + r#"[{"id":"bad id","page_patterns":["/"],"formats":[{"width":300,"height":250}]}]"#, + ), + || { + let err = Settings::from_toml_and_env(toml) + .expect_err("should reject env-injected invalid slot id"); + assert!( + format!("{err:?}").contains("Invalid creative opportunity slot config"), + "error should mention the invalid slot id, got: {err:?}" + ); + }, + ); + } + fn creative_opportunity_settings_toml(slot_body: &str) -> String { format!( r#" From c324e09c5fc937a5986255cbaa4ac4bcc66c7094 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 110/195] Use a valid glob as the page-pattern doc example "/20**" is an invalid glob that only matches via the **->* normalization fallback; using it as the canonical example invites copy-paste of broken config. Show "/2024/*" as the primary example and keep the normalization note as the edge-case caveat. --- .../trusted-server-core/src/creative_opportunities.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 67728bd28..cf2b401d6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -158,11 +158,12 @@ impl CreativeOpportunitySlot { /// Returns `true` if `path` matches any of this slot's [`page_patterns`](Self::page_patterns). /// - /// Patterns use glob syntax (e.g., `"/20**"` matches any path starting with `/20`, - /// `"/"` matches only the root). When a pattern contains `**` in a position that the - /// glob crate considers invalid (e.g., `b**`), the `**` is normalised to `*` before - /// matching. A single `*` matches any sequence of characters including path separators - /// because `require_literal_separator` is `false`. + /// Patterns use glob syntax (e.g., `"/2024/*"` matches any path under `/2024/`, + /// `"/"` matches only the root). A single `*` matches any sequence of characters + /// including path separators because `require_literal_separator` is `false`. + /// When a pattern contains `**` in a position the glob crate considers invalid + /// (e.g., `"/20**"` or `"b**"`), the `**` is normalised to `*` before matching — + /// prefer a valid single-`*` pattern over relying on this fallback. /// /// Patterns that cannot be compiled even after normalisation are silently skipped. #[must_use] From bdb00f56c27bf0d9d40c13ec2bf6eabfc21f0db6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 16 Jun 2026 11:22:33 +0530 Subject: [PATCH 111/195] Remove dead test-only parse_ts_eids_cookie helper parse_ts_eids_cookie was gated to #[cfg(test)] and exercised only by its own tests; production reads the ts-eids cookie through resolve_client_auction_eids -> parse_prebid_eids_cookie (which enforces its own size/length caps). Remove the function, its tests, and the now-orphaned cfg(test) imports/helpers. --- crates/trusted-server-core/src/cookies.rs | 109 ---------------------- 1 file changed, 109 deletions(-) diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 9ad09926d..a002d9c8f 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -9,12 +9,8 @@ use error_stack::{Report, ResultExt}; use http::header; use http::Request; -#[cfg(test)] -use crate::constants::COOKIE_TS_EIDS; use crate::constants::{COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_US_PRIVACY}; use crate::error::TrustedServerError; -#[cfg(test)] -use base64::{engine::general_purpose::STANDARD, Engine as _}; /// Cookie names carrying privacy consent signals. /// @@ -73,42 +69,6 @@ pub fn handle_request_cookies( } } -/// Parse Extended User IDs from the [`COOKIE_TS_EIDS`] cookie. -/// -/// The cookie value is a standard-base64-encoded JSON array of -/// [`crate::openrtb::Eid`] objects written by the Trusted Server JS SDK via -/// `btoa(JSON.stringify(eids))`. -/// -/// Returns `None` if the cookie is absent, base64-malformed, JSON-malformed, -/// or the decoded array is empty. Parse failures are logged at `debug` level -/// so operators can diagnose JS SDK / server mismatches. -#[cfg(test)] -#[must_use] -pub(crate) fn parse_ts_eids_cookie(jar: Option<&CookieJar>) -> Option> { - let value = jar?.get(COOKIE_TS_EIDS)?.value().to_owned(); - let decoded = match STANDARD.decode(&value) { - Ok(b) => b, - Err(e) => { - log::debug!("ts-eids cookie: base64 decode failed: {e}"); - return None; - } - }; - match serde_json::from_slice::>(&decoded) { - Ok(eids) if !eids.is_empty() => { - if eids.len() > 32 || eids.iter().any(|e| e.uids.len() > 32) { - log::debug!("ts-eids cookie: too many eids or uids, rejecting"); - return None; - } - Some(eids) - } - Ok(_) => None, - Err(e) => { - log::debug!("ts-eids cookie: JSON parse failed: {e}"); - None - } - } -} - /// Strips named cookies from a `Cookie` header value string. /// /// Parses the semicolon-separated cookie pairs, filters out any whose name @@ -448,73 +408,4 @@ mod tests { let stripped = strip_cookies(header, CONSENT_COOKIE_NAMES); assert_eq!(stripped, "session=abc=123=def"); } - - fn make_jar_with(name: &str, value: &str) -> CookieJar { - parse_cookies_to_jar(&format!("{name}={value}")) - } - - fn encode_eids(eids: &[serde_json::Value]) -> String { - use base64::{engine::general_purpose::STANDARD, Engine as _}; - STANDARD.encode(serde_json::to_string(eids).expect("should serialize eids")) - } - - #[test] - fn parse_ts_eids_cookie_returns_eids_for_valid_input() { - let encoded = encode_eids(&[serde_json::json!({ - "source": "id5-sync.com", - "uids": [{"id": "abc123", "atype": 1}] - })]); - let jar = make_jar_with(COOKIE_TS_EIDS, &encoded); - let eids = parse_ts_eids_cookie(Some(&jar)).expect("should parse valid ts-eids cookie"); - assert_eq!(eids.len(), 1, "should return one EID"); - assert_eq!(eids[0].source, "id5-sync.com", "should preserve source"); - assert_eq!(eids[0].uids[0].id, "abc123", "should preserve uid"); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_when_cookie_absent() { - let jar = CookieJar::new(); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None when cookie absent" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_empty_array() { - let encoded = encode_eids(&[]); - let jar = make_jar_with(COOKIE_TS_EIDS, &encoded); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None for empty EID array" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_corrupt_base64() { - let jar = make_jar_with(COOKIE_TS_EIDS, "not!!valid!!base64"); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None for corrupt base64" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_invalid_json() { - use base64::{engine::general_purpose::STANDARD, Engine as _}; - let encoded = STANDARD.encode(b"this is not json"); - let jar = make_jar_with(COOKIE_TS_EIDS, &encoded); - assert!( - parse_ts_eids_cookie(Some(&jar)).is_none(), - "should return None for invalid JSON" - ); - } - - #[test] - fn parse_ts_eids_cookie_returns_none_for_none_jar() { - assert!( - parse_ts_eids_cookie(None).is_none(), - "should return None when jar is None" - ); - } } From 83620ab0e11d0d5d0b0f6854f05ab233e1ce8afd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 13:25:45 +0530 Subject: [PATCH 112/195] Force EC Set-Cookie responses to stay shared-uncacheable finalize_response applies the cookie cache-privacy downgrade on the HttpResponse, but the EC identity cookie is written later by ec_finalize_response onto the converted Fastly response. A first-visit navigation whose only per-user payload is the EC cookie therefore kept any public/surrogate cache headers from the origin or operator response headers, so a shared cache could store and replay one visitor's EC cookie to others. Re-apply the downgrade with enforce_set_cookie_cache_privacy after EC finalization in both the buffered and streaming branches, mirror it in the route test helper, and cover the first-visit ordering with route tests. --- .../trusted-server-adapter-fastly/src/main.rs | 31 ++++++++ .../src/route_tests.rs | 71 +++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index e7f8a71a3..943ff94cd 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -253,6 +253,9 @@ fn main() { &mut fastly_resp, ); } + // EC finalization may have just added the identity Set-Cookie, which + // the HttpResponse-stage cache guard could not see. + enforce_set_cookie_cache_privacy(&mut fastly_resp); request_filter_effects.apply_to_fastly_response(&mut fastly_resp); fastly_resp.send_to_client(); @@ -281,6 +284,9 @@ fn main() { &mut fastly_resp, ); } + // EC finalization may have just added the identity Set-Cookie, which + // the HttpResponse-stage cache guard could not see. + enforce_set_cookie_cache_privacy(&mut fastly_resp); request_filter_effects.apply_to_fastly_response(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); let mut stream_succeeded = false; @@ -906,6 +912,31 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: } } +/// Forces cookie-bearing Fastly responses to stay private to shared caches. +/// +/// [`finalize_response`] applies this same downgrade on the [`HttpResponse`], +/// but the EC identity cookie is written later by [`ec_finalize_response`] onto +/// the converted [`FastlyResponse`], so the earlier guard never sees it. +/// Re-apply it here so a first-visit navigation whose only per-user payload is +/// the EC `Set-Cookie` can never be served with `public`/surrogate cache headers +/// inherited from the origin or operator response headers — a shared cache must +/// not be able to store and replay one visitor's EC cookie to others. +/// +/// Idempotent: a response already marked `private`/`no-store` is left untouched +/// so a stricter directive is never weakened. +fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { + let already_uncacheable = response + .get_header_str("cache-control") + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if already_uncacheable || response.get_header("set-cookie").is_none() { + return; + } + response.set_header("cache-control", "private, max-age=0"); + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); +} + fn http_error_response(report: &Report) -> HttpResponse { let root_error = report.current_context(); log::error!("Error occurred: {:?}", report); diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 2048816fc..11a32c8c0 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -638,6 +638,7 @@ fn route_result_to_fastly_response( &mut fastly_response, ); } + super::enforce_set_cookie_cache_privacy(&mut fastly_response); request_filter_effects.apply_to_fastly_response(&mut fastly_response); fastly_response } @@ -1463,6 +1464,76 @@ fn finalize_response_makes_cookie_bearing_responses_private() { ); } +#[test] +fn ec_set_cookie_added_after_finalize_downgrades_origin_public_cache() { + // First-visit navigation: the origin response is shared-cacheable and carries + // no cookie, so the HttpResponse-stage finalizer keeps its cache headers. EC + // finalization then mints the identity Set-Cookie on the converted Fastly + // response, after that guard has already run. The post-EC privacy guard must + // downgrade caching so a shared cache cannot replay one visitor's EC cookie. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"); + + // No cookie at this stage, so the cookie net does not fire and the origin + // cache directive survives finalize_response — reproducing the gap. + super::finalize_response(&settings, None, &mut response); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("public, max-age=3600"), + "a cookieless response should keep its origin cache directive" + ); + + let mut fastly_response = compat::to_fastly_response(response); + // Stand in for ec_finalize_response minting the first-visit identity cookie: + // its EcContext constructors are #[cfg(test)] in trusted-server-core and are + // not reachable from this crate, but the only behavior under test here is the + // post-EC ordering — a Set-Cookie appearing after finalize_response ran. + fastly_response.set_header(header::SET_COOKIE, "ec=abc; Path=/; HttpOnly"); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("private, max-age=0"), + "an EC Set-Cookie added after finalize_response must downgrade caching" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "EC Set-Cookie responses must not retain surrogate cacheability" + ); +} + +#[test] +fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { + // A stricter directive minted alongside the cookie must not be weakened to + // the `private, max-age=0` downgrade. + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("no-store"), + "an already-uncacheable response should keep its stricter directive" + ); +} + #[test] fn finalize_response_leaves_stricter_no_store_untouched() { let settings = create_test_settings(); From 0cf84e4634d9fcec53e1c4cbc65778524892b60f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 13:29:55 +0530 Subject: [PATCH 113/195] Preserve server-side bidder params on Prebid refresh auctions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthetic refresh ad unit only carried the trustedServer bid with a zone, so the requestBids shim had no original server-side bidder entries to collect into bidderParams. Refresh/scroll /auction requests therefore sent {} for inline PBS params and dropped demand the publisher configured only on the initial ad unit. Recover the matching original pbjs.adUnits server-side params by ad unit code — from both raw bidder entries and params already folded onto the initial trustedServer bid — and attach them to the synthetic refresh bid. --- .../js/lib/src/integrations/prebid/index.ts | 55 +++++++- .../test/integrations/prebid/index.test.ts | 124 ++++++++++++++++++ 2 files changed, 175 insertions(+), 4 deletions(-) diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index 61b546e5b..835d28fdb 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -372,6 +372,49 @@ function clientSideBidsForRefresh( return bids; } +/** + * Recover the publisher's inline server-side (PBS) bidder params for a slot. + * + * The synthetic refresh ad unit carries only the `trustedServer` bid, so the + * `requestBids` shim has no original server-side bidder entries to collect into + * `bidderParams` — without this, refresh/scroll `/auction` requests send `{}` + * and lose demand the publisher configured only on the initial ad unit. Source + * the params from the matching `pbjs.adUnits` entry by code, covering both + * states the initial auction can leave that entry in: + * - raw server-side bidder entries (`{ bidder, params }`) not yet folded, and + * - params already folded into that unit's `trustedServer` bid `bidderParams` + * by a prior `requestBids` call. + */ +function serverSideBidderParamsForRefresh(code: string): Record> { + const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; + const match = adUnits.find((unit) => unit.code === code); + if (!match?.bids) return {}; + + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); + const params: Record> = {}; + + for (const bid of match.bids) { + if (!bid?.bidder) continue; + if (bid.bidder === ADAPTER_CODE) { + // Params captured and folded onto the trustedServer bid by an earlier + // requestBids call. + const folded = (bid.params?.[BIDDER_PARAMS_KEY] ?? {}) as Record< + string, + Record + >; + for (const [bidder, bidderParams] of Object.entries(folded)) { + params[bidder] = bidderParams; + } + continue; + } + if (clientSideBidders.has(bid.bidder)) continue; + // Raw server-side bidder entry not yet folded by the shim. + params[bid.bidder] = bid.params ?? {}; + } + + return params; +} + function clearRefreshTargeting(slot: RefreshGptSlot): void { if (typeof slot.clearTargeting !== 'function') return; @@ -694,13 +737,17 @@ export function installRefreshHandler(timeoutMs = 1500): void { }; const code = refreshSlotElementId(slot) ?? 'refresh-slot'; + const tsParams: Record = zone ? { [ZONE_KEY]: zone } : {}; + // Carry the publisher's inline server-side (PBS) bidder params captured + // on the initial ad unit so refresh/scroll auctions don't drop them. + const serverSideParams = serverSideBidderParamsForRefresh(code); + if (Object.keys(serverSideParams).length > 0) { + tsParams[BIDDER_PARAMS_KEY] = serverSideParams; + } return { code, mediaTypes: { banner }, - bids: [ - { bidder: ADAPTER_CODE, params: zone ? { [ZONE_KEY]: zone } : {} }, - ...clientSideBidsForRefresh(code), - ], + bids: [{ bidder: ADAPTER_CODE, params: tsParams }, ...clientSideBidsForRefresh(code)], }; }); diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 2ca650e18..5edad541f 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -922,6 +922,130 @@ describe('prebid/installRefreshHandler', () => { mockPbjs.adUnits = []; }); + it('preserves raw server-side bidder params in refresh ad units', () => { + // Original publisher ad unit carries an inline server-side appnexus bid that + // the initial auction has not yet folded into the trustedServer bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [{ bidder: 'appnexus', params: { placementId: 12345 } }], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + + it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { + // After the initial auction, the requestBids shim has folded the publisher's + // server-side params into the original ad unit's trustedServer bid. A later + // refresh must still recover them by code. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { bidderParams: { appnexus: { placementId: 12345 } } }, + }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); const clearTargeting = vi.fn(); From 32de4aa5b8907f0b25bdb3b687c97bf16cda43a6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 13:38:53 +0530 Subject: [PATCH 114/195] Reject build-time creative-opportunity configs the runtime can't load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build script types price_granularity as a String and slots as raw JSON values, so values the runtime schema rejects — a price_granularity outside the PriceGranularity enum (e.g. custom), or unknown slot keys under the slot's deny_unknown_fields — embedded cleanly and then failed settings load on every non-health request, turning a green build into a request-time outage. Validate price_granularity against the real PriceGranularity enum and reject unknown top-level slot fields in the shared build-check validator before the merged config is embedded, with tests for both. --- crates/trusted-server-core/build.rs | 8 +- .../src/creative_slot_build_check.rs | 116 +++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index cee32e259..f52986c8a 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -103,7 +103,7 @@ mod settings; #[path = "src/creative_slot_build_check.rs"] mod creative_slot_build_check; -use creative_slot_build_check::validate_creative_slot; +use creative_slot_build_check::{validate_creative_slot, validate_price_granularity}; use std::fs; use std::path::Path; @@ -137,6 +137,12 @@ fn main() { // `creative_slot_build_check`) so it stays under test. Running it before the // write also means a rejected config is never persisted to the embedded file. if let Some(co) = &settings.creative_opportunities { + // price_granularity is a String stub in the build context, so validate it + // against the real PriceGranularity enum before embedding — an invalid + // value would otherwise fail runtime settings load on every request. + if let Err(err) = validate_price_granularity(&co.price_granularity) { + panic!("trusted-server.toml [creative_opportunities]: {err}"); + } for slot in &co.slot_raw { if let Err(err) = validate_creative_slot(slot, &co.gam_network_id) { panic!("trusted-server.toml [creative_opportunities.slot]: {err}"); diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 9970e0d28..6a0f446b7 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -14,6 +14,54 @@ //! intentionally empty in the build context, keeping `build.rs` free of the //! full runtime dependency graph. +/// Top-level slot fields the runtime [`CreativeOpportunitySlot`] accepts. +/// +/// The runtime struct is `#[serde(deny_unknown_fields)]`, but the build context +/// deserializes slots as raw `serde_json::Value`, which silently keeps unknown +/// keys. Mirror the runtime field set here so an env-injected typo or stray key +/// fails the build instead of failing settings load on every request. +/// +/// `compiled_patterns` is intentionally excluded: it is `#[serde(skip)]` on the +/// runtime struct and is never a valid input field. +/// +/// [`CreativeOpportunitySlot`]: crate::creative_opportunities::CreativeOpportunitySlot +const ALLOWED_SLOT_FIELDS: &[&str] = &[ + "id", + "gam_unit_path", + "div_id", + "page_patterns", + "formats", + "floor_price", + "targeting", + "providers", +]; + +/// Validate that `value` is a `price_granularity` the runtime can deserialize. +/// +/// The build context types `price_granularity` as a `String`, so an invalid +/// value such as `custom` would embed cleanly and then fail runtime settings +/// load — the real [`PriceGranularity`] enum cannot deserialize it — on every +/// non-health request. Delegating to that enum's `Deserialize` impl keeps the +/// accepted set in lockstep with the runtime, avoiding drift. +/// +/// # Errors +/// +/// Returns an error string when `value` is not one of the runtime +/// [`PriceGranularity`] variants. +/// +/// [`PriceGranularity`]: crate::price_bucket::PriceGranularity +pub(crate) fn validate_price_granularity(value: &str) -> Result<(), String> { + serde_json::from_value::(serde_json::Value::String( + value.to_string(), + )) + .map(|_| ()) + .map_err(|_| { + format!( + "price_granularity '{value}' is invalid; expected one of: low, medium, dense, high, auto" + ) + }) +} + /// Returns `true` when `id` is non-empty and only `[A-Za-z0-9_-]`. fn is_valid_slot_id(id: &str) -> bool { !id.is_empty() @@ -62,6 +110,17 @@ pub(crate) fn validate_creative_slot( )); } + // Reject unknown top-level keys, mirroring the runtime slot's + // `#[serde(deny_unknown_fields)]`. The raw-JSON build path would otherwise + // accept env-injected typos that the runtime rejects at settings load. + if let Some(object) = slot.as_object() { + for key in object.keys() { + if !ALLOWED_SLOT_FIELDS.contains(&key.as_str()) { + return Err(format!("slot `{id}` has unknown field '{key}'")); + } + } + } + // At least one page pattern that is non-empty and compiles as a glob. // Runtime preparation drops uncompilable patterns and rejects the slot when // none remain, so a private/env config like `page_patterns = ["["]` would @@ -119,9 +178,64 @@ pub(crate) fn validate_creative_slot( #[cfg(test)] mod tests { - use super::validate_creative_slot; + use super::{validate_creative_slot, validate_price_granularity}; use serde_json::json; + #[test] + fn rejects_unknown_slot_field() { + // The runtime slot is deny_unknown_fields, so an env-injected typo like + // `floorprice` must fail the build, not pass it and break settings load. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "floorprice": 1.5 + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown slot field must fail at build time"); + assert!(err.contains("unknown field 'floorprice'"), "got: {err}"); + } + + #[test] + fn accepts_all_known_slot_fields() { + let slot = json!({ + "id": "atf", + "gam_unit_path": "/123456789/publisher/atf", + "div_id": "atf-div", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "floor_price": 1.5, + "targeting": { "pos": "atf" }, + "providers": {} + }); + assert!( + validate_creative_slot(&slot, "123456789").is_ok(), + "all documented slot fields must be accepted" + ); + } + + #[test] + fn accepts_valid_price_granularities() { + for value in ["low", "medium", "dense", "high", "auto"] { + assert!( + validate_price_granularity(value).is_ok(), + "'{value}' should be a valid price_granularity" + ); + } + } + + #[test] + fn rejects_invalid_price_granularity() { + // The runtime PriceGranularity enum has no `custom` variant, so a build + // that embeds it would fail settings load on every request. + let err = validate_price_granularity("custom") + .expect_err("invalid price_granularity must fail at build time"); + assert!( + err.contains("price_granularity 'custom' is invalid"), + "got: {err}" + ); + } + #[test] fn accepts_a_well_formed_slot() { let slot = json!({ From f456b506b0df3a64cd7fbb1578e82045913e551b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 17 Jun 2026 22:45:51 +0530 Subject: [PATCH 115/195] Close EC and ad-stack gaps in page-bids and publisher flow Stop handle_publisher_request from minting its own EC ID. EC generation is the adapter's real-browser-gated responsibility; the duplicate inline call re-ran for any navigation with no real-browser signal, so a non-real-browser client could get an IP-derived EC minted in memory and forwarded to PBS/APS even though the adapter blocked EC operations. Gate /__ts/page-bids slot output on the effective ad-stack condition (auction kill switch + consent), not just winning bids. Returning slots while the stack is disabled let the SPA hook run adInit() and create or refresh GPT slots client-side, defeating the kill switch. This matches the publisher navigation path's should_run_server_side_ad_stack gate. Add deny_unknown_fields to the top-level creative-opportunities config and nested provider/format structs so misspelled keys fail at startup instead of silently disabling or mis-timing the ad stack. Add regression tests for all three and update the page-bids tests to isolate the bot/prefetch variable from the consent gate. --- .../src/creative_opportunities.rs | 46 ++++ crates/trusted-server-core/src/publisher.rs | 245 ++++++++++++++---- 2 files changed, 239 insertions(+), 52 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cf2b401d6..2b3fa6e72 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -16,6 +16,7 @@ use crate::settings::vec_from_seq_or_map; /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { /// GAM network ID used to build default unit paths. pub gam_network_id: String, @@ -288,6 +289,7 @@ impl CreativeOpportunitySlot { /// An ad format combining a media type with pixel dimensions. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct CreativeOpportunityFormat { /// Creative width in pixels. pub width: u32, @@ -320,6 +322,7 @@ impl CreativeOpportunityFormat { /// Provider-specific slot identifiers for a [`CreativeOpportunitySlot`]. #[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct SlotProviders { /// Amazon Publisher Services (APS/TAM) slot parameters. pub aps: Option, @@ -333,6 +336,7 @@ pub struct SlotProviders { /// APS-specific parameters for a slot. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ApsSlotParams { /// The APS slot ID string used when making TAM bid requests. pub slot_id: String, @@ -345,6 +349,7 @@ pub struct ApsSlotParams { /// When `bidders` is non-empty the map is forwarded verbatim, bypassing /// automatic expansion (useful for slots that need explicit per-bidder params). #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct PrebidSlotParams { /// Per-bidder inline params map. Bidder name → params object. /// @@ -595,6 +600,47 @@ mod tests { ); } + #[test] + fn config_rejects_unknown_top_level_key() { + // A typo such as `slots` instead of `slot` must surface as a config + // error rather than silently deserializing to an empty (disabled) stack. + let typo = serde_json::json!({ "gam_network_id": "12345", "slots": [] }); + assert!( + serde_json::from_value::(typo).is_err(), + "unknown top-level key should be rejected by deny_unknown_fields" + ); + + let correct = serde_json::json!({ "gam_network_id": "12345", "slot": [] }); + assert!( + serde_json::from_value::(correct).is_ok(), + "the correct `slot` key should still deserialize" + ); + } + + #[test] + fn config_rejects_unknown_nested_keys() { + // Format typo: `med.a_type` instead of `media_type`. + let format_typo = serde_json::json!({ "width": 300, "height": 250, "meda_type": "banner" }); + assert!( + serde_json::from_value::(format_typo).is_err(), + "unknown format key should be rejected" + ); + + // Provider typo: `prebd` instead of `prebid`. + let providers_typo = serde_json::json!({ "prebd": {} }); + assert!( + serde_json::from_value::(providers_typo).is_err(), + "unknown provider key should be rejected" + ); + + // APS typo: `slotId` instead of `slot_id`. + let aps_typo = serde_json::json!({ "slotId": "x" }); + assert!( + serde_json::from_value::(aps_typo).is_err(), + "unknown APS key should be rejected" + ); + } + #[test] fn prebid_slot_params_deserializes_without_bidders_field() { let json = r#"{"bidders": {}}"#; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 2bcb7a751..d19f761d3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1072,18 +1072,14 @@ pub async fn handle_publisher_request( let is_navigation = is_navigation_request(&req); - // Generate a new EC ID only for document navigations. Subresource - // requests (fonts, images, CSS) may lack consent signals such as the - // Sec-GPC header, so we skip generation to avoid setting identity - // cookies when the user's consent preference is unknown. - if is_navigation { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { - log::warn!("EC generation failed: {err:?}"); - } - } else { - log::debug!("EC generation skipped: non-document request"); - } - + // EC generation is the caller's responsibility — it must run only for real + // browsers on document navigations, and that real-browser decision lives in + // the adapter (TLS/JA4/device gate). Generating here, with only the + // navigation signal, would mint an IP-derived EC for clients the adapter + // classified as non-real browsers and forward it to SSPs/APS even though EC + // operations were blocked for them. The adapter calls + // `EcContext::generate_if_needed` (real-browser-gated) before dispatching to + // this handler; subresource requests are likewise filtered there. let ec_allowed = ec_context.ec_allowed(); log::debug!( "Proxy EC state: has_ec_id={}, ec_allowed={ec_allowed}", @@ -1815,12 +1811,16 @@ pub async fn handle_page_bids( ); } - let winning_bids = if auction_enabled - && !matched_slots.is_empty() - && consent_allows_auction - && !is_bot - && !is_prefetch - { + // The [auction].enabled kill switch and a consent denial disable the entire + // server-side ad stack. In those states the endpoint must return no slots, + // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — + // otherwise the kill switch/consent gate would stop SSP calls but still let + // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, + // keep their slot definitions (the placement structure is unchanged) but + // skip the live auction, matching the existing bot/prefetch behaviour. + let ad_stack_enabled = auction_enabled && consent_allows_auction; + + let winning_bids = if ad_stack_enabled && !matched_slots.is_empty() && !is_bot && !is_prefetch { let slots_ctx = MatchedSlotsContext { matched_slots: &matched_slots, request_path: &path_param, @@ -1892,30 +1892,36 @@ pub async fn handle_page_bids( settings.debug.inject_adm_for_testing, ); - let slots_json: Vec = matched_slots - .iter() - .map(|slot| { - let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); - let div_id = slot.resolved_div_id(); - let formats: Vec = slot - .formats - .iter() - .map(|f| serde_json::json!([f.width, f.height])) - .collect(); - let targeting: serde_json::Map = slot - .targeting - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) - .collect(); - serde_json::json!({ - "id": slot.id, - "gam_unit_path": gam_path, - "div_id": div_id, - "formats": formats, - "targeting": targeting, + // Gate slots on the ad-stack kill switch / consent: when disabled, return no + // slots so the SPA hook does not call `adInit()` / create GPT slots. + let slots_json: Vec = if ad_stack_enabled { + matched_slots + .iter() + .map(|slot| { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| serde_json::json!([f.width, f.height])) + .collect(); + let targeting: serde_json::Map = slot + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + serde_json::json!({ + "id": slot.id, + "gam_unit_path": gam_path, + "div_id": div_id, + "formats": formats, + "targeting": targeting, + }) }) - }) - .collect(); + .collect() + } else { + Vec::new() + }; let body = serde_json::json!({ "slots": slots_json, @@ -2254,6 +2260,67 @@ mod tests { ); } + #[tokio::test] + async fn handle_publisher_request_does_not_self_generate_ec() { + // EC generation is the adapter's real-browser-gated responsibility. This + // handler must never mint an EC ID on its own: for a navigation from a + // client the adapter did not pre-generate for (e.g. a non-real browser), + // `ec_value` must stay `None` so no IP-derived identifier reaches the + // auction. Consent allows EC creation and a client IP is present here — + // exactly the conditions under which the old inline call would have + // generated one. + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + assert!( + ec_context.ec_allowed(), + "test precondition: consent must allow EC creation" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/article") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = handle_publisher_request( + &settings, + ®istry, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + ec_context.ec_value(), + None, + "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + ); + } + #[test] fn test_content_type_detection() { let test_cases = vec![ @@ -3923,6 +3990,34 @@ mod tests { serde_json::from_slice(&response.into_body().into_bytes()).expect("should be json") } + /// `run_page_bids` with an EC context whose jurisdiction allows the + /// server-side auction, so slot-counting tests isolate the variable + /// under test (bot/prefetch) from the consent gate. The default + /// request resolves to `Jurisdiction::Unknown`, which fails the + /// consent gate and now suppresses slots. + async fn run_page_bids_consent_allowed( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> serde_json::Value { + let ec_context = consent_allowing_ec_context(); + let response = + run_page_bids_response_with_ec(settings, orchestrator, slots, &ec_context, req) + .await; + serde_json::from_slice(&response.into_body().into_bytes()).expect("should be json") + } + + /// Builds an [`EcContext`] whose consent context permits the server-side + /// auction (known non-GDPR jurisdiction, no EU TCF signal). + fn consent_allowing_ec_context() -> EcContext { + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + EcContext::new_for_test(None, consent) + } + fn article_slot() -> Vec { vec![CreativeOpportunitySlot { id: "atf".to_string(), @@ -3968,10 +4063,20 @@ mod tests { slots: &[CreativeOpportunitySlot], req: Request, ) -> Response { - let services = noop_services(); let fastly_req = crate::compat::to_fastly_request_ref(&req); let ec_context = EcContext::read_from_request(settings, &fastly_req) .expect("should read EC context"); + run_page_bids_response_with_ec(settings, orchestrator, slots, &ec_context, req).await + } + + async fn run_page_bids_response_with_ec( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + ec_context: &EcContext, + req: Request, + ) -> Response { + let services = noop_services(); handle_page_bids( settings, &services, @@ -3981,7 +4086,7 @@ mod tests { slots, registry: None, }, - &ec_context, + ec_context, req, ) .await @@ -4102,7 +4207,7 @@ mod tests { "Mozilla/5.0 (compatible; Googlebot/2.1)", ); - let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -4132,7 +4237,7 @@ mod tests { let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-purpose", "prefetch"); - let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] @@ -4210,24 +4315,27 @@ mod tests { } #[tokio::test] - async fn disabled_auction_returns_slots_but_no_bids() { - // [auction].enabled = false is a global kill switch: slot definitions - // are still returned (HTML structure unchanged) but no server-side - // auction may be dispatched. + async fn disabled_auction_returns_no_slots_or_bids() { + // [auction].enabled = false is a global kill switch: it must disable + // the entire server-side ad stack, not just SSP calls. Returning slot + // definitions would let the SPA hook assign `ts.adSlots` and call + // `adInit()`, creating/refreshing GPT slots client-side even though + // the auction is off. Consent is allowed here so the test isolates + // the kill switch. let settings = settings_with_co_auction_disabled(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); - let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; assert_eq!( body["slots"] .as_array() .expect("slots should be array") .len(), - 1, - "disabled auction should still return slot definitions" + 0, + "disabled auction must not return slot definitions (kill switch stops the ad stack)" ); assert_eq!( body["bids"] @@ -4238,5 +4346,38 @@ mod tests { "disabled auction must not produce bids" ); } + + #[tokio::test] + async fn consent_denied_returns_no_slots_or_bids() { + // When consent denies the server-side auction (here: Jurisdiction + // Unknown fails closed), the endpoint must return no slots so the SPA + // hook does not create GPT slots client-side — matching the publisher + // navigation path's `should_run_server_side_ad_stack` gate. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + // run_page_bids uses the default EC context, which resolves to + // Jurisdiction::Unknown (consent denied). + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "consent denial must suppress slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "consent denial must produce no bids" + ); + } } } From a3160d5a33a4f2a473888e766dfb41871c130cc5 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 19 Jun 2026 23:29:03 +0530 Subject: [PATCH 116/195] Close cache-privacy and refresh-recovery gaps from PR review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply request-filter response effects before the final Set-Cookie cache guard in every Fastly response path (buffered, streaming, asset streaming) so a per-user cookie added by a DataDome allow can no longer leave with public/surrogate cache headers. Strip surrogate cache headers on every Set-Cookie response — even one keeping a stricter no-store directive — and treat no-store as protected in the operator-header guard. Reject OPTIONS /__ts/page-bids at the adapter so the side-effecting endpoint never grants a CORS preflight the publisher origin might. Drain every dispatched SSP request in the collect loop instead of breaking on the auction deadline, so a slow origin can no longer discard SSP responses that already arrived. Reject empty/whitespace div_id overrides at runtime validation, which would otherwise bind a slot to the first id-bearing DOM element. Recover Prebid refresh params and client-side bids from candidate codes ([gpt element id, injected div_id]) so container-backed slots keep the publisher's configured demand on refresh/scroll auctions. --- .../js/lib/src/integrations/prebid/index.ts | 52 +++-- .../test/integrations/prebid/index.test.ts | 68 +++++++ .../trusted-server-adapter-fastly/src/main.rs | 100 +++++++--- .../src/route_tests.rs | 185 +++++++++++++++++- .../src/auction/orchestrator.rs | 18 +- .../src/creative_opportunities.rs | 41 ++++ 6 files changed, 409 insertions(+), 55 deletions(-) diff --git a/crates/js/lib/src/integrations/prebid/index.ts b/crates/js/lib/src/integrations/prebid/index.ts index 835d28fdb..fe175b44d 100644 --- a/crates/js/lib/src/integrations/prebid/index.ts +++ b/crates/js/lib/src/integrations/prebid/index.ts @@ -342,6 +342,27 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } +/** + * Find the publisher's original `pbjs.adUnits` entry for a refreshing slot. + * + * A TS-owned GPT slot may be defined on `${div_id}-container`, so the GPT + * element id used as the synthetic refresh ad unit code can differ from the + * inner `div_id` the publisher keyed their Prebid ad unit by. Try each candidate + * code in order and return the first matching ad unit, so container-backed slots + * still recover the publisher's configured params and bidders. + */ +function findRefreshAdUnit( + candidateCodes: Array +): TrustedServerAdUnit | undefined { + const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; + for (const code of candidateCodes) { + if (!code) continue; + const match = adUnits.find((unit) => unit.code === code); + if (match) return match; + } + return undefined; +} + /** * Collect the configured client-side bidder entries for a refreshing slot. * @@ -350,17 +371,16 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { * already present on the ad unit, so without re-attaching them here publishers * that split demand between server-side and native Prebid adapters would lose * all client-side demand on refresh/scroll impressions. Bids are sourced from - * the matching `pbjs.adUnits` entry (by ad unit code) so the publisher's - * configured params are preserved. + * the matching `pbjs.adUnits` entry (by candidate ad unit code) so the + * publisher's configured params are preserved. */ function clientSideBidsForRefresh( - code: string + candidateCodes: Array ): Array<{ bidder: string; params: Record }> { const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); if (clientSideBidders.size === 0) return []; - const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; - const match = adUnits.find((unit) => unit.code === code); + const match = findRefreshAdUnit(candidateCodes); if (!match?.bids) return []; const bids: Array<{ bidder: string; params: Record }> = []; @@ -379,15 +399,16 @@ function clientSideBidsForRefresh( * `requestBids` shim has no original server-side bidder entries to collect into * `bidderParams` — without this, refresh/scroll `/auction` requests send `{}` * and lose demand the publisher configured only on the initial ad unit. Source - * the params from the matching `pbjs.adUnits` entry by code, covering both - * states the initial auction can leave that entry in: + * the params from the matching `pbjs.adUnits` entry by candidate code, covering + * both states the initial auction can leave that entry in: * - raw server-side bidder entries (`{ bidder, params }`) not yet folded, and * - params already folded into that unit's `trustedServer` bid `bidderParams` * by a prior `requestBids` call. */ -function serverSideBidderParamsForRefresh(code: string): Record> { - const adUnits = (pbjs.adUnits ?? []) as TrustedServerAdUnit[]; - const match = adUnits.find((unit) => unit.code === code); +function serverSideBidderParamsForRefresh( + candidateCodes: Array +): Record> { + const match = findRefreshAdUnit(candidateCodes); if (!match?.bids) return {}; const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); @@ -737,17 +758,24 @@ export function installRefreshHandler(timeoutMs = 1500): void { }; const code = refreshSlotElementId(slot) ?? 'refresh-slot'; + // A TS-owned slot may be defined on `${div_id}-container`, so the GPT + // element id used as the synthetic refresh code can differ from the + // inner `div_id` the publisher keyed their ad unit by. Recover from both. + const candidateCodes = [code, injectedSlot?.div_id]; const tsParams: Record = zone ? { [ZONE_KEY]: zone } : {}; // Carry the publisher's inline server-side (PBS) bidder params captured // on the initial ad unit so refresh/scroll auctions don't drop them. - const serverSideParams = serverSideBidderParamsForRefresh(code); + const serverSideParams = serverSideBidderParamsForRefresh(candidateCodes); if (Object.keys(serverSideParams).length > 0) { tsParams[BIDDER_PARAMS_KEY] = serverSideParams; } return { code, mediaTypes: { banner }, - bids: [{ bidder: ADAPTER_CODE, params: tsParams }, ...clientSideBidsForRefresh(code)], + bids: [ + { bidder: ADAPTER_CODE, params: tsParams }, + ...clientSideBidsForRefresh(candidateCodes), + ], }; }); diff --git a/crates/js/lib/test/integrations/prebid/index.test.ts b/crates/js/lib/test/integrations/prebid/index.test.ts index 5edad541f..fd7703546 100644 --- a/crates/js/lib/test/integrations/prebid/index.test.ts +++ b/crates/js/lib/test/integrations/prebid/index.test.ts @@ -981,6 +981,74 @@ describe('prebid/installRefreshHandler', () => { mockPbjs.adUnits = []; }); + it('recovers params and client-side bids for container-backed slots by injected div_id', () => { + // A TS-owned GPT slot may be defined on `${div_id}-container`, but the + // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic + // refresh code stays the GPT element id (so GPT can match it), while params + // and client-side bids are recovered from the injected div_id candidate. + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + mockPbjs.adUnits = [ + { + code: 'div-ad-x', + bids: [ + { bidder: 'appnexus', params: { placementId: 12345 } }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-x-container'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'x_ad', + gam_unit_path: '/123/x', + div_id: 'div-ad-x', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + // Synthetic refresh code stays the GPT element id, not the div_id. + code: 'div-ad-x-container', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }), + ], + }) + ); + + delete (window as any).__tsjs_prebid; + mockPbjs.adUnits = []; + }); + it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { // After the initial auction, the requestBids shim has folded the publisher's // server-side params into the original ad unit's trustedServer bid. A later diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 943ff94cd..2ef07eada 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -253,10 +253,13 @@ fn main() { &mut fastly_resp, ); } - // EC finalization may have just added the identity Set-Cookie, which - // the HttpResponse-stage cache guard could not see. - enforce_set_cookie_cache_privacy(&mut fastly_resp); + // Apply request-filter response effects (e.g. a DataDome allow + // Set-Cookie) before the final cache guard so any per-user cookie + // they add is covered. EC finalization above may also have added the + // identity Set-Cookie, which the HttpResponse-stage guard could not + // see — the guard runs last so it observes both. request_filter_effects.apply_to_fastly_response(&mut fastly_resp); + enforce_set_cookie_cache_privacy(&mut fastly_resp); fastly_resp.send_to_client(); if is_real_browser { @@ -284,10 +287,13 @@ fn main() { &mut fastly_resp, ); } - // EC finalization may have just added the identity Set-Cookie, which - // the HttpResponse-stage cache guard could not see. - enforce_set_cookie_cache_privacy(&mut fastly_resp); + // Apply request-filter response effects (e.g. a DataDome allow + // Set-Cookie) before the final cache guard so any per-user cookie + // they add is covered. EC finalization above may also have added the + // identity Set-Cookie, which the HttpResponse-stage guard could not + // see — the guard runs last so it observes both. request_filter_effects.apply_to_fastly_response(&mut fastly_resp); + enforce_set_cookie_cache_privacy(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); let mut stream_succeeded = false; match futures::executor::block_on(stream_publisher_body_async( @@ -324,7 +330,11 @@ fn main() { finalize_response(&settings, geo_info.as_ref(), &mut response); asset_cache_policy.apply_after_route_finalization(&mut response); let mut fastly_resp = compat::to_fastly_response_skeleton(response); + // A request filter (e.g. DataDome allow) can append a per-user + // Set-Cookie via response effects even on an otherwise cacheable + // asset, so guard against shared caching after applying them. request_filter_effects.apply_to_fastly_response(&mut fastly_resp); + enforce_set_cookie_cache_privacy(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); if let Err(e) = futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) @@ -633,6 +643,22 @@ async fn route_request( false, ), + // Reject CORS preflight for the side-effecting page-bids endpoint at the + // adapter. The GET handler's legacy fallback trusts `X-TSJS-Page-Bids` + // precisely because this endpoint never grants a preflight; letting + // OPTIONS fall through to the publisher origin (which may return + // permissive CORS) would defeat that, allowing a cross-site page to + // trigger real PBS/APS auctions from a visitor's browser. + (Method::OPTIONS, "/__ts/page-bids") => { + let mut response = HttpResponse::new(EdgeBody::from("Forbidden")); + *response.status_mut() = edgezero_core::http::StatusCode::FORBIDDEN; + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + (Ok(response), false) + } + // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path (Method::GET, "/__ts/page-bids") => ( handle_page_bids( @@ -870,34 +896,41 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: // stricter directive (e.g. `no-store`). // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if !already_uncacheable && response.headers().contains_key(header::SET_COOKIE) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); + if response.headers().contains_key(header::SET_COOKIE) { + // Surrogate cache headers must come off every cookie-bearing response, + // even one already carrying a stricter `no-store`/`private` directive — + // they are independent of Cache-Control and would otherwise let a shared + // cache store and replay one visitor's Set-Cookie. response.headers_mut().remove("surrogate-control"); response.headers_mut().remove("fastly-surrogate-control"); + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } } // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) - // carry a private Cache-Control directive. Operator headers must not - // re-enable shared caching for them — neither by replacing Cache-Control nor - // by reintroducing the surrogate cache headers the privacy paths stripped. - let response_is_private = response + // carry an uncacheable Cache-Control directive (`private` or `no-store`). + // Operator headers must not re-enable shared caching for them — neither by + // replacing Cache-Control nor by reintroducing the surrogate cache headers + // the privacy paths stripped. + let response_is_uncacheable = response .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()) .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private")); + .is_some_and(|v| v.contains("private") || v.contains("no-store")); for (key, value) in &settings.response_headers { - if response_is_private + if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) || key.eq_ignore_ascii_case("surrogate-control") || key.eq_ignore_ascii_case("fastly-surrogate-control")) @@ -922,19 +955,26 @@ fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: /// inherited from the origin or operator response headers — a shared cache must /// not be able to store and replay one visitor's EC cookie to others. /// -/// Idempotent: a response already marked `private`/`no-store` is left untouched -/// so a stricter directive is never weakened. +/// Idempotent: a response already marked `private`/`no-store` keeps its stricter +/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a +/// `no-store` cookie response can never retain shared Fastly cacheability. fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { + if response.get_header("set-cookie").is_none() { + return; + } + // Strip surrogate cache headers on every cookie-bearing response, even when + // keeping a stricter `no-store`/`private` directive — Surrogate-Control is + // independent of Cache-Control and would otherwise let a shared cache store + // and replay one visitor's Set-Cookie. + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); let already_uncacheable = response .get_header_str("cache-control") .map(str::to_ascii_lowercase) .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if already_uncacheable || response.get_header("set-cookie").is_none() { - return; + if !already_uncacheable { + response.set_header("cache-control", "private, max-age=0"); } - response.set_header("cache-control", "private, max-age=0"); - response.remove_header("surrogate-control"); - response.remove_header("fastly-surrogate-control"); } fn http_error_response(report: &Report) -> HttpResponse { diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index 11a32c8c0..ebdec221a 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -638,8 +638,11 @@ fn route_result_to_fastly_response( &mut fastly_response, ); } - super::enforce_set_cookie_cache_privacy(&mut fastly_response); + // Mirror main's ordering: apply request-filter response effects (which may + // append a per-user Set-Cookie) before the final cache guard so the guard + // observes them. request_filter_effects.apply_to_fastly_response(&mut fastly_response); + super::enforce_set_cookie_cache_privacy(&mut fastly_response); fastly_response } @@ -1534,6 +1537,134 @@ fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { ); } +#[test] +fn enforce_set_cookie_cache_privacy_strips_surrogate_on_no_store() { + // A `no-store` cookie response keeps its stricter Cache-Control but must still + // lose the surrogate cache headers — they are independent of Cache-Control and + // would otherwise let a shared cache store and replay the visitor's cookie. + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header("surrogate-control", "max-age=86400") + .header("fastly-surrogate-control", "max-age=86400") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("no-store"), + "should keep the stricter no-store directive" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "no-store cookie responses must not retain Surrogate-Control" + ); + assert!( + fastly_response + .get_header("fastly-surrogate-control") + .is_none(), + "no-store cookie responses must not retain Fastly-Surrogate-Control" + ); +} + +#[test] +fn request_filter_set_cookie_after_guard_still_downgrades_cache() { + // A request filter (e.g. a DataDome allow) can append a per-user Set-Cookie via + // response effects. main applies those effects before the final cache guard, so + // an origin response still marked `public` with surrogate headers must be + // downgraded once the filter cookie is present. + use trusted_server_core::integrations::{HeaderMutation, RequestFilterEffects}; + + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + let effects = RequestFilterEffects { + request_headers: vec![], + response_headers: vec![HeaderMutation::append( + "set-cookie", + "datadome=allow; Path=/; HttpOnly", + )], + }; + + // Mirror main's ordering: apply effects first, then the guard. + effects.apply_to_fastly_response(&mut fastly_response); + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("private, max-age=0"), + "a filter-added Set-Cookie must downgrade a public origin response" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "a filter-added Set-Cookie must strip surrogate cacheability" + ); +} + +#[test] +fn finalize_response_no_store_cookie_blocks_operator_surrogate_reenable() { + // Operator response_headers must not re-add surrogate caching to a Set-Cookie + // response carrying the stricter `no-store` directive — the operator guard must + // treat no-store as protected, not just `private`. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + settings.response_headers.insert( + "Fastly-Surrogate-Control".to_string(), + "max-age=86400".to_string(), + ); + settings.response_headers.insert( + header::CACHE_CONTROL.as_str().to_string(), + "public, max-age=3600".to_string(), + ); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "operator Cache-Control must not weaken the stricter no-store directive" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable caching for a no-store cookie response" + ); + assert_eq!( + response + .headers() + .get("fastly-surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Fastly-Surrogate-Control must not re-enable caching for a no-store cookie response" + ); +} + #[test] fn finalize_response_leaves_stricter_no_store_untouched() { let settings = create_test_settings(); @@ -1757,6 +1888,58 @@ fn page_bids_cross_site_request_is_rejected_at_the_route() { ); } +#[test] +fn page_bids_options_preflight_is_rejected_at_the_route() { + // OPTIONS must not fall through to the publisher origin (which may return + // permissive CORS); the GET handler's legacy `X-TSJS-Page-Bids` fallback + // relies on this endpoint never granting a preflight. + let base = base_route_settings_toml(); + let prebid = prebid_integration_toml(); + let config = format!( + r#"{base} + +{prebid} + + [auction] + enabled = true + providers = ["prebid"] + timeout_ms = 2000 + + [creative_opportunities] + gam_network_id = "1234" + "#, + ); + let settings = + Settings::from_toml(&config).expect("should parse page-bids route test settings"); + let (orchestrator, integration_registry) = build_route_stack(&settings); + + let req = Request::new( + Method::OPTIONS, + "https://test-publisher.com/__ts/page-bids?path=/2024/article/", + ); + let services = test_runtime_services(&req); + + let resp = route_buffered_response( + &settings, + &orchestrator, + &integration_registry, + &services, + req, + "should route page-bids preflight request", + ); + + assert_eq!( + resp.get_status(), + StatusCode::FORBIDDEN, + "should reject the page-bids CORS preflight at the adapter" + ); + assert_eq!( + resp.get_header_str(header::CACHE_CONTROL), + Some("private, no-store"), + "preflight rejection must not be shared-cached" + ); +} + #[test] fn s3_asset_origin_error_stays_uncacheable_after_global_headers() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 48aebaa97..c654d39ee 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -936,18 +936,12 @@ impl AuctionOrchestrator { } } - // Defense-in-depth deadline guard, mirroring run_providers_parallel. - // Dispatch already caps each backend's first_byte_timeout at the - // remaining auction budget, so this should not fire in practice — - // it protects against the two paths drifting apart. - if remaining_budget_ms(auction_start, timeout_ms) == 0 && !remaining.is_empty() { - log::warn!( - "Auction timeout ({}ms) reached during collection, dropping {} remaining request(s)", - timeout_ms, - remaining.len() - ); - break; - } + // Drain every dispatched request. Each backend was capped with a + // first-byte timeout at dispatch time, so by the collect phase the + // remaining handles may already be ready even if wall-clock time + // elapsed while the origin was slow — dropping them here would + // discard SSP responses that already arrived. The mediator launch + // below still observes A_deadline via `remaining_budget_ms`. } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 2b3fa6e72..c55d98bc0 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -143,6 +143,21 @@ impl CreativeOpportunitySlot { format.validate_runtime(&self.id)?; } + // An explicit empty/whitespace `div_id` override is rejected: the + // injected JS resolves slots with `candidate.id.startsWith(slot.div_id)`, + // and every element id starts with the empty string, so an empty override + // would bind the slot to the first id-bearing element in the document. + if self + .div_id + .as_deref() + .is_some_and(|div_id| div_id.trim().is_empty()) + { + return Err(format!( + "slot `{}` div_id override must not be empty", + self.id + )); + } + if self .resolved_gam_unit_path(gam_network_id) .trim() @@ -509,6 +524,32 @@ mod tests { assert_eq!(slot.resolved_div_id(), "atf"); } + #[test] + fn validate_runtime_rejects_empty_div_id_override() { + // An empty/whitespace div_id would resolve every slot to the first + // id-bearing element via `candidate.id.startsWith(slot.div_id)`. + let mut slot = make_slot("atf", vec!["/"]); + slot.compile_patterns(); + + slot.div_id = Some(String::new()); + assert!( + slot.validate_runtime("1234").is_err(), + "empty div_id override should fail validation" + ); + + slot.div_id = Some(" ".to_string()); + assert!( + slot.validate_runtime("1234").is_err(), + "whitespace-only div_id override should fail validation" + ); + + slot.div_id = Some("div-ad-x".to_string()); + assert!( + slot.validate_runtime("1234").is_ok(), + "a concrete div_id override should pass validation" + ); + } + #[test] fn to_ad_slot_wires_aps_params_into_bidders() { let mut slot = make_slot("atf", vec!["/"]); From bf76d41125d9bb8e09f26892ad70b993ea120293 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 20 Jun 2026 19:28:22 +0530 Subject: [PATCH 117/195] Close EID-consent and GPT initial-load gaps from PR review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate /auction client EID resolution on the same identity-consent condition as the EC ID (`ec_id.is_some()`, already filtered by `ec_allowed()`). Previously client-provided EIDs from the request body or ts-eids cookie were resolved unconditionally, so a US/GPC or US-Privacy opt-out context — where EC identity use is denied but a non-personalized auction may still run — could forward persistent EIDs, since `gate_eids_by_consent` only strips on TCF/GDPR signals. This matches the publisher and /__ts/page-bids paths. Refresh TS-defined GPT slots when the publisher disabled initial load. With pubads().disableInitialLoad(), display() only registers a freshly defined slot and the ad request must come from refresh(); TS-owned first-impression slots were only display()ed, so they rendered blank. A wrapper around disableInitialLoad() records the state on window.tsjs, and adInit() refreshes its own slots when it is set (bundle and gpt_bootstrap.js). The detector only hooks an existing googletag stub so a plain import never touches window.googletag. --- crates/js/lib/src/core/types.ts | 8 + crates/js/lib/src/integrations/gpt/index.ts | 67 +++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 56 +++++++ .../src/auction/endpoints.rs | 148 +++++++++++++++++- .../src/integrations/gpt.rs | 29 ++++ .../src/integrations/gpt_bootstrap.js | 35 ++++- 6 files changed, 330 insertions(+), 13 deletions(-) diff --git a/crates/js/lib/src/core/types.ts b/crates/js/lib/src/core/types.ts index 70d40e2b6..ec2882efb 100644 --- a/crates/js/lib/src/core/types.ts +++ b/crates/js/lib/src/core/types.ts @@ -113,6 +113,14 @@ export interface TsjsApi { * client-side auction that would clear the just-applied TS targeting. */ adInitRefreshInProgress?: boolean; + /** + * True once the publisher has called `googletag.pubads().disableInitialLoad()`. + * GPT exposes no getter for this state, so it is tracked by wrapping the + * setter. When set, `display()` only registers a slot and the ad request must + * come from a `refresh()`; adInit() uses this to refresh its own freshly + * defined slots so they are not left blank. + */ + gptInitialLoadDisabled?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index 2effbc593..aad98c96c 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -106,6 +106,7 @@ interface GoogleTagPubAdsService { addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; refresh(slots?: GoogleTagSlot[]): void; getSlots?(): GoogleTagSlot[]; + disableInitialLoad?(): void; } interface GoogleTag { @@ -371,8 +372,48 @@ function queueWinBillingBeacon(url: string): boolean { * Idempotent: destroys previously created TS-managed slots before redefining them, * so it is safe to call again after SPA navigation updates `tsjs.adSlots`/`tsjs.bids`. */ +/** + * Track whether the publisher disabled GPT initial load. + * + * GPT exposes no getter for the initial-load-disabled flag, so wrap + * `pubads().disableInitialLoad()` to record it on `window.tsjs`. With initial + * load disabled, `display()` only registers a slot — the ad request must come + * from a later `refresh()`. adInit() reads this to refresh its own freshly + * defined slots so they are not left blank. + * + * Installed via the command queue so it runs before the publisher's own + * `disableInitialLoad()` call (the TS core script is injected ahead of the + * publisher's GPT setup). Idempotent per pubads service. + * + * Only hooks an existing `googletag` stub — it never creates one. A plain module + * import that does not activate the GPT integration must not touch + * `window.googletag`. When the GPT shim is active it creates the stub before + * `installTsAdInit` runs, so the detector is still queued ahead of the + * publisher's GPT setup. + */ +function installInitialLoadDetector(ts: TsjsApi): void { + const win = window as GptWindow; + const cmd = win.googletag?.cmd; + if (!cmd) return; + cmd.push(() => { + const pubads = win.googletag?.pubads?.(); + if (!pubads) return; + const service = pubads as GoogleTagPubAdsService & { __tsInitialLoadHooked?: boolean }; + if (typeof service.disableInitialLoad !== 'function' || service.__tsInitialLoadHooked) { + return; + } + const original = service.disableInitialLoad.bind(service); + service.disableInitialLoad = function () { + ts.gptInitialLoadDisabled = true; + return original(); + }; + service.__tsInitialLoadHooked = true; + }); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); + installInitialLoadDetector(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; // Snapshot bids at adInit() call time — correct for targeting setup. @@ -524,16 +565,28 @@ export function installTsAdInit(): void { // enabled, so this runs unconditionally for any newly-defined slots. slotsToDisplay.forEach((divId) => g.display?.(divId)); - if (slotsToRefresh.length > 0) { + // Slots needing an explicit ad request via refresh(). Reused + // publisher-owned slots always need one to pick up the just-applied + // server-side targeting. TS-defined slots are normally fetched by the + // display() above — but when the publisher called + // pubads().disableInitialLoad(), display() only registers the slot and the + // ad request must come from refresh(). Without this, a TS-owned + // first-impression slot renders blank on initial-load-disabled pages. Only + // add them in that case; otherwise display() + refresh() would + // double-request the impression. + const slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; + + if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied - // server-side targeting to GAM for reused publisher-owned slots. If - // slim-Prebid has wrapped refresh(), it must pass this call straight - // through — not clear the targeting and run a duplicate client-side - // auction. Later publisher-initiated refreshes of the same slots still - // go through the wrapper normally. + // server-side targeting to GAM. If slim-Prebid has wrapped refresh(), it + // must pass this call straight through — not clear the targeting and run + // a duplicate client-side auction. Later publisher-initiated refreshes of + // the same slots still go through the wrapper normally. ts.adInitRefreshInProgress = true; try { - g.pubads!().refresh(slotsToRefresh); + g.pubads!().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; } diff --git a/crates/js/lib/test/integrations/gpt/ad_init.test.ts b/crates/js/lib/test/integrations/gpt/ad_init.test.ts index 43551644a..d649778bc 100644 --- a/crates/js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/js/lib/test/integrations/gpt/ad_init.test.ts @@ -187,6 +187,62 @@ describe('installTsAdInit', () => { expect(mockPubads.refresh).not.toHaveBeenCalled(); }); + it('refreshes TS-defined slots when the publisher disabled GPT initial load', async () => { + // With pubads().disableInitialLoad(), display() only registers a freshly + // defined slot — the ad request must come from refresh(). A TS-owned slot + // must therefore be refreshed too, or it renders blank. + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + // Publisher has not defined this slot, so TS defines (owns) it. + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + disableInitialLoad: vi.fn(), + }; + const displayMock = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + display: displayMock, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + + // Publisher disables initial load — goes through the wrapper the detector + // installed, recording the state on window.tsjs. + mockPubads.disableInitialLoad(); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); + + (window as TestWindow).tsjs!.adInit!(); + + // The slot is still registered via display(), and additionally refreshed so + // it actually requests an ad under disableInitialLoad(). + expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + }); + it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 5ed59aae5..42e9d3939 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -207,10 +207,23 @@ pub async fn handle_auction( // current request does not include them, fall back to the persisted // `ts-eids` cookie so later requests can still forward the browser's // full OpenRTB-style EID structure. - let client_eids = resolve_client_auction_eids( - body.eids.as_ref(), - extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), - ); + // + // Gate this on the same identity-consent condition as the EC ID + // (`ec_id.is_some()`, which is already filtered by `ec_context.ec_allowed()`). + // Otherwise a US/GPC or US-Privacy opt-out context — where EC identity use is + // denied but a non-personalized auction may still run — could forward + // persistent client EIDs from the body/cookie, since `gate_eids_by_consent` + // only strips on TCF/GDPR signals. This matches the publisher and + // `/__ts/page-bids` paths, which also resolve client EIDs only when + // `ec_id.is_some()`. + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids( + body.eids.as_ref(), + extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), + ) + } else { + None + }; // Resolve partner EIDs from the KV identity graph when the user has // a valid EC and both KV and partner stores are available. @@ -609,6 +622,133 @@ mod tests { ); } + /// Provider that records whether the auction request it received carried + /// EIDs, then fails its launch so no real transport handle is needed. + struct EidCapturingProvider { + had_eids: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for EidCapturingProvider { + fn provider_name(&self) -> &'static str { + "eid_capturing_provider" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + *self.had_eids.lock().expect("should lock captured eids") = + Some(request.user.eids.is_some()); + Err(Report::new(TrustedServerError::Auction { + message: "capture only".to_string(), + })) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run when the launch fails"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _timeout_ms: u32) -> Option { + Some("capture-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_strips_client_eids_when_ec_identity_denied() { + // US-state opt-out via GPC: the server-side auction consent gate still + // allows a non-personalized auction, but EC identity use is denied + // (`ec_allowed()` is false) and `gate_eids_by_consent` does not strip + // because no TCF signal is present and GDPR does not apply. Client EIDs + // supplied in the request body/cookie must NOT be forwarded — the + // outgoing auction request must have `user.eids == None`. + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: true, + providers: vec!["eid_capturing_provider".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + let had_eids = Arc::new(std::sync::Mutex::new(None)); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::clone(&had_eids), + })); + let services = noop_services(); + + // US-state jurisdiction with an explicit GPC opt-out: auction allowed, + // EC identity denied. + let ec_context = EcContext::new_for_test( + None, + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..ConsentContext::default() + }, + ); + + // Persistent EIDs supplied in both the request body and the ts-eids cookie. + let cookie_payload = json!([ + { + "source": "sharedid.org", + "uids": [{ "id": "cookie_uid", "atype": 3 }] + } + ]); + let encoded_cookie = BASE64 + .encode(serde_json::to_vec(&cookie_payload).expect("should serialize cookie payload")); + let body = json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ], + "eids": [ + { + "source": "id5-sync.com", + "uids": [{ "id": "body_uid", "atype": 1 }] + } + ] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .header("cookie", format!("{COOKIE_TS_EIDS}={encoded_cookie}")) + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + // The capturing provider fails its launch, so the auction errors overall; + // the assertion is on the EIDs observed by the provider, not the result. + let _ = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await; + + assert_eq!( + *had_eids.lock().expect("should lock captured eids"), + Some(false), + "outgoing auction request must carry no EIDs when EC identity is denied" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_kv() { let registry = PartnerRegistry::empty(); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 118527971..e24a138d4 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1231,6 +1231,35 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_refreshes_ts_slots_when_initial_load_disabled() { + // Mirrors the bundle: when the publisher calls disableInitialLoad(), + // display() only registers a TS-defined slot, so the bootstrap must also + // refresh those slots or they render blank. + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("disableInitialLoad"), + "bootstrap should wrap disableInitialLoad() to detect the disabled state" + ); + assert!( + combined.contains("gptInitialLoadDisabled"), + "bootstrap should record the initial-load-disabled state on window.tsjs" + ); + assert!( + combined.contains("slotsNeedingRefresh"), + "bootstrap should refresh TS-defined slots when initial load is disabled" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 46cfe0fd3..f1bd9d833 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -19,6 +19,29 @@ var ts = (window.tsjs = window.tsjs || {}); if (ts.adInit) return; + // Track whether the publisher disabled GPT initial load. GPT exposes no + // getter for this, so wrap pubads().disableInitialLoad() to record it. With + // initial load disabled, display() only registers a slot and the ad request + // must come from a later refresh(); adInit() reads this to refresh its own + // freshly defined slots so they are not left blank. Pushed onto the command + // queue so it runs before the publisher's own disableInitialLoad() call. + (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { + var pubads = googletag.pubads && googletag.pubads(); + if ( + !pubads || + typeof pubads.disableInitialLoad !== "function" || + pubads.__tsInitialLoadHooked + ) { + return; + } + var original = pubads.disableInitialLoad.bind(pubads); + pubads.disableInitialLoad = function () { + ts.gptInitialLoadDisabled = true; + return original(); + }; + pubads.__tsInitialLoadHooked = true; + }); + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; @@ -120,7 +143,15 @@ slotsToDisplay.forEach(function (divId) { googletag.display(divId); }); - if (slotsToRefresh.length > 0) { + // Reused publisher-owned slots always need a refresh to pick up the + // server-side targeting. TS-defined slots are fetched by display() above + // unless the publisher disabled initial load, in which case display() only + // registers them and refresh() must request the ad — otherwise they render + // blank. Only add them in that case to avoid double-requesting. + var slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; + if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied // server-side targeting to GAM. If slim-Prebid has already wrapped // refresh(), it must pass this call straight through — not clear the @@ -128,7 +159,7 @@ // bundle's adInit() in crates/js/lib/src/integrations/gpt/index.ts. ts.adInitRefreshInProgress = true; try { - googletag.pubads().refresh(slotsToRefresh); + googletag.pubads().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; } From d2f538b0f3efe7daeca8facb07aaad8892b27aac Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 22 Jun 2026 16:32:51 +0530 Subject: [PATCH 118/195] Close build/runtime validation parity and observability gaps from PR review Address PR #680 review findings: Blocking build/runtime parity: - Remove the dead glob stub in build.rs so creative-slot page-pattern validation runs against the real glob crate. An invalid pattern such as `["["]` now fails the build instead of being embedded and dropped at runtime settings load. - Reject an empty/whitespace div_id override at build time, mirroring CreativeOpportunitySlot::validate_runtime. - Validate nested creative-slot fields (formats, providers, aps, prebid) against the runtime structs' deny_unknown_fields so env-injected typos like `mediatype` or `slotId` fail the build, not runtime. Observability and correctness: - Mirror the parallel auction path on the dispatch/collect path: attribute provider parse failures (error_type + message) and transport failures (via failed_backend_name) in provider_details. - Warn on each page pattern dropped during compile_patterns so a mixed valid/invalid set is visible to operators. - Escape the terminator in the configured slim_prebid_url so it cannot break out of its inline script tag. - Guard SPA navigation: onNavigate no-ops when the path is unchanged, so popstate (hash-only or same-path back/forward) no longer re-requests impressions. Docs and comments: - Update the GPT scroll/refresh handoff comment to reflect installSpaAuctionHook + /__ts/page-bids ownership of SPA navigation. - Note that targeting.zone is not forwarded when explicit prebid.bidders are set. - Split the page-bids same-origin-gate and path-normalization docs onto their own functions; remove the stale # Panics section on handle_publisher_request. - Correct the stale slotRenderEnded/beacon comment in gpt_bootstrap.js. Tests added for div_id, nested-field, slim_prebid_url escaping, and SPA same-path guard behavior. --- crates/js/lib/src/integrations/gpt/index.ts | 13 +- .../test/integrations/gpt/spa_hook.test.ts | 50 ++++- crates/trusted-server-core/build.rs | 18 +- .../src/auction/orchestrator.rs | 48 ++++- .../src/creative_opportunities.rs | 24 ++- .../src/creative_slot_build_check.rs | 186 +++++++++++++++++- .../src/integrations/gpt.rs | 56 +++++- .../src/integrations/gpt_bootstrap.js | 7 +- crates/trusted-server-core/src/publisher.rs | 16 +- 9 files changed, 369 insertions(+), 49 deletions(-) diff --git a/crates/js/lib/src/integrations/gpt/index.ts b/crates/js/lib/src/integrations/gpt/index.ts index aad98c96c..30071220d 100644 --- a/crates/js/lib/src/integrations/gpt/index.ts +++ b/crates/js/lib/src/integrations/gpt/index.ts @@ -661,8 +661,15 @@ export function installSpaAuctionHook(): void { ts.spaHookInstalled = true; let inflight: AbortController | null = null; + // Last path an auction was run for. popstate fires for hash-only and + // same-pathname back/forward (scroll restoration), and pushState/replaceState + // can be called with the current URL, so guard every entry point against + // re-requesting impressions for a path we already loaded. + let currentPath = location.pathname; async function onNavigate(path: string): Promise { + if (path === currentPath) return; + currentPath = path; inflight?.abort(); const controller = new AbortController(); inflight = controller; @@ -696,12 +703,10 @@ export function installSpaAuctionHook(): void { function patchHistoryMethod(method: 'pushState' | 'replaceState'): void { const original = history[method].bind(history); history[method] = function (state: unknown, unused: string, url?: string | URL | null): void { - const prevPath = location.pathname; original(state, unused, url); const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; - if (newPath !== prevPath) { - void onNavigate(newPath); - } + // onNavigate no-ops when newPath equals the last loaded path. + void onNavigate(newPath); }; } diff --git a/crates/js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts index 751b081f0..6be0a8484 100644 --- a/crates/js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/js/lib/test/integrations/gpt/spa_hook.test.ts @@ -21,6 +21,12 @@ async function flushAsync(): Promise { describe('installSpaAuctionHook', () => { let fetchStub: ReturnType; + // popstate listeners registered by each module import. In production the hook + // installs once (guarded by `ts.spaHookInstalled`), but tests wipe + // `window.tsjs` and re-import per test, so without explicit removal the + // listeners accumulate on the shared window and all fire on every dispatch. + let popstateHandlers: EventListenerOrEventListenerObject[] = []; + const realAddEventListener = window.addEventListener.bind(window); beforeEach(() => { vi.resetModules(); @@ -31,6 +37,11 @@ describe('installSpaAuctionHook', () => { history.replaceState = originalReplaceState; fetchStub = vi.fn(); vi.stubGlobal('fetch', fetchStub); + popstateHandlers = []; + vi.spyOn(window, 'addEventListener').mockImplementation((type, listener, options) => { + if (type === 'popstate' && listener) popstateHandlers.push(listener); + return realAddEventListener(type, listener, options); + }); }); afterEach(() => { @@ -40,6 +51,10 @@ describe('installSpaAuctionHook', () => { originalReplaceState({}, '', '/'); // Drop any ad containers inserted by a test so DOM state does not leak. document.body.innerHTML = ''; + // Remove this test's popstate listener(s) so they do not fire in later tests. + popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); + popstateHandlers = []; + vi.restoreAllMocks(); vi.unstubAllGlobals(); }); @@ -155,7 +170,7 @@ describe('installSpaAuctionHook', () => { expect(fetchStub).not.toHaveBeenCalled(); }); - it('fetches on replaceState and popstate navigation', async () => { + it('fetches on replaceState navigation', async () => { fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), @@ -168,15 +183,44 @@ describe('installSpaAuctionHook', () => { '/__ts/page-bids?path=%2Freplaced', expect.objectContaining({ credentials: 'include' }) ); + }); + it('fetches on popstate navigation to a new path', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + // Browsers change the URL out-of-band on back/forward, then fire popstate. + // Use the unwrapped history method so the patched handler is not invoked. + originalReplaceState({}, '', '/popped'); window.dispatchEvent(new PopStateEvent('popstate')); await flushAsync(); - expect(fetchStub).toHaveBeenLastCalledWith( - '/__ts/page-bids?path=%2Freplaced', + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Fpopped', expect.objectContaining({ credentials: 'include' }) ); }); + it('does not re-fetch on popstate to the same path', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + history.replaceState({}, '', '/replaced'); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledTimes(1); + + // popstate on the same path (hash-only change or scroll-restoration + // back/forward) must not re-request impressions. + window.dispatchEvent(new PopStateEvent('popstate')); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledTimes(1); + }); + it('drops a stale response that resolves after a newer navigation started', async () => { let resolveFirst: ((value: unknown) => void) | undefined; fetchStub diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index f52986c8a..8b5776298 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -3,18 +3,12 @@ // in the build context, so `dead_code` is expected. #![allow(clippy::unwrap_used, clippy::panic, dead_code)] -// Stub out dependencies for build.rs context -mod glob { - pub struct Pattern; - impl Pattern { - pub fn new(_: &str) -> Result { - Ok(Pattern) - } - pub fn matches(&self, _: &str) -> bool { - false - } - } -} +// `glob` is a real build-dependency (see Cargo.toml `[build-dependencies]`), so +// `creative_slot_build_check::pattern_compiles` resolves `glob::Pattern::new` +// against the actual glob crate. It must NOT be stubbed here: a stub that always +// returned `Ok` would let an invalid env-injected pattern such as +// `page_patterns = ["["]` pass the build-time check and embed into the config, +// only to be dropped by the real glob crate at runtime settings load. #[path = "src/error.rs"] mod error; diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index c654d39ee..4b901ee20 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -890,9 +890,16 @@ impl AuctionOrchestrator { break; } }; - remaining = select_result.remaining; + // Destructure so transport failures can be attributed to a provider + // via `failed_backend_name`, mirroring run_providers_parallel. + let crate::platform::PlatformSelectResult { + ready, + remaining: new_remaining, + failed_backend_name, + } = select_result; + remaining = new_remaining; - match select_result.ready { + match ready { Ok(platform_response) => { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some((provider_name, start_time, provider)) = @@ -920,8 +927,14 @@ impl AuctionOrchestrator { } Err(e) => { log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); - responses - .push(AuctionResponse::error(&provider_name, response_time_ms)); + // Mirror the parallel path so a parse failure is + // attributed (error_type + message) in provider_details. + responses.push(provider_error_response( + &provider_name, + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &e, + )); } } } else { @@ -932,7 +945,32 @@ impl AuctionOrchestrator { } } Err(e) => { - log::warn!("A provider request failed during collection: {:?}", e); + // Mirror the parallel path: attribute the transport failure to + // the provider behind `failed_backend_name` so it appears in + // provider_details instead of vanishing. + if let Some(ref backend_name) = failed_backend_name { + if let Some((provider_name, start_time, _)) = + backend_to_provider.remove(backend_name) + { + let response_time_ms = start_time.elapsed().as_millis() as u64; + log::warn!("Provider '{}' request failed: {:?}", provider_name, e); + responses.push(provider_transport_failed_response( + &provider_name, + response_time_ms, + )); + } else { + log::warn!( + "A provider request failed (backend '{}' not tracked): {:?}", + backend_name, + e + ); + } + } else { + log::warn!( + "A provider request failed during collection (backend not identified): {:?}", + e + ); + } } } diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index c55d98bc0..3090898fb 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -222,9 +222,22 @@ impl CreativeOpportunitySlot { .page_patterns .iter() .filter_map(|pattern| { - Pattern::new(pattern) - .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) - .ok() + match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { + Ok(compiled) => Some(compiled), + Err(_) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", + self.id, + pattern + ); + None + } + } }) .collect(); } @@ -370,6 +383,11 @@ pub struct PrebidSlotParams { /// /// Leave empty (or omit `bidders` in config) to auto-expand all /// `config.bidders` with zone-aware param overrides. + /// + /// Note: when this map is non-empty it is forwarded verbatim, so a slot's + /// `targeting.zone` is **not** injected for these bidders (the `trustedServer` + /// expansion key that carries it is only added when `bidders` is empty). Set + /// explicit per-bidder params only when you do not need zone-aware overrides. #[serde(default)] pub bidders: HashMap, } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 6a0f446b7..aa2892f9a 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -36,6 +36,48 @@ const ALLOWED_SLOT_FIELDS: &[&str] = &[ "providers", ]; +/// Fields the runtime [`CreativeOpportunityFormat`] accepts. +/// +/// Mirrors the struct's `#[serde(deny_unknown_fields)]`; the build path +/// deserializes formats as raw JSON, so a typo like `mediatype` (for +/// `media_type`) would otherwise embed and fail runtime settings load. +/// +/// [`CreativeOpportunityFormat`]: crate::creative_opportunities::CreativeOpportunityFormat +const ALLOWED_FORMAT_FIELDS: &[&str] = &["width", "height", "media_type"]; + +/// Provider keys the runtime [`SlotProviders`] accepts. +/// +/// [`SlotProviders`]: crate::creative_opportunities::SlotProviders +const ALLOWED_PROVIDER_FIELDS: &[&str] = &["aps", "prebid"]; + +/// Fields the runtime [`ApsSlotParams`] accepts. +/// +/// [`ApsSlotParams`]: crate::creative_opportunities::ApsSlotParams +const ALLOWED_APS_FIELDS: &[&str] = &["slot_id"]; + +/// Fields the runtime [`PrebidSlotParams`] accepts. +/// +/// [`PrebidSlotParams`]: crate::creative_opportunities::PrebidSlotParams +const ALLOWED_PREBID_FIELDS: &[&str] = &["bidders"]; + +/// Rejects any key in `object` that is not in `allowed`, mirroring the runtime +/// struct's `#[serde(deny_unknown_fields)]`. +/// +/// `context` names the offending object in the error (e.g. `` slot `atf` +/// format ``) so a build failure points at the exact config location. +fn reject_unknown_keys( + object: &serde_json::Map, + allowed: &[&str], + context: &str, +) -> Result<(), String> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(format!("{context} has unknown field '{key}'")); + } + } + Ok(()) +} + /// Validate that `value` is a `price_granularity` the runtime can deserialize. /// /// The build context types `price_granularity` as a `String`, so an invalid @@ -114,12 +156,60 @@ pub(crate) fn validate_creative_slot( // `#[serde(deny_unknown_fields)]`. The raw-JSON build path would otherwise // accept env-injected typos that the runtime rejects at settings load. if let Some(object) = slot.as_object() { - for key in object.keys() { - if !ALLOWED_SLOT_FIELDS.contains(&key.as_str()) { - return Err(format!("slot `{id}` has unknown field '{key}'")); + reject_unknown_keys(object, ALLOWED_SLOT_FIELDS, &format!("slot `{id}`"))?; + } + + // Reject nested unknown/mistyped fields too. The runtime's typed structs are + // all `#[serde(deny_unknown_fields)]`, but the raw-JSON build path bypasses + // those checks, so a config like `formats=[{width,height,mediatype}]` or + // `providers={aps={slotId}}` would otherwise pass the build and fail runtime + // settings load. + if let Some(formats) = slot.get("formats").and_then(serde_json::Value::as_array) { + for format in formats { + if let Some(object) = format.as_object() { + reject_unknown_keys( + object, + ALLOWED_FORMAT_FIELDS, + &format!("slot `{id}` format"), + )?; } } } + if let Some(providers) = slot.get("providers").and_then(serde_json::Value::as_object) { + reject_unknown_keys( + providers, + ALLOWED_PROVIDER_FIELDS, + &format!("slot `{id}` providers"), + )?; + if let Some(aps) = providers.get("aps").and_then(serde_json::Value::as_object) { + reject_unknown_keys( + aps, + ALLOWED_APS_FIELDS, + &format!("slot `{id}` providers.aps"), + )?; + } + if let Some(prebid) = providers + .get("prebid") + .and_then(serde_json::Value::as_object) + { + reject_unknown_keys( + prebid, + ALLOWED_PREBID_FIELDS, + &format!("slot `{id}` providers.prebid"), + )?; + } + } + + // An explicit empty/whitespace `div_id` override is rejected, mirroring + // `CreativeOpportunitySlot::validate_runtime`: the injected JS resolves slots + // with `candidate.id.startsWith(slot.div_id)`, and every element id starts + // with the empty string, so an empty override would bind the slot to the + // first id-bearing element in the document. + if let Some(div_id) = slot.get("div_id").and_then(serde_json::Value::as_str) { + if div_id.trim().is_empty() { + return Err(format!("slot `{id}` div_id override must not be empty")); + } + } // At least one page pattern that is non-empty and compiles as a glob. // Runtime preparation drops uncompilable patterns and rejects the slot when @@ -343,6 +433,96 @@ mod tests { assert!(err.contains("GAM unit path"), "got: {err}"); } + #[test] + fn rejects_blank_div_id_override() { + // An empty div_id override binds the slot to the first id-bearing + // element at runtime, so validate_runtime rejects it — the build must + // too, or a CI-green config fails settings load on the deployed service. + let slot = json!({ + "id": "atf", + "div_id": " ", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("blank div_id override must fail at build time"); + assert!( + err.contains("div_id override must not be empty"), + "got: {err}" + ); + } + + #[test] + fn rejects_unknown_format_field() { + // `mediatype` is a typo for `media_type`; the runtime format struct is + // deny_unknown_fields, so the build must reject it. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "mediatype": "banner" }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown format field must fail at build time"); + assert!(err.contains("unknown field 'mediatype'"), "got: {err}"); + } + + #[test] + fn rejects_unknown_provider_field() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "appnexus": {} } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown provider field must fail at build time"); + assert!(err.contains("unknown field 'appnexus'"), "got: {err}"); + } + + #[test] + fn rejects_unknown_aps_field() { + // `slotId` is a typo for `slot_id`. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "aps": { "slotId": "abc" } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown aps field must fail at build time"); + assert!(err.contains("unknown field 'slotId'"), "got: {err}"); + } + + #[test] + fn rejects_unknown_prebid_field() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "prebid": { "bidder": {} } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("unknown prebid field must fail at build time"); + assert!(err.contains("unknown field 'bidder'"), "got: {err}"); + } + + #[test] + fn accepts_well_formed_nested_provider_config() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": "banner" }], + "providers": { + "aps": { "slot_id": "abc" }, + "prebid": { "bidders": {} } + } + }); + assert!( + validate_creative_slot(&slot, "123456789").is_ok(), + "well-formed nested provider config must be accepted" + ); + } + #[test] fn rejects_missing_id() { let slot = json!({ "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index e24a138d4..efaac43fd 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -477,9 +477,12 @@ impl IntegrationHeadInjector for GptIntegration { /// Prebid Universal Creative request proves the TS creative rendered. /// It does **not** trigger refresh auctions or handle GPT slot refresh events. /// - /// Post-`window.load`, slim-Prebid takes over: it listens for GPT refresh - /// events, runs client-side auctions, and sets targeting for subsequent - /// impressions. SPA pushState navigation is also slim-Prebid's domain. + /// Post-`window.load`, slim-Prebid owns scroll and GPT refresh: it listens + /// for GPT refresh events, runs client-side auctions, and sets targeting for + /// subsequent impressions. SPA navigation is handled separately by + /// `installSpaAuctionHook()` in the GPT bundle, which re-runs the server-side + /// auction via `GET /__ts/page-bids` on pushState / replaceState / popstate + /// route changes (see `auction/endpoints.rs`). /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { let mut scripts = vec![ @@ -490,9 +493,14 @@ impl IntegrationHeadInjector for GptIntegration { ]; if let Some(ref url) = self.config.slim_prebid_url { + // JSON-encode the URL, then escape `` cannot close this inline tag and + // let trailing markup execute (standard JSON-in-HTML mitigation). + let encoded = serde_json::to_string(url) + .expect("should serialize string") + .replace("window.__tsjs_slim_prebid_url={};", - serde_json::to_string(url).expect("should serialize string") + "" )); } @@ -1298,6 +1306,44 @@ mod tests { ); } + #[test] + fn head_inserts_escapes_script_terminator_in_slim_prebid_url() { + // A configured URL containing `` must not close the inline tag. + let config = GptConfig { + slim_prebid_url: Some("https://cdn.example.com/x".to_string()), + ..test_config() + }; + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + // The injected `` must be neutralised: the only + // `` left is the tag's own legitimate closer. + assert!( + !inserts[2].contains(" terminator, got: {}", + inserts[2] + ); + assert_eq!( + inserts[2].matches("").count(), + 1, + "only the tag's own closing should remain, got: {}", + inserts[2] + ); + assert!( + inserts[2].contains("<\\/script>"), + "should emit the escaped terminator, got: {}", + inserts[2] + ); + } + #[test] fn head_inserts_omits_slim_prebid_url_when_not_configured() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index f1bd9d833..e069f3481 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -113,9 +113,10 @@ // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); // Map both the inner div and the GPT slot's element ID (the - // "-container" div when TS defined the slot there) so slotRenderEnded - // — which reports the GPT slot element ID — can find the slot for - // nurl/burl beacon firing. + // "-container" div when TS defined the slot there) into divToSlotId. + // This bootstrap fires no beacons and registers no slotRenderEnded + // listener; the map is consumed by the bundle's render bridge (index.ts) + // once it loads, which reports the GPT slot element ID. divToSlotId[actualDivId] = slot.id; var slotElementId = s.getSlotElementId(); if (slotElementId && slotElementId !== actualDivId) { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 44172a985..4b3979ed9 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1036,12 +1036,6 @@ pub struct AuctionDispatch<'a> { /// /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. -/// -/// # Panics -/// -/// Panics if `should_run_auction` is `true` but `settings.creative_opportunities` is `None`. -/// This is a logic invariant: `should_run_auction` is only set when creative opportunities -/// are configured, so this state is unreachable in practice. pub async fn handle_publisher_request( settings: &Settings, integration_registry: &IntegrationRegistry, @@ -1679,11 +1673,6 @@ fn is_supported_content_encoding(encoding: &str) -> bool { matches!(encoding, "" | "identity" | "gzip" | "deflate" | "br") } -/// Normalizes the client-supplied `path` query parameter before glob matching. -/// -/// The SPA hook sends `location.pathname`, but the parameter is -/// client-controlled: strip any query string or fragment and force a leading -/// `/` so slot `page_patterns` always match against a canonical path shape. /// Same-origin gate for `/__ts/page-bids`. /// /// The endpoint is a side-effecting GET: it dispatches real PBS/APS auctions @@ -1713,6 +1702,11 @@ fn page_bids_request_allowed(req: &Request) -> bool { } } +/// Normalizes the client-supplied `path` query parameter before glob matching. +/// +/// The SPA hook sends `location.pathname`, but the parameter is +/// client-controlled: strip any query string or fragment and force a leading +/// `/` so slot `page_patterns` always match against a canonical path shape. fn normalize_page_bids_path(raw: &str) -> String { let path = raw.split(['?', '#']).next().unwrap_or(""); if path.starts_with('/') { From dc2b18c3ce193fc52719ff06822cca85afe9f59f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 23 Jun 2026 18:33:50 +0530 Subject: [PATCH 119/195] Ignore leftover artifacts in pre-rename crate dirs The EdgeZero sync (#761) renamed crates/js and crates/integration-tests to crates/trusted-server-*. The old directories still hold local-only build artifacts (node_modules, target, dist) whose gitignore rules moved with the rename, so git now sees them as untracked. Ignore the defunct paths until the directories are removed from disk. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 25e2fa11f..9c6f49e76 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,9 @@ src/*.html /crates/trusted-server-integration-tests/browser/test-results/ /crates/trusted-server-integration-tests/browser/playwright-report/ /crates/trusted-server-integration-tests/browser/.browser-test-state.json + +# Defunct pre-rename crate dirs (renamed to crates/trusted-server-*); ignore the +# leftover local build artifacts (node_modules, target, dist) that remain on disk. +/crates/js/ +/crates/integration-tests/ + From 7d34bbbafed5bb0a3aa43e06e2e8774516cb2818 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 23 Jun 2026 18:34:18 +0530 Subject: [PATCH 120/195] Address PR #680 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 — EdgeZero finalize cache/Set-Cookie privacy parity: Share the protected finalizer between the legacy and EdgeZero paths. apply_finalize_headers now strips surrogate cache headers and downgrades cookie-bearing responses to private, and skips operator response_headers that would re-enable shared caching on uncacheable responses; finalize_response delegates to it. The EdgeZero entry point re-applies an HttpResponse enforce_set_cookie_cache_privacy after ec_finalize_response and request-filter effects so a late EC Set-Cookie cannot reach a shared cache. Adds middleware tests for both cases. P1 — empty page-bids must not enable GPT services: adInit() only enables GPT services when it has a slot to display or refresh, and the SPA hook skips adInit() for an empty page-bids response unless prior TS state needs sweeping. Prevents a consent-denied or kill-switched navigation from activating the publisher's GPT setup. P2 — scope Prebid refresh targeting to the refreshed slots: setTargetingForGPTAsync is called with the synthetic refresh ad-unit codes so a one-slot refresh no longer mutates unrelated GPT slots. P2 — validate nested slot value shapes at build time: The creative-slot build check now validates media_type against the runtime MediaType variants, targeting as a string map, page_patterns as strings, providers.aps.slot_id as a string, providers.prebid.bidders as a map, and floor_price as a number — closing build-green/runtime-broken gaps. A drift-guard test ties media_type to the runtime enum. CI — suppress CodeQL cleartext-logging false positives: Annotate the provider/mediator "not registered" warnings; they log static config identifiers, not secrets. --- .../trusted-server-adapter-fastly/src/main.rs | 98 +------ .../src/middleware.rs | 218 +++++++++++++++- .../src/auction/orchestrator.rs | 4 + .../src/creative_slot_build_check.rs | 246 ++++++++++++++++++ .../lib/src/integrations/gpt/index.ts | 26 +- .../lib/src/integrations/prebid/index.ts | 7 +- .../lib/test/integrations/gpt/ad_init.test.ts | 34 +++ .../test/integrations/gpt/spa_hook.test.ts | 43 +++ .../test/integrations/prebid/index.test.ts | 51 ++++ 9 files changed, 636 insertions(+), 91 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 274b37c22..711aa2008 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -6,8 +6,7 @@ use edgezero_core::app::Hooks as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::http::{ - header, HeaderName, HeaderValue, Method, Request as HttpRequest, Response as HttpResponse, - StatusCode, + header, HeaderValue, Method, Request as HttpRequest, Response as HttpResponse, StatusCode, }; use error_stack::Report; use fastly::http::Method as FastlyMethod; @@ -16,10 +15,7 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::AuctionOrchestrator; use trusted_server_core::auth::enforce_basic_auth; -use trusted_server_core::constants::{ - COOKIE_SHAREDID, COOKIE_TS_EIDS, ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, - HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, HEADER_X_TS_VERSION, -}; +use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -374,6 +370,11 @@ fn edgezero_main(mut req: FastlyRequest, config_store: ConfigStoreHandle) { if let Some(effects) = &request_filter_effects { effects.apply_to_response(&mut response); } + // Final cache guard: EC finalization and request-filter + // effects above may have added a per-user Set-Cookie after + // `apply_finalize_headers` ran, so re-apply the privacy + // downgrade before send, mirroring legacy_main. + crate::middleware::enforce_set_cookie_cache_privacy(&mut response); compat::to_fastly_response(response).send_to_client(); if ec_state.is_real_browser { @@ -403,6 +404,9 @@ fn edgezero_main(mut req: FastlyRequest, config_store: ConfigStoreHandle) { if let Some(effects) = &request_filter_effects { effects.apply_to_response(&mut response); } + // Final cache guard for the no-EC-finalization fallback: request-filter + // effects may still have added a per-user Set-Cookie after finalize headers. + crate::middleware::enforce_set_cookie_cache_privacy(&mut response); compat::to_fastly_response(response).send_to_client(); } @@ -1212,84 +1216,10 @@ fn publisher_response_carries_body(method: &Method, status: StatusCode) -> bool /// version/staging, then operator-configured `settings.response_headers`. /// This means operators can intentionally override any managed header. fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: &mut HttpResponse) { - if let Some(geo) = geo_info { - geo.set_response_headers(response); - } else { - response.headers_mut().insert( - HEADER_X_GEO_INFO_AVAILABLE, - HeaderValue::from_static("false"), - ); - } - - if let Ok(v) = ::std::env::var(ENV_FASTLY_SERVICE_VERSION) { - if let Ok(value) = HeaderValue::from_str(&v) { - response.headers_mut().insert(HEADER_X_TS_VERSION, value); - } else { - log::warn!("Skipping invalid FASTLY_SERVICE_VERSION response header value"); - } - } - if ::std::env::var(ENV_FASTLY_IS_STAGING).as_deref() == Ok("1") { - response - .headers_mut() - .insert(HEADER_X_TS_ENV, HeaderValue::from_static("staging")); - } - - // Any response that sets a per-user cookie (notably the EC identity cookie - // minted on a visitor's first navigation) must never be shared-cached, or a - // shared cache could replay one user's Set-Cookie to others. The publisher - // path only forces `private` for HTML that carries inline ad data, so this - // net covers ordinary navigations whose sole per-user payload is the cookie. - // Skip when the response is already uncacheable so we don't clobber a - // stricter directive (e.g. `no-store`). - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - if response.headers().contains_key(header::SET_COOKIE) { - // Surrogate cache headers must come off every cookie-bearing response, - // even one already carrying a stricter `no-store`/`private` directive — - // they are independent of Cache-Control and would otherwise let a shared - // cache store and replay one visitor's Set-Cookie. - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if !already_uncacheable { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - } - } - - // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) - // carry an uncacheable Cache-Control directive (`private` or `no-store`). - // Operator headers must not re-enable shared caching for them — neither by - // replacing Cache-Control nor by reintroducing the surrogate cache headers - // the privacy paths stripped. - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - - for (key, value) in &settings.response_headers { - if response_is_uncacheable - && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || key.eq_ignore_ascii_case("surrogate-control") - || key.eq_ignore_ascii_case("fastly-surrogate-control")) - { - continue; - } - let header_name = HeaderName::from_bytes(key.as_bytes()) - .expect("settings.response_headers validated at load time"); - let header_value = - HeaderValue::from_str(value).expect("settings.response_headers validated at load time"); - response.headers_mut().insert(header_name, header_value); - } + // Legacy and EdgeZero paths share one protected finalizer so the cache / + // Set-Cookie privacy hardening cannot drift between them. `HttpResponse` and + // the middleware's `Response` are the same `edgezero_core::http::Response`. + apply_finalize_headers(settings, geo_info, response); } /// Forces cookie-bearing Fastly responses to stay private to shared caches. diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index ceb470b7d..7c24d2dbb 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use edgezero_adapter_fastly::context::FastlyRequestContext; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response, StatusCode}; +use edgezero_core::http::{header, HeaderName, HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; use std::net::IpAddr; @@ -181,13 +181,19 @@ where /// Applies all standard Trusted Server response headers to the given response. /// /// Mirrors [`crate::finalize_response`] exactly, operating on [`Response`] from -/// `edgezero_core::http` instead of `HttpResponse`. +/// `edgezero_core::http` instead of `HttpResponse`. [`crate::finalize_response`] +/// delegates here so the legacy and `EdgeZero` paths share one protected +/// finalizer. /// /// Header write order (last write wins): /// 1. Geo headers (`x-geo-*`) — or `X-Geo-Info-Available: false` when absent /// 2. `X-TS-Version` from `FASTLY_SERVICE_VERSION` env var /// 3. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` -/// 4. `settings.response_headers` — operator-configured overrides applied last +/// 4. Set-Cookie cache privacy — strip surrogate cache headers and downgrade +/// `Cache-Control` to `private, max-age=0` on cookie-bearing responses +/// 5. `settings.response_headers` — operator-configured overrides, except the +/// cache-controlling headers are skipped on uncacheable (`private`/`no-store`) +/// responses so operators cannot re-enable shared caching for per-user payloads pub(crate) fn apply_finalize_headers( settings: &Settings, geo_info: Option<&GeoInfo>, @@ -216,7 +222,32 @@ pub(crate) fn apply_finalize_headers( .insert(HEADER_X_TS_ENV, HeaderValue::from_static("staging")); } + // Any response that sets a per-user cookie (notably the EC identity cookie) + // must never be shared-cached, or a shared cache could replay one user's + // Set-Cookie to others. Skip when the response is already uncacheable so we + // don't clobber a stricter directive (e.g. `no-store`). + enforce_set_cookie_cache_privacy(response); + + // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) + // carry an uncacheable Cache-Control directive (`private` or `no-store`). + // Operator headers must not re-enable shared caching for them — neither by + // replacing Cache-Control nor by reintroducing the surrogate cache headers + // the privacy paths stripped. + let response_is_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + for (key, value) in &settings.response_headers { + if response_is_uncacheable + && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) + || key.eq_ignore_ascii_case("surrogate-control") + || key.eq_ignore_ascii_case("fastly-surrogate-control")) + { + continue; + } let header_name = HeaderName::from_bytes(key.as_bytes()) .expect("should be a valid header name: response_headers validated in prepare_runtime"); let header_value = HeaderValue::from_str(value).expect( @@ -226,6 +257,44 @@ pub(crate) fn apply_finalize_headers( } } +/// Forces cookie-bearing responses to stay private to shared caches. +/// +/// Mirrors [`crate::enforce_set_cookie_cache_privacy`] for the [`Response`] type +/// from `edgezero_core::http`. The `EdgeZero` entry point re-applies this after +/// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) +/// and request-filter effects, because the EC identity `Set-Cookie` is written +/// after [`apply_finalize_headers`] runs and would otherwise reach a shared cache +/// with inherited `public`/surrogate cache headers. +/// +/// Idempotent: a response already marked `private`/`no-store` keeps its stricter +/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a +/// `no-store` cookie response can never retain shared cacheability. +pub(crate) fn enforce_set_cookie_cache_privacy(response: &mut Response) { + if !response.headers().contains_key(header::SET_COOKIE) { + return; + } + // Surrogate cache headers must come off every cookie-bearing response, even + // one already carrying a stricter `no-store`/`private` directive — they are + // independent of Cache-Control and would otherwise let a shared cache store + // and replay one visitor's Set-Cookie. + response.headers_mut().remove("surrogate-control"); + response.headers_mut().remove("fastly-surrogate-control"); + // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match + // against a lowercased copy — `No-Store` / `Private` must count. + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -342,6 +411,149 @@ mod tests { ); } + fn response_with_headers(pairs: &[(&'static str, &'static str)]) -> Response { + let mut response = empty_response(); + for (key, value) in pairs { + response.headers_mut().insert( + HeaderName::from_static(key), + HeaderValue::from_static(value), + ); + } + response + } + + #[test] + fn apply_finalize_headers_downgrades_public_set_cookie_response() { + // A per-user cookie response that arrives shared-cacheable (origin-public + // plus a surrogate directive) must be downgraded so a shared cache cannot + // store and replay one visitor's Set-Cookie. + let settings = settings_with_response_headers(vec![]); + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade a public cookie response to private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from a cookie-bearing response" + ); + } + + #[test] + fn apply_finalize_headers_blocks_operator_surrogate_on_private_response() { + // Operator response_headers must not re-enable shared caching for an + // uncacheable (private) per-user response — neither by replacing + // Cache-Control nor by reintroducing surrogate cache headers. + let settings = settings_with_response_headers(vec![ + ("cache-control", "public, max-age=3600"), + ("surrogate-control", "max-age=3600"), + ("x-operator", "kept"), + ]); + let mut response = response_with_headers(&[("cache-control", "private, max-age=0")]); + + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "operator cache-control must not weaken a private response" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "operator surrogate-control must not be applied to a private response" + ); + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("kept"), + "non-cache operator headers must still apply" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_downgrades_late_cookie() { + // Mirrors the EdgeZero post-ec_finalize guard: a Set-Cookie added after + // finalize headers ran (origin-public response) must be downgraded. + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade a late public cookie response to private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from the late cookie response" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { + // Idempotent: a stricter no-store directive is preserved, but surrogate + // headers still come off. + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "no-store"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "should keep the stricter no-store directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control even when keeping no-store" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_ignores_cookieless_response() { + let mut response = response_with_headers(&[("cache-control", "public, max-age=600")]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("public, max-age=600"), + "should leave a cookieless response untouched" + ); + } + // --------------------------------------------------------------------------- // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 36ae00c58..18efd1865 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -748,6 +748,8 @@ impl AuctionOrchestrator { let provider = match self.providers.get(provider_name) { Some(p) => p, None => { + // lgtm[rust/cleartext-logging] + // The provider name is a static config identifier (e.g. "prebid"), not a secret. log::warn!("Provider '{}' not registered, skipping", provider_name); continue; } @@ -1108,6 +1110,8 @@ impl AuctionOrchestrator { } } None => { + // lgtm[rust/cleartext-logging] + // The mediator name is a static config identifier, not a secret. log::warn!("Mediator '{}' not registered", mediator_name); (None, self.select_winning_bids(&responses, &floor_prices)) } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index aa2892f9a..15e5ca98a 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -104,6 +104,57 @@ pub(crate) fn validate_price_granularity(value: &str) -> Result<(), String> { }) } +/// Accepted `media_type` values, mirroring the runtime [`MediaType`] enum's +/// `#[serde(rename_all = "lowercase")]` variants. +/// +/// The build path types a format's `media_type` as raw JSON, so a value such as +/// `"bannerr"` would embed cleanly and then fail runtime settings load — the real +/// [`MediaType`] enum cannot deserialize it. A crate-context test +/// (`media_type_values_match_runtime_enum`) asserts this list stays in lockstep +/// with the enum's `Deserialize` impl, so the two cannot drift. +/// +/// [`MediaType`]: crate::auction::types::MediaType +const MEDIA_TYPE_VALUES: &[&str] = &["banner", "video", "native"]; + +/// Validate a format's `media_type` value against the runtime [`MediaType`] enum. +/// +/// # Errors +/// +/// Returns an error string when `value` is not a JSON string naming one of the +/// runtime [`MediaType`] variants. +/// +/// [`MediaType`]: crate::auction::types::MediaType +fn validate_media_type(value: &serde_json::Value, slot_id: &str) -> Result<(), String> { + let media_type = value + .as_str() + .ok_or_else(|| format!("slot `{slot_id}` format media_type must be a string"))?; + if !MEDIA_TYPE_VALUES.contains(&media_type) { + return Err(format!( + "slot `{slot_id}` format media_type '{media_type}' is invalid; expected one of: banner, video, native" + )); + } + Ok(()) +} + +/// Validate that `value` is a string→string map, mirroring a runtime +/// `HashMap` field. +/// +/// # Errors +/// +/// Returns an error string when `value` is not a JSON object or any of its values +/// is not a JSON string. `context` names the offending field in the error. +fn validate_string_map(value: &serde_json::Value, context: &str) -> Result<(), String> { + let object = value + .as_object() + .ok_or_else(|| format!("{context} must be a map of string keys to string values"))?; + for (key, entry) in object { + if !entry.is_string() { + return Err(format!("{context} value for '{key}' must be a string")); + } + } + Ok(()) +} + /// Returns `true` when `id` is non-empty and only `[A-Za-z0-9_-]`. fn is_valid_slot_id(id: &str) -> bool { !id.is_empty() @@ -172,6 +223,12 @@ pub(crate) fn validate_creative_slot( ALLOWED_FORMAT_FIELDS, &format!("slot `{id}` format"), )?; + // Validate the nested `media_type` value, not just the field + // name: a value like `"bannerr"` passes the key check but the + // runtime `MediaType` enum cannot deserialize it. + if let Some(media_type) = object.get("media_type") { + validate_media_type(media_type, id)?; + } } } } @@ -187,6 +244,15 @@ pub(crate) fn validate_creative_slot( ALLOWED_APS_FIELDS, &format!("slot `{id}` providers.aps"), )?; + // `ApsSlotParams::slot_id` is a `String`; a non-string value embeds + // cleanly but fails runtime deserialization. + if let Some(slot_id_value) = aps.get("slot_id") { + if !slot_id_value.is_string() { + return Err(format!( + "slot `{id}` providers.aps.slot_id must be a string" + )); + } + } } if let Some(prebid) = providers .get("prebid") @@ -197,6 +263,29 @@ pub(crate) fn validate_creative_slot( ALLOWED_PREBID_FIELDS, &format!("slot `{id}` providers.prebid"), )?; + // `PrebidSlotParams::bidders` is a map; a non-object value (e.g. a + // bare string or array) fails runtime deserialization. + if let Some(bidders) = prebid.get("bidders") { + if !bidders.is_object() { + return Err(format!( + "slot `{id}` providers.prebid.bidders must be a map of bidder names to params" + )); + } + } + } + } + + // `targeting` is a runtime `HashMap`; a non-string value + // (e.g. `targeting = { pos = 1 }`) embeds cleanly but fails settings load. + if let Some(targeting) = slot.get("targeting") { + validate_string_map(targeting, &format!("slot `{id}` targeting"))?; + } + + // `floor_price` is an `Option`; a non-numeric value would fail the + // runtime deserialization the build path otherwise bypasses. + if let Some(floor_price) = slot.get("floor_price") { + if !floor_price.is_null() && floor_price.as_f64().is_none() { + return Err(format!("slot `{id}` floor_price must be a number")); } } @@ -211,6 +300,18 @@ pub(crate) fn validate_creative_slot( } } + // `page_patterns` is a runtime `Vec`; a non-string entry (e.g. + // `page_patterns = [123]`) fails deserialization. The validity check below + // skips non-strings via `filter_map`, so reject them explicitly first. + if let Some(patterns) = slot + .get("page_patterns") + .and_then(serde_json::Value::as_array) + { + if patterns.iter().any(|p| !p.is_string()) { + return Err(format!("slot `{id}` page_patterns entries must be strings")); + } + } + // At least one page pattern that is non-empty and compiles as a glob. // Runtime preparation drops uncompilable patterns and rejects the slot when // none remain, so a private/env config like `page_patterns = ["["]` would @@ -523,6 +624,151 @@ mod tests { ); } + #[test] + fn rejects_invalid_media_type() { + // `bannerr` passes the field-name check but the runtime MediaType enum + // cannot deserialize it, so settings load would fail on the service. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": "bannerr" }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("invalid media_type must fail at build time"); + assert!( + err.contains("media_type 'bannerr' is invalid"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_string_media_type() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": 1 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string media_type must fail at build time"); + assert!(err.contains("media_type must be a string"), "got: {err}"); + } + + #[test] + fn accepts_all_media_types() { + for media_type in ["banner", "video", "native"] { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250, "media_type": media_type }] + }); + assert!( + validate_creative_slot(&slot, "123456789").is_ok(), + "'{media_type}' should be a valid media_type" + ); + } + } + + #[test] + fn media_type_values_match_runtime_enum() { + use crate::auction::types::MediaType; + // Every listed value must deserialize into the runtime enum. + for value in super::MEDIA_TYPE_VALUES { + serde_json::from_value::(json!(value)) + .unwrap_or_else(|_| panic!("'{value}' should deserialize into MediaType")); + } + // Exhaustive match so a newly added MediaType variant forces this test + // (and MEDIA_TYPE_VALUES) to be updated, preventing silent drift. + for variant in [MediaType::Banner, MediaType::Video, MediaType::Native] { + let covered = match variant { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + }; + assert!( + super::MEDIA_TYPE_VALUES.contains(&covered), + "MEDIA_TYPE_VALUES is missing runtime variant '{covered}'" + ); + } + } + + #[test] + fn rejects_non_string_targeting_value() { + // `targeting` is a runtime HashMap; a numeric value + // embeds cleanly but fails settings load. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "targeting": { "pos": 1 } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string targeting value must fail at build time"); + assert!( + err.contains("targeting value for 'pos' must be a string"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_string_aps_slot_id() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "aps": { "slot_id": 123 } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string aps slot_id must fail at build time"); + assert!( + err.contains("providers.aps.slot_id must be a string"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_object_prebid_bidders() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "providers": { "prebid": { "bidders": "appnexus" } } + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-object prebid bidders must fail at build time"); + assert!( + err.contains("providers.prebid.bidders must be a map"), + "got: {err}" + ); + } + + #[test] + fn rejects_non_numeric_floor_price() { + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 300, "height": 250 }], + "floor_price": "high" + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-numeric floor_price must fail at build time"); + assert!(err.contains("floor_price must be a number"), "got: {err}"); + } + + #[test] + fn rejects_non_string_page_pattern_entry() { + let slot = json!({ + "id": "atf", + "page_patterns": [123], + "formats": [{ "width": 300, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("non-string page_patterns entry must fail at build time"); + assert!( + err.contains("page_patterns entries must be strings"), + "got: {err}" + ); + } + #[test] fn rejects_missing_id() { let slot = json!({ "page_patterns": ["/"], "formats": [{ "width": 1, "height": 1 }] }); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 30071220d..73b9419c6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -536,8 +536,18 @@ export function installTsAdInit(): void { ts.divToSlotId = divToSlotId; ts.prevSlotTargetingKeys = nextSlotTargetingKeys; - // enableSingleRequest and enableServices must only be called once per page load. - if (!ts.servicesEnabled) { + // Whether this call produced any TS slot to render. A gated page-bids + // response (auction kill switch or consent denial) returns no slots, so + // the loops above leave these empty. + const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; + + // enableSingleRequest and enableServices must only be called once per page + // load. Skip activating GPT services when TS has nothing to display or + // refresh and has not already enabled them: a consent-denied or + // kill-switched navigation must not turn on the publisher's GPT services + // or race their own setup. The targeting sweep above still runs so stale + // TS targeting from a prior navigation is cleared. + if (!ts.servicesEnabled && hasRenderableWork) { g.pubads!().enableSingleRequest(); g.enableServices?.(); ts.servicesEnabled = true; @@ -693,7 +703,17 @@ export function installSpaAuctionHook(): void { if (inflight !== controller) return; ts.adSlots = data.slots; ts.bids = data.bids; - ts.adInit?.(); + // An empty page-bids response (auction kill switch or consent gate) carries + // no TS slots. Only run adInit() when there are slots to apply or prior TS + // state to sweep — otherwise a consent-denied or kill-switched navigation + // must not enter the GPT command queue and risk activating services. + const hasPriorTsState = + (ts.prevGptSlots?.length ?? 0) > 0 || + Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || + Object.keys(ts.divToSlotId ?? {}).length > 0; + if (data.slots.length > 0 || hasPriorTsState) { + ts.adInit?.(); + } } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; log.warn('SPA auction hook: fetch failed', err); diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index fe175b44d..40a8d9e2e 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -779,10 +779,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { }; }); + // Scope GPT targeting to just the synthetic refresh ad units. An unscoped + // call would set hb_* targeting on every ad unit with known bids, mutating + // unrelated GPT slots whose targeting this wrapper only cleared for + // `targetSlots` — leaving their next request dependent on stale state. + const refreshAdUnitCodes = adUnits.map((unit) => unit.code); pbjs.requestBids({ adUnits, bidsBackHandler: () => { - pbjs.setTargetingForGPTAsync?.(); + pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); originalRefresh(targetSlots, opts); }, timeout: timeoutMs, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index d649778bc..b82542695 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -337,6 +337,40 @@ describe('installTsAdInit', () => { expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); }); + it('does not enable GPT services when the page-bids response has no slots', async () => { + // A gated page-bids response returns no slots. With nothing to display or + // refresh and services not already enabled, adInit() must not call + // enableSingleRequest()/enableServices() and activate the publisher's GPT + // services on a consent-denied or kill-switched navigation. + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const enableServices = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn(), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices, + }; + (window as TestWindow).tsjs = { + adSlots: [], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(mockPubads.enableSingleRequest).not.toHaveBeenCalled(); + expect(enableServices).not.toHaveBeenCalled(); + expect((window as TestWindow).tsjs!.servicesEnabled).toBeFalsy(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + }); + it('keeps the GAM path when debug adm is present', async () => { const slotEl = document.getElementById('div-atf-sidebar')!; const mockSlot = { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 6be0a8484..28854a902 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -89,6 +89,49 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('skips adInit on an empty page-bids response with no prior TS state', async () => { + // A gated page-bids response (auction kill switch or consent denial) returns + // no slots. With no prior TS state to sweep, the hook must not call adInit() + // so a consent-denied navigation cannot activate the publisher's GPT setup. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/gated-route'); + await flushAsync(); + + expect(ts.adSlots).toEqual([]); + expect(ts.bids).toEqual({}); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('runs adInit on an empty page-bids response when prior TS state exists', async () => { + // When TS touched slots on a previous navigation, an empty response still + // needs adInit() to sweep the stale TS targeting from those slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.prevSlotTargetingKeys = { 'div-prev': ['hb_pb'] }; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/cleanup-route'); + await flushAsync(); + + expect(ts.adSlots).toEqual([]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + it('defers applying bids until the route ad container is inserted', async () => { fetchStub.mockResolvedValue({ ok: true, diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index fd7703546..738a1cc78 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -864,6 +864,57 @@ describe('prebid/installRefreshHandler', () => { ); }); + it('scopes the GPT targeting call to the refreshed slot code', () => { + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + // Run the bidsBackHandler synchronously so the targeting call fires. + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const originalRefresh = vi.fn(); + // Only the header slot is refreshed; the footer slot must be untouched. + const headerSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn().mockReturnThis(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [headerSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[728, 90]], + targeting: { zone: 'header' }, + }, + { + id: 'footer_ad', + gam_unit_path: '/123/footer', + div_id: 'div-ad-footer', + formats: [[728, 90]], + targeting: { zone: 'footer' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([headerSlot]); + + expect(setTargetingForGPTAsync).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-header']); + expect(originalRefresh).toHaveBeenCalledWith([headerSlot], undefined); + + delete (mockPbjs as any).setTargetingForGPTAsync; + }); + it('includes configured client-side bidders in refresh ad units', () => { (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; // Original publisher ad unit carries a client-side rubicon bid. From 5a835c13a137e8850f82a71df426a4d88a82c768 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 23 Jun 2026 18:40:26 +0530 Subject: [PATCH 121/195] Add glob to the integration-tests lockfile The merge took main's trusted-server-integration-tests Cargo.lock, but the branch's trusted-server-core now pulls in glob (the creative-slot build check uses glob::Pattern). The integration crate path-depends on core, so its locked graph was missing glob and the --locked CI build refused to update it. Add only glob v0.3.3; no other versions change, keeping the shared direct-dependency parity check green. --- crates/trusted-server-integration-tests/Cargo.lock | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-integration-tests/Cargo.lock b/crates/trusted-server-integration-tests/Cargo.lock index 48d0af29e..692858beb 100644 --- a/crates/trusted-server-integration-tests/Cargo.lock +++ b/crates/trusted-server-integration-tests/Cargo.lock @@ -1502,6 +1502,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -3755,7 +3761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4154,6 +4160,7 @@ dependencies = [ "fastly", "flate2", "futures", + "glob", "hex", "hmac", "http", @@ -4169,6 +4176,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "subtle", + "tokio", "toml", "trusted-server-js", "trusted-server-openrtb", From 975b4aa0dbb578d92735616dba2d45900c42b062 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 25 Jun 2026 20:25:21 +0530 Subject: [PATCH 122/195] Fix server-side ad template review blockers --- .../src/backend.rs | 82 +++++++++++++++---- .../trusted-server-adapter-fastly/src/main.rs | 74 ++++++++++++++++- .../src/platform.rs | 14 +++- .../src/auction/orchestrator.rs | 38 +++++---- .../trusted-server-core/src/ec/pull_sync.rs | 1 + .../src/integrations/datadome/protection.rs | 1 + .../src/integrations/mod.rs | 1 + .../src/integrations/prebid.rs | 68 ++++++++++++--- .../src/platform/test_support.rs | 1 + .../trusted-server-core/src/platform/types.rs | 2 + crates/trusted-server-core/src/proxy.rs | 2 + crates/trusted-server-core/src/publisher.rs | 1 + 12 files changed, 232 insertions(+), 53 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index 7763eaf0e..4056c81da 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -49,6 +49,8 @@ fn sanitize_backend_name_component(value: &str) -> String { /// Default first-byte timeout for backends (15 seconds). pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +/// Default timeout between response body bytes for backends (10 seconds). +pub(crate) const DEFAULT_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(10); /// Configuration for creating a dynamic Fastly backend. /// @@ -60,6 +62,7 @@ pub struct BackendConfig<'a> { port: Option, certificate_check: bool, first_byte_timeout: Duration, + between_bytes_timeout: Duration, host_header_override: Option<&'a str>, } @@ -76,6 +79,7 @@ impl<'a> BackendConfig<'a> { port: None, certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_BETWEEN_BYTES_TIMEOUT, host_header_override: None, } } @@ -106,6 +110,17 @@ impl<'a> BackendConfig<'a> { self } + /// Set the maximum time to wait between response body bytes. + /// + /// Defaults to 10 seconds. Auction backends should set this to the same + /// remaining budget as the first-byte timeout so slow-drip bodies cannot + /// hold the auction past its deadline. + #[must_use] + pub fn between_bytes_timeout(mut self, timeout: Duration) -> Self { + self.between_bytes_timeout = timeout; + self + } + /// Set the outbound Host header sent to the backend origin. #[must_use] pub fn host_header_override(mut self, host: Option<&'a str>) -> Self { @@ -159,13 +174,15 @@ impl<'a> BackendConfig<'a> { } else { "_nocert" }; - let timeout_ms = self.first_byte_timeout.as_millis(); + let first_byte_timeout_ms = self.first_byte_timeout.as_millis(); + let between_bytes_timeout_ms = self.between_bytes_timeout.as_millis(); let backend_name = format!( - "backend_{}{}{}_t{}", + "backend_{}{}{}_fb{}_bb{}", sanitize_backend_name_component(&name_base), host_override_suffix, cert_suffix, - timeout_ms + first_byte_timeout_ms, + between_bytes_timeout_ms ); Ok((backend_name, target_port)) @@ -187,9 +204,10 @@ impl<'a> BackendConfig<'a> { /// Ensure a dynamic backend exists for this configuration and return its name. /// /// The backend name is derived from the scheme, host, port, certificate - /// setting, and `first_byte_timeout` to avoid collisions. Different - /// timeout values produce different backend registrations so that a - /// tight deadline cannot be silently widened by an earlier registration. + /// setting, `first_byte_timeout`, and `between_bytes_timeout` to avoid + /// collisions. Different timeout values produce different backend + /// registrations so that a tight deadline cannot be silently widened by an + /// earlier registration. /// /// # Errors /// @@ -210,7 +228,7 @@ impl<'a> BackendConfig<'a> { .override_host(&host_header) .connect_timeout(Duration::from_secs(1)) .first_byte_timeout(self.first_byte_timeout) - .between_bytes_timeout(Duration::from_secs(10)); + .between_bytes_timeout(self.between_bytes_timeout); if self.scheme.eq_ignore_ascii_case("https") { builder = builder.enable_ssl().sni_hostname(self.host); if self.certificate_check { @@ -381,7 +399,7 @@ mod tests { let name = BackendConfig::new("https", "origin.example.com") .ensure() .expect("should create backend for valid HTTPS origin"); - assert_eq!(name, "backend_https_origin_example_com_443_t15000"); + assert_eq!(name, "backend_https_origin_example_com_443_fb15000_bb10000"); } #[test] @@ -390,7 +408,10 @@ mod tests { .certificate_check(false) .ensure() .expect("should create backend with cert check disabled"); - assert_eq!(name, "backend_https_origin_example_com_443_nocert_t15000"); + assert_eq!( + name, + "backend_https_origin_example_com_443_nocert_fb15000_bb10000" + ); } #[test] @@ -399,7 +420,7 @@ mod tests { .port(Some(8080)) .ensure() .expect("should create backend for HTTP origin with explicit port"); - assert_eq!(name, "backend_http_api_test-site_org_8080_t15000"); + assert_eq!(name, "backend_http_api_test-site_org_8080_fb15000_bb10000"); } #[test] @@ -407,7 +428,7 @@ mod tests { let name = BackendConfig::new("http", "example.org") .ensure() .expect("should create backend defaulting to port 80 for HTTP"); - assert_eq!(name, "backend_http_example_org_80_t15000"); + assert_eq!(name, "backend_http_example_org_80_fb15000_bb10000"); } #[test] @@ -464,11 +485,11 @@ mod tests { ); assert_eq!( name_a, - "backend_https_origin_example_com_443_oh_www_example_com_t15000" + "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb10000" ); assert_eq!( name_b, - "backend_https_origin_example_com_443_oh_m_example_com_t15000" + "backend_https_origin_example_com_443_oh_m_example_com_fb15000_bb10000" ); } @@ -523,12 +544,39 @@ mod tests { "backends with different timeouts should have different names" ); assert!( - name_a.ends_with("_t2000"), - "name should include timeout suffix" + name_a.ends_with("_fb2000_bb10000"), + "name should include first-byte and between-bytes timeout suffix" + ); + assert!( + name_b.ends_with("_fb500_bb10000"), + "name should include first-byte and between-bytes timeout suffix" + ); + } + + #[test] + fn different_between_bytes_timeouts_produce_different_names() { + use std::time::Duration; + + let (name_a, _) = BackendConfig::new("https", "origin.example.com") + .between_bytes_timeout(Duration::from_secs(2)) + .compute_name() + .expect("should compute name with 2000ms between-bytes timeout"); + let (name_b, _) = BackendConfig::new("https", "origin.example.com") + .between_bytes_timeout(Duration::from_millis(500)) + .compute_name() + .expect("should compute name with 500ms between-bytes timeout"); + + assert_ne!( + name_a, name_b, + "backends with different between-bytes timeouts should have different names" + ); + assert!( + name_a.ends_with("_fb15000_bb2000"), + "name should include first-byte and between-bytes timeout suffix" ); assert!( - name_b.ends_with("_t500"), - "name should include timeout suffix" + name_b.ends_with("_fb15000_bb500"), + "name should include first-byte and between-bytes timeout suffix" ); } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 711aa2008..017f1a9ef 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -143,6 +143,10 @@ fn is_edgezero_enabled(config_store: &ConfigStoreHandle) -> Result bool { + settings.creative_opportunities.is_none() +} + fn health_response(req: &FastlyRequest) -> Option { if req.get_method() == FastlyMethod::GET && req.get_path() == "/health" { return Some(FastlyResponse::from_status(200).with_body_text_plain("ok")); @@ -194,8 +198,24 @@ fn main() { log::warn!("failed to read edgezero_enabled flag, falling back to legacy path: {e}"); false }) { - log::debug!("routing request through EdgeZero path"); - edgezero_main(req, edgezero_config_store); + match get_settings() { + Ok(settings) if edgezero_can_handle_settings(&settings) => { + log::debug!("routing request through EdgeZero path"); + edgezero_main(req, edgezero_config_store); + } + Ok(_) => { + log::warn!( + "EdgeZero path does not yet support creative_opportunities; routing through legacy path" + ); + legacy_main(req); + } + Err(e) => { + log::warn!( + "failed to load settings for EdgeZero compatibility check, falling back to legacy path: {e:?}" + ); + legacy_main(req); + } + } } else { log::debug!("routing request through legacy path"); legacy_main(req); @@ -1340,6 +1360,36 @@ mod tests { .expect("should parse test settings") } + fn test_settings_with_creative_opportunities() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [creative_opportunities] + gam_network_id = "12345" + auction_timeout_ms = 500 + "#, + ) + .expect("should parse test settings with creative opportunities") + } + #[test] fn parses_true_flag_values() { assert!(parse_edgezero_flag("true"), "should parse 'true'"); @@ -1367,6 +1417,26 @@ mod tests { assert!(!parse_edgezero_flag("yes"), "should not parse 'yes'"); } + #[test] + fn edgezero_accepts_settings_without_creative_opportunities() { + let settings = test_settings(); + + assert!( + edgezero_can_handle_settings(&settings), + "should allow EdgeZero when server-side ad templates are not configured" + ); + } + + #[test] + fn edgezero_rejects_settings_with_creative_opportunities() { + let settings = test_settings_with_creative_opportunities(); + + assert!( + !edgezero_can_handle_settings(&settings), + "should route through legacy path while EdgeZero lacks server-side ad-template support" + ); + } + #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 9b1a73422..935c957ea 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -159,6 +159,7 @@ fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { .host_header_override(spec.host_header_override.as_deref()) .certificate_check(spec.certificate_check) .first_byte_timeout(spec.first_byte_timeout) + .between_bytes_timeout(spec.between_bytes_timeout) } impl PlatformBackend for FastlyPlatformBackend { @@ -676,6 +677,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -683,7 +685,7 @@ mod tests { .expect("should compute backend name for valid spec"); assert_eq!( - name, "backend_https_origin_example_com_443_t15000", + name, "backend_https_origin_example_com_443_fb15000_bb15000", "should match BackendConfig naming convention" ); } @@ -698,6 +700,7 @@ mod tests { host_header_override: Some("www.example.com".to_string()), certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -705,7 +708,7 @@ mod tests { .expect("should compute backend name for host header override"); assert_eq!( - name, "backend_https_origin_example_com_443_oh_www_example_com_t15000", + name, "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb15000", "should match BackendConfig naming convention with host header override" ); } @@ -720,6 +723,7 @@ mod tests { host_header_override: None, certificate_check: false, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -742,6 +746,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let result = backend.predict_name(&spec); @@ -759,6 +764,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_millis(2000), + between_bytes_timeout: Duration::from_millis(2000), }; let name = backend @@ -766,8 +772,8 @@ mod tests { .expect("should compute name with custom timeout"); assert!( - name.ends_with("_t2000"), - "should encode 2000ms timeout in name" + name.ends_with("_fb2000_bb2000"), + "should encode 2000ms first-byte and between-bytes timeouts in name" ); } diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 18efd1865..53656568a 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -389,9 +389,9 @@ impl AuctionOrchestrator { } // Give each provider only the remaining time from the auction - // deadline so that its backend first_byte_timeout doesn't extend - // past the overall budget. Also respect the provider's own - // configured timeout when it is tighter than the remaining budget. + // deadline so that backend transport timeouts do not extend past + // the overall budget. Also respect the provider's own configured + // timeout when it is tighter than the remaining budget. let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); let effective_timeout = remaining_ms.min(provider.timeout_ms()); @@ -488,10 +488,11 @@ impl AuctionOrchestrator { // Enforce the auction deadline: after each select() returns, check // elapsed time and drop remaining requests if the timeout is exceeded. // - // NOTE: `select()` blocks until at least one backend responds (or its - // transport timeout fires). Hard deadline enforcement therefore depends - // on every backend's `first_byte_timeout` being set to at most the - // remaining auction budget — which Phase 1 above guarantees. + // NOTE: `select()` blocks until at least one backend responds and, on + // some adapters, buffers the selected response body before returning. + // Hard deadline enforcement therefore depends on every backend's + // first-byte and between-bytes timeouts being set to at most the + // remaining auction budget, which Phase 1 above guarantees. let mut remaining = pending_requests; while !remaining.is_empty() { @@ -976,12 +977,13 @@ impl AuctionOrchestrator { } } - // Drain every dispatched request. Each backend was capped with a - // first-byte timeout at dispatch time, so by the collect phase the - // remaining handles may already be ready even if wall-clock time - // elapsed while the origin was slow — dropping them here would - // discard SSP responses that already arrived. The mediator launch - // below still observes A_deadline via `remaining_budget_ms`. + // Drain every dispatched request. Each backend was capped with + // first-byte and between-bytes timeouts at dispatch time, so by the + // collect phase the remaining handles may already be ready even if + // wall-clock time elapsed while the origin was slow. Dropping them + // here would discard SSP responses that already arrived. The + // mediator launch below still observes A_deadline via + // `remaining_budget_ms`. } let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { @@ -990,11 +992,11 @@ impl AuctionOrchestrator { // Cap the mediator at whichever is tighter: its own configured // timeout or the remaining auction budget (A_deadline). The old // comment here claimed origin drain could exhaust the budget before - // collection, but SSP backends are given first_byte_timeout = - // effective_timeout (capped at their provider timeout) at dispatch - // time, so they cannot run past A_deadline independently. Giving - // the mediator an uncapped timeout lets it run past A_deadline, - // violating the bounded hold invariant. + // collection, but SSP backends are given first-byte and between-bytes + // timeouts equal to effective_timeout (capped at their provider + // timeout) at dispatch time, so they cannot run past A_deadline + // independently. Giving the mediator an uncapped timeout lets it run + // past A_deadline, violating the bounded hold invariant. let remaining = remaining_budget_ms(auction_start, timeout_ms); if remaining == 0 { log::warn!( diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index fbc64f776..fa096d59d 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -173,6 +173,7 @@ pub fn dispatch_pull_sync( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) { Ok(name) => name, Err(err) => { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4ae15c927..717ad46e8 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -159,6 +159,7 @@ impl DataDomeIntegration { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), + between_bytes_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), }; services.backend().ensure(&spec).change_context(Self::error( diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 3678f949c..7777b79d4 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -152,6 +152,7 @@ fn integration_backend_spec( host_header_override: None, certificate_check, first_byte_timeout, + between_bytes_timeout: first_byte_timeout, }) } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index f81268724..5aa7827a7 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1080,13 +1080,13 @@ impl PrebidAuctionProvider { // Build user object — populate consent at both OpenRTB 2.6 top-level // and Prebid ext-based locations (dual placement). - // In cookies_only mode, body consent fields are omitted — consent - // travels exclusively through the forwarded Cookie header. - let consent_ctx = if self.config.consent_forwarding.includes_body_consent() { - request.user.consent.as_ref() - } else { - None - }; + // In cookies_only mode, cookie-sourced consent travels through the + // forwarded Cookie header. KV/policy-sourced consent has no inbound + // cookie to forward, so carry it in the OpenRTB body instead. + let consent_ctx = request.user.consent.as_ref().filter(|ctx| { + self.config.consent_forwarding.includes_body_consent() + || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) + }); let raw_tc = consent_ctx.and_then(|c| c.raw_tc_string.clone()); let user = Some(User { id: request.user.id.clone(), @@ -1809,7 +1809,7 @@ mod tests { AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, }; - use crate::consent::ConsentContext; + use crate::consent::{ConsentContext, ConsentSource}; use crate::geo::GeoInfo; use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; use crate::integrations::{ @@ -1863,10 +1863,11 @@ mod tests { spec: &PlatformBackendSpec, ) -> Result> { Ok(format!( - "predicted_{}_{}_{}", + "predicted_{}_{}_{}_{}", spec.scheme, spec.host, - spec.first_byte_timeout.as_millis() + spec.first_byte_timeout.as_millis(), + spec.between_bytes_timeout.as_millis() )) } @@ -1902,8 +1903,8 @@ mod tests { .expect("should predict backend name through platform backend"); assert_eq!( - backend_name, "predicted_https_prebid.example_123", - "should use PlatformBackend::predict_name instead of duplicating the naming scheme" + backend_name, "predicted_https_prebid.example_123_123", + "should cap both first-byte and between-bytes timeouts to the auction budget" ); } @@ -2713,6 +2714,49 @@ server_url = "https://prebid.example" ); } + #[test] + fn to_openrtb_includes_kv_consent_when_cookies_only_has_no_cookie_to_forward() { + let mut config = base_config(); + config.consent_forwarding = ConsentForwardingMode::CookiesOnly; + let provider = PrebidAuctionProvider::new(config); + let mut auction_request = create_test_auction_request(); + auction_request.user.consent = Some(ConsentContext { + raw_tc_string: Some("BOkv-backed-consent-string".to_string()), + raw_us_privacy: Some("1YNN".to_string()), + gdpr_applies: true, + source: ConsentSource::KvStore, + ..Default::default() + }); + + let settings = make_settings(); + let request = build_test_request(); + assert!( + !request.headers().contains_key(header::COOKIE), + "test request should not carry a consent cookie to forward" + ); + let context = create_test_auction_context(&settings, &request); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert_eq!( + openrtb.user.as_ref().and_then(|u| u.consent.as_deref()), + Some("BOkv-backed-consent-string"), + "cookies_only should fall back to body consent when consent came from KV" + ); + let regs = openrtb.regs.as_ref().expect("should include consent regs"); + assert_eq!(regs.gdpr, Some(true), "should carry GDPR applicability"); + assert_eq!( + regs.us_privacy.as_deref(), + Some("1YNN"), + "should carry non-cookie consent strings from KV" + ); + } + #[test] fn to_openrtb_sets_gdpr_true_for_non_eu_country_with_consent() { // When geo says non-GDPR but a consent string is present, the consent diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 0b14afe65..d744060cf 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -808,6 +808,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }; let name = stub.ensure(&spec).expect("should return a backend name"); assert_eq!(name, "stub-backend", "should return fixed name"); diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 77f7d6c5e..23f57a580 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -139,6 +139,8 @@ pub struct PlatformBackendSpec { pub certificate_check: bool, /// Maximum time to wait for the first response byte. pub first_byte_timeout: Duration, + /// Maximum time to wait between response body bytes. + pub between_bytes_timeout: Duration, } /// Cloneable container of platform services for a single request. diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index f8f6af4ca..8e9072d48 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1073,6 +1073,7 @@ pub async fn handle_asset_proxy_request( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "asset backend registration failed".to_string(), @@ -1256,6 +1257,7 @@ async fn proxy_with_redirects( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 95c3cfeaa..2f79114ed 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1150,6 +1150,7 @@ pub async fn handle_publisher_request( host_header_override: settings.publisher.origin_host_header_override.clone(), certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), From ae17b45a3fc7d237c14797b06bfbafe9e608ad11 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 25 Jun 2026 22:13:11 +0530 Subject: [PATCH 123/195] Fix EdgeZero empty ad-template config gate --- .../trusted-server-adapter-fastly/src/main.rs | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 017f1a9ef..423e4d1f5 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -144,7 +144,10 @@ fn is_edgezero_enabled(config_store: &ConfigStoreHandle) -> Result bool { - settings.creative_opportunities.is_none() + settings + .creative_opportunities + .as_ref() + .is_none_or(|creative_opportunities| creative_opportunities.slot.is_empty()) } fn health_response(req: &FastlyRequest) -> Option { @@ -205,7 +208,7 @@ fn main() { } Ok(_) => { log::warn!( - "EdgeZero path does not yet support creative_opportunities; routing through legacy path" + "EdgeZero path does not yet support configured creative_opportunity slots; routing through legacy path" ); legacy_main(req); } @@ -1360,7 +1363,7 @@ mod tests { .expect("should parse test settings") } - fn test_settings_with_creative_opportunities() -> Settings { + fn test_settings_with_empty_creative_opportunities() -> Settings { Settings::from_toml( r#" [[handlers]] @@ -1390,6 +1393,41 @@ mod tests { .expect("should parse test settings with creative opportunities") } + fn test_settings_with_configured_creative_opportunities() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [creative_opportunities] + gam_network_id = "12345" + auction_timeout_ms = 500 + + [[creative_opportunities.slot]] + id = "atf" + page_patterns = ["/article/*"] + formats = [{ width = 300, height = 250 }] + "#, + ) + .expect("should parse test settings with configured creative opportunities") + } + #[test] fn parses_true_flag_values() { assert!(parse_edgezero_flag("true"), "should parse 'true'"); @@ -1428,8 +1466,18 @@ mod tests { } #[test] - fn edgezero_rejects_settings_with_creative_opportunities() { - let settings = test_settings_with_creative_opportunities(); + fn edgezero_accepts_settings_with_empty_creative_opportunities() { + let settings = test_settings_with_empty_creative_opportunities(); + + assert!( + edgezero_can_handle_settings(&settings), + "should allow EdgeZero when server-side ad templates are configured but no slots are enabled" + ); + } + + #[test] + fn edgezero_rejects_settings_with_configured_creative_opportunity_slots() { + let settings = test_settings_with_configured_creative_opportunities(); assert!( !edgezero_can_handle_settings(&settings), From a11f94689279bb4ba8f686a2cd74572b0341ad3d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 29 Jun 2026 22:56:17 +0530 Subject: [PATCH 124/195] Address fourth-pass PR review findings Blocking: - Move tokio to [dev-dependencies] in trusted-server-core; it was only used by #[tokio::test] and was linking the runtime into the wasm prod build. Confirmed the release wasm adapter build no longer pulls tokio. - Roll back the SPA currentPath on a failed /__ts/page-bids fetch so a transient error no longer permanently strands that route (gpt/index.ts). Build/runtime parity and diagnostics: - Bound creative-opportunity format width/height to u32 range at build time so values the runtime u32 cannot hold are rejected early. - Add #[serde(deny_unknown_fields)] to the build.rs config stub to match the runtime type and reject mistyped table keys at build time. - Warn when the end-tag handler is absent so a silently non-rendering server-side ad feature is diagnosable. - Log dropped slot bidders that are neither configured nor the aps provider. - Log build_bid_index collisions (multiple bids per seat/imp). JS correctness: - Narrow uid.atype to a number before the range check in sanitizeAuctionUid. - Resolve findInjectedSlotForRefresh by exact/container match before the prefix fallback, with a regression test for prefix-overlapping div_ids. - Guard the gpt_bootstrap prefix scan against an empty div_id. - Route injectAdmIntoSlot through findSlotElementByDivId for consistency. Cleanup and docs: - Remove the dead has_post_processors routing dependency from classify_response_route and (now unused) handle_publisher_request. - Extract the duplicated EID resolution/consent-gating/device tail shared by the initial-page and page-bids dispatch paths into one helper. - Anchor the surrogate cache-header list in a shared const so the legacy and EdgeZero Set-Cookie privacy paths stay aligned. - Refresh stale docs (PublisherResponse::Stream, the publisher module platform-coupling note, and UserInfo.eids consent-gate location). --- crates/trusted-server-adapter-axum/src/app.rs | 1 - .../src/app.rs | 1 - .../trusted-server-adapter-fastly/src/app.rs | 1 - .../trusted-server-adapter-fastly/src/main.rs | 6 +- .../src/middleware.rs | 11 +- crates/trusted-server-adapter-spin/src/app.rs | 1 - crates/trusted-server-core/Cargo.toml | 2 +- crates/trusted-server-core/build.rs | 1 + .../trusted-server-core/src/auction/types.rs | 6 +- .../src/creative_slot_build_check.rs | 10 +- .../trusted-server-core/src/html_processor.rs | 9 + .../src/integrations/adserver_mock.rs | 24 +- .../src/integrations/gpt_bootstrap.js | 1 + .../src/integrations/prebid.rs | 11 + crates/trusted-server-core/src/publisher.rs | 262 +++++++++--------- .../lib/src/integrations/gpt/index.ts | 16 +- .../lib/src/integrations/prebid/index.ts | 24 +- .../test/integrations/prebid/index.test.ts | 57 ++++ 18 files changed, 276 insertions(+), 168 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 882aa69c8..8cd53d48f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -176,7 +176,6 @@ async fn dispatch_fallback( }; handle_publisher_request( &state.settings, - &state.registry, services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 92b1c17e7..d4e8fb3d6 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -305,7 +305,6 @@ fn build_router(state: &Arc) -> RouterService { }; handle_publisher_request( &state.settings, - &state.registry, &services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 8d2d9ae13..13a348314 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -729,7 +729,6 @@ async fn dispatch_fallback( }; handle_publisher_request( &state.settings, - &state.registry, &publisher_services, ec.kv_graph.as_ref(), &mut ec.ec_context, diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index e1c0da9a5..30e91b01f 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1270,7 +1270,6 @@ async fn route_request( match handle_publisher_request( settings, - integration_registry, runtime_services, kv_graph.as_ref(), &mut ec_context, @@ -1399,8 +1398,9 @@ fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { // keeping a stricter `no-store`/`private` directive — Surrogate-Control is // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - response.remove_header("surrogate-control"); - response.remove_header("fastly-surrogate-control"); + for name in crate::middleware::SURROGATE_CACHE_HEADERS { + response.remove_header(*name); + } let already_uncacheable = response .get_header_str("cache-control") .map(str::to_ascii_lowercase) diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 6aa9cddd4..91b7dcdec 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -257,6 +257,12 @@ pub(crate) fn apply_finalize_headers( } } +/// Surrogate cache headers stripped from every cookie-bearing response. A single +/// source of truth so the legacy ([`crate::enforce_set_cookie_cache_privacy`]) +/// and `EdgeZero` copies of the privacy downgrade cannot drift apart. +pub(crate) const SURROGATE_CACHE_HEADERS: &[&str] = + &["surrogate-control", "fastly-surrogate-control"]; + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Mirrors [`crate::enforce_set_cookie_cache_privacy`] for the [`Response`] type @@ -277,8 +283,9 @@ pub(crate) fn enforce_set_cookie_cache_privacy(response: &mut Response) { // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); + for name in SURROGATE_CACHE_HEADERS { + response.headers_mut().remove(*name); + } // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. let already_uncacheable = response diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index b1b341c17..287bbfd8a 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -607,7 +607,6 @@ fn build_router(state: &Arc) -> RouterService { }; handle_publisher_request( &state.settings, - &state.registry, &services, None, &mut ec_context, diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index cdb280ed1..ab62f53e8 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -40,7 +40,6 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } subtle = { workspace = true } -tokio = { workspace = true } toml = { workspace = true } trusted-server-js = { path = "../trusted-server-js" } trusted-server-openrtb = { path = "../trusted-server-openrtb" } @@ -83,6 +82,7 @@ test-utils = [] criterion = { workspace = true } edgezero-core = { workspace = true, features = ["test-utils"] } temp-env = { workspace = true } +tokio = { workspace = true } [[bench]] name = "consent_decode" diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index a95c307a1..ef6546285 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -53,6 +53,7 @@ mod creative_opportunities { } #[derive(Debug, Clone, Deserialize, Serialize)] + #[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { pub gam_network_id: String, #[serde(default)] diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 2a2985926..ffe918aa4 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -87,8 +87,10 @@ pub struct UserInfo { /// Extended User IDs parsed from the [`crate::constants::COOKIE_TS_EIDS`] cookie. /// /// Raw (un-gated) values from the browser; consent gating via - /// [`crate::consent::gate_eids_by_consent`] is applied in the provider - /// layer before any EID reaches a bid request. + /// [`crate::consent::gate_eids_by_consent`] is applied centrally in the + /// endpoint handlers (the auction and page-bids paths) before any EID + /// reaches a bid request — the provider layer just forwards already-gated + /// EIDs. #[serde(skip)] pub eids: Option>, } diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 15e5ca98a..066cdde1a 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -337,9 +337,15 @@ pub(crate) fn validate_creative_slot( for format in formats { let width = format.get("width").and_then(serde_json::Value::as_u64); let height = format.get("height").and_then(serde_json::Value::as_u64); - if !matches!((width, height), (Some(w), Some(h)) if w > 0 && h > 0) { + // Runtime dimensions are `u32`, so a value above `u32::MAX` passes + // a bare `> 0` check here but fails `from_value::` at runtime + // settings load on every request — the exact failure this build + // check exists to prevent. + let in_u32 = + |v: Option| matches!(v, Some(n) if n > 0 && n <= u64::from(u32::MAX)); + if !(in_u32(width) && in_u32(height)) { return Err(format!( - "slot `{id}` format must have positive width and height" + "slot `{id}` format must have positive width and height within u32 range" )); } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 627468adc..a3170084a 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -366,6 +366,15 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso Ok(()) }); handlers.push(handler); + } else { + // No end tag (implicitly closed or EOF ``): lol_html + // cannot attach an end-tag handler, so tsjs.bids/adInit() are + // never injected even though adSlots was injected at ``. + // The whole server-side ad feature then silently fails to + // render — warn so the failure is diagnosable. + log::warn!( + "`` has no end tag (implicitly closed or EOF); tsjs.bids and adInit() were not injected — server-side ads will not render" + ); } Ok(()) } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 4fff04660..bd3538c71 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -108,14 +108,24 @@ fn build_bid_index(bidder_responses: &[AuctionResponse]) -> BidIndex { let mut index = BidIndex::new(); for response in bidder_responses { for bid in &response.bids { - index.insert( - ( - response.provider.clone(), - bid.slot_id.clone(), - bid.bidder.clone(), - ), - bid.clone(), + let key = ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), ); + // OpenRTB permits a seat to return multiple bids per imp. This index + // is last-write-wins, so a collision means an earlier bid's + // nurl/burl/cache_* are dropped and win/billing-URL restoration can + // be mis-attributed during mediation. Low severity for the mock + // mediator, but log it so the collision is visible. + if index.insert(key, bid.clone()).is_some() { + log::debug!( + "adserver_mock: duplicate bid for (provider '{}', slot '{}', bidder '{}'); keeping the last — win/billing URL restoration may be mis-attributed", + response.provider, + bid.slot_id, + bid.bidder + ); + } } } index diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index f40283e87..cc4c5c00c 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -68,6 +68,7 @@ for (var i = 0; i < idElements.length; i++) { var candidate = idElements[i]; if ( + slot.div_id && candidate.id.startsWith(slot.div_id) && !candidate.id.endsWith("-container") ) { diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 6e0703d45..451650611 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1017,6 +1017,17 @@ impl PrebidAuctionProvider { bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); } else if self.config.bidders.iter().any(|b| b == name) { bidder.insert(name.clone(), params.clone()); + } else if name != "aps" { + // `aps` is intentionally handled by its own provider. Any + // other unrecognized key is likely a misconfiguration (a + // slot bidder absent from `config.bidders`) that silently + // yields an empty bidder map and a stored-request no-bid — + // log it so the drop is diagnosable. + log::debug!( + "prebid: dropping slot '{}' bidder '{}' — not in config.bidders and not a known provider key", + slot.id, + name + ); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 3ce825c86..b85a5ad22 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -10,21 +10,19 @@ //! streaming processor treats unknown encodings as identity, so publisher code //! must gate them out before the body enters the rewrite pipeline. //! -//! **Note on platform coupling:** This module is currently coupled to -//! `fastly::Body`/`Request`/`Response` at its handler boundaries — the entry -//! points ([`handle_publisher_request`], [`stream_publisher_body`]) still -//! accept and return `fastly::Body` and `fastly::Response`. The streaming -//! processor itself is generic: `process_response_streaming` writes into -//! any [`Write`] (a `Vec` for buffered routes, a `StreamingBody` for the -//! streaming route). The HTTP-type coupling will be addressed in the -//! platform HTTP-type migration alongside all other -//! `fastly::Request`/`Response`/`Body` migrations. It is not a -//! content-rewriting concern. +//! **Note on platform coupling:** The handler boundaries use portable HTTP +//! types: [`handle_publisher_request`] and [`stream_publisher_body`] take and +//! return `http::Request`/`http::Response` over `EdgeBody`, and platform I/O is +//! reached through `RuntimeServices` rather than `fastly::*` directly. The +//! streaming processor itself is generic: `process_response_streaming` writes +//! into any [`Write`] (a `Vec` for buffered routes, a streaming writer for +//! the streaming route). It is not a content-rewriting concern. use std::io::Write; use std::sync::{Arc, Mutex}; use std::time::Duration; +use cookie::CookieJar; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{header, HeaderValue, Method, Request, Response, StatusCode, Uri}; @@ -45,7 +43,7 @@ use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::http_util::{is_navigation_request, serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; use crate::price_bucket::{price_bucket, PriceGranularity}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; @@ -333,9 +331,9 @@ pub enum PublisherResponse { Buffered(Response), /// Response headers are ready for a streaming response. Covers processable /// content on any status (2xx or non-2xx — e.g., branded 404/500 HTML and - /// error JSON still get URL rewriting) where the encoding is supported - /// and either the content is non-HTML or no HTML post-processors are - /// registered. The caller must: + /// error JSON still get URL rewriting) where the encoding is supported. + /// Post-processors run inside the streaming processor, so processable HTML + /// is streamed regardless of whether any are registered. The caller must: /// 1. Call `finalize_response()` on the response /// 2. Call `response.stream_to_client()` to get a `StreamingBody` /// 3. Call `stream_publisher_body()` with the body and streaming writer @@ -398,7 +396,6 @@ pub(crate) fn classify_response_route( content_type: &str, content_encoding: &str, request_host: &str, - _has_post_processors: bool, ) -> ResponseRoute { if status == StatusCode::NO_CONTENT || status == StatusCode::RESET_CONTENT { return ResponseRoute::BufferedUnmodified; @@ -1150,7 +1147,6 @@ pub struct AuctionDispatch<'a> { /// origin backend is unreachable. pub async fn handle_publisher_request( settings: &Settings, - integration_registry: &IntegrationRegistry, services: &RuntimeServices, kv: Option<&KvIdentityGraph>, ec_context: &mut EcContext, @@ -1307,33 +1303,19 @@ pub async fn handle_publisher_request( .get("user-agent") .and_then(|v| v.to_str().ok()), ); - let ts_eids_value = cookie_jar - .as_ref() - .and_then(|j| j.get(COOKIE_TS_EIDS)) - .map(|c| c.value().to_owned()); - let client_eids = if ec_id.is_some() { - resolve_client_auction_eids(None, ts_eids_value.as_deref()) - } else { - None - }; - let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); - let merged_eids = merge_auction_eids(client_eids, kv_eids); - let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); - if had_eids && auction_request.user.eids.is_none() { - log::warn!("Server-side auction EIDs stripped by TCF consent gating"); - } - let client_ip = services.client_info().client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: cookie_jar.as_ref(), + ec_id, + kv, + partner_registry: auction.registry, + ec_context, + services, + geo: geo.as_ref(), + path_label: "Server-side", + }, + ); let auction_context = AuctionContext { settings, request: &req, @@ -1439,15 +1421,7 @@ pub async fn handle_publisher_request( .map(|h| h.to_str().unwrap_or_default()) .unwrap_or_default() .to_lowercase(); - let has_post_processors = integration_registry.has_html_post_processors(); - - let route = classify_response_route( - status, - &content_type, - &content_encoding, - request_host, - has_post_processors, - ); + let route = classify_response_route(status, &content_type, &content_encoding, request_host); match route { ResponseRoute::PassThrough => { @@ -1541,6 +1515,70 @@ pub(crate) struct MatchedSlotsContext<'a> { pub request_path: &'a str, } +/// Borrowed inputs for [`apply_auction_eids_and_device`], bundled to keep the +/// helper within the project's 7-argument cap. +struct AuctionEidTargeting<'a> { + cookie_jar: Option<&'a CookieJar>, + ec_id: Option<&'a str>, + kv: Option<&'a KvIdentityGraph>, + partner_registry: Option<&'a PartnerRegistry>, + ec_context: &'a EcContext, + services: &'a RuntimeServices, + geo: Option<&'a GeoInfo>, + /// Prefix for the consent-stripped warning (e.g. `"Server-side"`). + path_label: &'a str, +} + +/// Resolves client + KV EIDs, consent-gates them onto `auction_request`, and +/// attaches the client IP/geo to its device record. +/// +/// Shared verbatim by the initial-page and page-bids dispatch paths so the EID +/// resolution and consent gating live in one place; `path_label` only varies +/// the consent-stripped warning message. +fn apply_auction_eids_and_device( + auction_request: &mut AuctionRequest, + targeting: &AuctionEidTargeting<'_>, +) { + let ts_eids_value = targeting + .cookie_jar + .and_then(|j| j.get(COOKIE_TS_EIDS)) + .map(|c| c.value().to_owned()); + let client_eids = if targeting.ec_id.is_some() { + resolve_client_auction_eids(None, ts_eids_value.as_deref()) + } else { + None + }; + let kv_eids = resolve_auction_eids( + targeting.kv, + targeting.partner_registry, + targeting.ec_context, + ); + let merged_eids = merge_auction_eids(client_eids, kv_eids); + let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); + auction_request.user.eids = + gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + if had_eids && auction_request.user.eids.is_none() { + log::warn!( + "{} auction EIDs stripped by TCF consent gating", + targeting.path_label + ); + } + let client_ip = targeting + .services + .client_info() + .client_ip + .map(|ip| ip.to_string()); + if client_ip.is_some() || targeting.geo.is_some() { + let device = auction_request.device.get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }); + device.ip = client_ip; + device.geo = targeting.geo.cloned(); + } +} + /// Build an [`AuctionRequest`] from matched creative opportunity slots. pub(crate) fn build_auction_request( slots_ctx: &MatchedSlotsContext<'_>, @@ -1948,33 +1986,19 @@ pub async fn handle_page_bids( .get("user-agent") .and_then(|v| v.to_str().ok()), ); - let ts_eids_value = cookie_jar - .as_ref() - .and_then(|j| j.get(COOKIE_TS_EIDS)) - .map(|c| c.value().to_owned()); - let client_eids = if ec_id.is_some() { - resolve_client_auction_eids(None, ts_eids_value.as_deref()) - } else { - None - }; - let kv_eids = resolve_auction_eids(kv, auction.registry, ec_context); - let merged_eids = merge_auction_eids(client_eids, kv_eids); - let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); - if had_eids && auction_request.user.eids.is_none() { - log::warn!("Page-bids auction EIDs stripped by TCF consent gating"); - } - let client_ip = services.client_info().client_ip.map(|ip| ip.to_string()); - if client_ip.is_some() || geo.is_some() { - let device = auction_request.device.get_or_insert(DeviceInfo { - user_agent: None, - ip: None, - geo: None, - }); - device.ip = client_ip; - device.geo = geo.clone(); - } + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: cookie_jar.as_ref(), + ec_id, + kv, + partner_registry: auction.registry, + ec_context, + services, + geo: geo.as_ref(), + path_label: "Page-bids", + }, + ); let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); @@ -2306,10 +2330,9 @@ mod tests { /// Drive `handle_publisher_request` with no creative opportunities — a plain /// proxy with no server-side auction. Hides the auction/EC wiring so callers - /// read like a simple `(settings, registry, services, req)` proxy. + /// read like a simple `(settings, services, req)` proxy. async fn run_publisher_proxy( settings: &Settings, - integration_registry: &IntegrationRegistry, services: &RuntimeServices, req: Request, ) -> PublisherResponse { @@ -2318,7 +2341,6 @@ mod tests { EcContext::read_from_request(settings, &req, services).expect("should read EC context"); handle_publisher_request( settings, - integration_registry, services, None, &mut ec_context, @@ -2336,8 +2358,6 @@ mod tests { #[tokio::test] async fn publisher_request_uses_platform_http_client_with_http_types() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"origin response".to_vec()); let services = build_services_with_http_client( @@ -2350,7 +2370,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let response = match run_publisher_proxy(&settings, ®istry, &services, req).await { + let response = match run_publisher_proxy(&settings, &services, req).await { PublisherResponse::Buffered(r) => r, PublisherResponse::PassThrough { mut response, body } => { *response.body_mut() = body; @@ -2378,8 +2398,6 @@ mod tests { // exactly the conditions under which the old inline call would have // generated one. let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"ok".to_vec()); let services = build_services_with_http_client( @@ -2408,7 +2426,6 @@ mod tests { let _ = handle_publisher_request( &settings, - ®istry, &services, None, &mut ec_context, @@ -2647,8 +2664,7 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "zstd", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, ); @@ -2702,8 +2718,7 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -2716,8 +2731,7 @@ mod tests { StatusCode::OK, "Text/HTML; Charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, "HTML MIME type matching must be case-insensitive", @@ -2731,8 +2745,7 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::Stream, ); @@ -2741,13 +2754,7 @@ mod tests { #[test] fn route_streams_non_html_even_with_post_processors_registered() { assert_eq!( - classify_response_route( - StatusCode::OK, - "application/json", - "gzip", - "example.com", - true, - ), + classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), ResponseRoute::Stream, ); } @@ -2755,7 +2762,7 @@ mod tests { #[test] fn route_buffers_unmodified_on_unsupported_encoding() { assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com", false,), + classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2763,7 +2770,7 @@ mod tests { #[test] fn route_passes_through_non_processable_2xx() { assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "example.com", false,), + classify_response_route(StatusCode::OK, "image/png", "", "example.com"), ResponseRoute::PassThrough, ); } @@ -2771,7 +2778,7 @@ mod tests { #[test] fn route_buffers_non_processable_error_responses() { assert_eq!( - classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com", false,), + classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2779,13 +2786,7 @@ mod tests { #[test] fn route_excludes_204_from_pass_through() { assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "image/png", - "", - "example.com", - false, - ), + classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2793,13 +2794,7 @@ mod tests { #[test] fn route_excludes_205_from_pass_through() { assert_eq!( - classify_response_route( - StatusCode::RESET_CONTENT, - "image/png", - "", - "example.com", - false, - ), + classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } @@ -2811,8 +2806,7 @@ mod tests { StatusCode::NO_CONTENT, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, "204 + HTML must not route to Stream", @@ -2822,8 +2816,7 @@ mod tests { StatusCode::NO_CONTENT, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::BufferedUnmodified, "204 + HTML + post-processors must not route to Stream", @@ -2837,8 +2830,7 @@ mod tests { StatusCode::RESET_CONTENT, "application/json", "", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, "205 + JSON must not route to Stream", @@ -2852,8 +2844,7 @@ mod tests { StatusCode::NOT_FOUND, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -2862,8 +2853,7 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR, "application/json", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -2876,8 +2866,7 @@ mod tests { StatusCode::NOT_FOUND, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::Stream, ); @@ -2886,7 +2875,7 @@ mod tests { #[test] fn route_passes_through_non_processable_even_with_empty_request_host() { assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "", false,), + classify_response_route(StatusCode::OK, "image/png", "", ""), ResponseRoute::PassThrough, ); } @@ -2894,7 +2883,7 @@ mod tests { #[test] fn route_buffers_processable_content_with_empty_request_host() { assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "gzip", "", false,), + classify_response_route(StatusCode::OK, "text/html", "gzip", ""), ResponseRoute::BufferedUnmodified, ); } @@ -3186,8 +3175,6 @@ mod tests { async fn publisher_request_sends_configured_host_header_override() { let mut settings = create_test_settings(); settings.publisher.origin_host_header_override = Some("www.example.com".to_string()); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"origin response".to_vec()); let services = build_services_with_http_client( @@ -3200,7 +3187,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let _ = run_publisher_proxy(&settings, ®istry, &services, req).await; + let _ = run_publisher_proxy(&settings, &services, req).await; let recorded_headers = stub.recorded_request_headers(); let outbound_headers = recorded_headers @@ -3465,7 +3452,6 @@ mod tests { "text/html; charset=utf-8", "", "proxy.example.com", - registry.has_html_post_processors(), ), ResponseRoute::Stream, "HTML with post-processors must route to Stream" diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 73b9419c6..22e8a1547 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -259,8 +259,9 @@ export function installGptShim(): boolean { function injectAdmIntoSlot(divId: string, adm: string): void { try { // divId may be the container div (used by GPT slot) or the inner div. - // Search both so we can find the GAM iframe wherever it was rendered. - const slotEl = document.getElementById(divId); + // Resolve it the same way the rest of adInit does (exact then prefix) so + // a config div_id prefix with a render-time suffix still finds the element. + const slotEl = findSlotElementByDivId(divId); if (!slotEl) return; // Extract the first iframe src from the adm (e.g. mocktioneer creative @@ -679,6 +680,7 @@ export function installSpaAuctionHook(): void { async function onNavigate(path: string): Promise { if (path === currentPath) return; + const previousPath = currentPath; currentPath = path; inflight?.abort(); const controller = new AbortController(); @@ -694,7 +696,14 @@ export function installSpaAuctionHook(): void { headers: { 'X-TSJS-Page-Bids': '1' }, signal: controller.signal, }); - if (!res.ok) return; + if (!res.ok) { + // A transient page-bids failure must not strand this route: roll the + // committed path back so a later navigation here retries instead of + // being skipped by the no-op guard at the top. Only roll back when no + // newer navigation has already advanced currentPath. + if (inflight === controller) currentPath = previousPath; + return; + } const data = (await res.json()) as PageBidsResponse; if (inflight !== controller) return; // Defer applying bids until the new route's ad containers exist, so a @@ -716,6 +725,7 @@ export function installSpaAuctionHook(): void { } } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; + if (inflight === controller) currentPath = previousPath; log.warn('SPA auction hook: fetch failed', err); } } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 40a8d9e2e..f6d55fb4a 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -270,7 +270,12 @@ function sanitizeAuctionUid(uid: { const sanitizedUid: AuctionEid['uids'][number] = { id: uid.id }; - if (Number.isInteger(uid.atype) && uid.atype >= 0 && uid.atype <= 255) { + if ( + typeof uid.atype === 'number' && + Number.isInteger(uid.atype) && + uid.atype >= 0 && + uid.atype <= 255 + ) { sanitizedUid.atype = uid.atype; } @@ -330,11 +335,18 @@ function findInjectedSlotForRefresh(slot: RefreshGptSlot): AuctionSlot | undefin return undefined; } - return window.tsjs?.adSlots?.find( - (adSlot) => - elementId === adSlot.div_id || - elementId === `${adSlot.div_id}-container` || - elementId.startsWith(adSlot.div_id) + const slots = window.tsjs?.adSlots; + if (!slots) { + return undefined; + } + + // Prefer an exact (or container) match across all slots before the prefix + // fallback, so prefix-overlapping div_ids (e.g. "ad" and "ad-header") resolve + // to the correct slot instead of the first slot whose div_id is a prefix. + return ( + slots.find( + (adSlot) => elementId === adSlot.div_id || elementId === `${adSlot.div_id}-container` + ) ?? slots.find((adSlot) => adSlot.div_id.length > 0 && elementId.startsWith(adSlot.div_id)) ); } diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 738a1cc78..6d52c368f 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -864,6 +864,63 @@ describe('prebid/installRefreshHandler', () => { ); }); + it('resolves the exact slot when div_ids share a prefix', () => { + // Regression: a single find() with a startsWith() clause returned the + // first slot whose div_id is a prefix of the element id. With div_ids + // "div-ad" and "div-ad-header", refreshing the "div-ad-header" element + // must resolve to the header slot, not the shorter prefix slot. + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'prefix_ad', + gam_unit_path: '/123/prefix', + div_id: 'div-ad', + formats: [[300, 250]], + targeting: { zone: 'prefix' }, + }, + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[970, 250]], + targeting: { zone: 'header' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-header', + mediaTypes: { + banner: { + name: 'header', + sizes: [[970, 250]], + }, + }, + }), + ], + }) + ); + }); + it('scopes the GPT targeting call to the refreshed slot code', () => { const setTargetingForGPTAsync = vi.fn(); (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; From ae9d50a2cad9eabd52e55fa390de73c48fc4a540 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 29 Jun 2026 23:03:45 +0530 Subject: [PATCH 125/195] Drop tokio from the integration-tests lockfile Moving tokio to trusted-server-core dev-dependencies removed it from the crate's normal dependency list, so the integration-tests lockfile (which resolves core's non-dev deps) no longer pins tokio under core. Keeps `cargo --locked` green for the integration job. --- crates/trusted-server-integration-tests/Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/trusted-server-integration-tests/Cargo.lock b/crates/trusted-server-integration-tests/Cargo.lock index 4f7c723d3..6fa535a57 100644 --- a/crates/trusted-server-integration-tests/Cargo.lock +++ b/crates/trusted-server-integration-tests/Cargo.lock @@ -4593,7 +4593,6 @@ dependencies = [ "serde_json", "sha2", "subtle", - "tokio", "toml", "trusted-server-js", "trusted-server-openrtb", From 802e841896c01e5e8aebf15e92d47428132cd857 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 10:06:25 +0530 Subject: [PATCH 126/195] Run the server-side auction on the Axum, Cloudflare, and Spin adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These EdgeZero-style adapters finalize buffered, and the sync `buffer_publisher_response` drives `stream_publisher_body`, which ignores `params.dispatched_auction` — so they injected an empty `tsjs.bids = {}` while Fastly (legacy streaming finalize) served real bids. - Add `buffer_publisher_response_async` in core: for the Stream variant it drives `stream_publisher_body_async`, which awaits `collect_dispatched_auction`, writes `ad_bids_state`, and injects the bids before ``. - Pass the configured `creative_opportunities.slot` (not empty) to `handle_publisher_request` on all three adapters; it matches them against the request path internally. - Call the async finalize from each adapter (Cloudflare/Spin via their now-async `resolve_publisher_response`). EID targeting stays off for now (these adapters pass `kv: None`). --- crates/trusted-server-adapter-axum/src/app.rs | 28 +++++++-- .../src/app.rs | 46 ++++++++++++--- crates/trusted-server-adapter-spin/src/app.rs | 51 ++++++++++++---- crates/trusted-server-core/src/publisher.rs | 59 +++++++++++++++++++ 4 files changed, 158 insertions(+), 26 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 8cd53d48f..ceab9eac4 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -19,7 +19,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -166,15 +166,21 @@ async fn dispatch_fallback( }); } - // Server-side auction is deferred for the EdgeZero adapters: pass no slots - // so `handle_publisher_request` dispatches no auction. + // Run the server-side auction with the configured creative-opportunity + // slots; `handle_publisher_request` matches them against the request path. let mut ec_context = EcContext::default(); + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + let publisher_response = handle_publisher_request( &state.settings, services, None, @@ -182,8 +188,18 @@ async fn dispatch_fallback( auction, req, ) + .await?; + // Async finalize so the dispatched auction is collected and its bids are + // injected before `` (the sync buffer path would drop them). + buffer_publisher_response_async( + publisher_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) .await - .and_then(|pr| buffer_publisher_response(pr, &method, &state.settings, &state.registry)) } fn fallback_handler( diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index d4e8fb3d6..e62f3a535 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -19,7 +19,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response, handle_publisher_request, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, }; use trusted_server_core::request_signing::{ @@ -118,16 +118,27 @@ where /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// -/// Delegates to the shared [`buffer_publisher_response`], which enforces +/// Delegates to the shared [`buffer_publisher_response_async`], which collects +/// the dispatched server-side auction and enforces /// `settings.publisher.max_buffered_body_bytes`, then removes any /// `Transfer-Encoding` header since the buffered body is no longer chunked. -fn resolve_publisher_response( +async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, settings: &Settings, registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result> { - let mut response = buffer_publisher_response(publisher_response, method, settings, registry)?; + let mut response = buffer_publisher_response_async( + publisher_response, + method, + settings, + registry, + orchestrator, + services, + ) + .await?; response.headers_mut().remove(header::TRANSFER_ENCODING); Ok(response) } @@ -298,12 +309,18 @@ fn build_router(state: &Arc) -> RouterService { }) } else { let mut ec_context = EcContext::default(); + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + match handle_publisher_request( &state.settings, &services, None, @@ -312,9 +329,20 @@ fn build_router(state: &Arc) -> RouterService { req, ) .await - .and_then(|pr| { - resolve_publisher_response(pr, &method, &state.settings, &state.registry) - }) + { + Ok(pr) => { + resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + ) + .await + } + Err(e) => Err(e), + } }; Ok(result.unwrap_or_else(|e| http_error(&e))) diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 287bbfd8a..26852eacb 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -14,12 +14,13 @@ use trusted_server_core::ec::EcContext; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; +use trusted_server_core::platform::RuntimeServices; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response, handle_publisher_request, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, }; use trusted_server_core::request_signing::{ @@ -79,16 +80,27 @@ fn build_state_with_settings( /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// -/// Delegates to the shared [`buffer_publisher_response`], which enforces -/// `settings.publisher.max_buffered_body_bytes` so a large processable -/// origin response fails safely instead of exhausting the Wasm heap. -fn resolve_publisher_response( +/// Delegates to the shared [`buffer_publisher_response_async`], which collects +/// the dispatched server-side auction and enforces +/// `settings.publisher.max_buffered_body_bytes` so a large processable origin +/// response fails safely instead of exhausting the Wasm heap. +async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, settings: &Settings, registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result> { - buffer_publisher_response(publisher_response, method, settings, registry) + buffer_publisher_response_async( + publisher_response, + method, + settings, + registry, + orchestrator, + services, + ) + .await } // --------------------------------------------------------------------------- @@ -600,12 +612,18 @@ fn build_router(state: &Arc) -> RouterService { }) } else { let mut ec_context = EcContext::default(); + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + match handle_publisher_request( &state.settings, &services, None, @@ -614,9 +632,20 @@ fn build_router(state: &Arc) -> RouterService { req, ) .await - .and_then(|pr| { - resolve_publisher_response(pr, &method, &state.settings, &state.registry) - }) + { + Ok(pr) => { + resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + ) + .await + } + Err(e) => Err(e), + } }; Ok(result.unwrap_or_else(|e| http_error(&e))) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b85a5ad22..4f6f7823d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -495,6 +495,65 @@ pub fn buffer_publisher_response( } } +/// Async variant of [`buffer_publisher_response`] that collects the dispatched +/// server-side auction before buffering. +/// +/// The sync [`buffer_publisher_response`] drives [`stream_publisher_body`], +/// which ignores `params.dispatched_auction`, so its `` injection always +/// falls back to empty `tsjs.bids`. Adapters that finalize on an async runtime +/// (Axum, Cloudflare, Spin) call this instead: it drives +/// [`stream_publisher_body_async`], which awaits +/// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids +/// into `ad_bids_state`, and injects them before ``. +/// +/// # Errors +/// +/// Returns an error if the streaming pipeline fails to process the response +/// body, or if the processed body exceeds the configured buffer cap. +pub async fn buffer_publisher_response_async( + publisher_response: PublisherResponse, + method: &Method, + settings: &Settings, + integration_registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result, Report> { + match publisher_response { + PublisherResponse::Buffered(response) => Ok(response), + PublisherResponse::Stream { + mut response, + body, + mut params, + } => { + if !response_carries_body(method, response.status()) { + return Ok(response); + } + let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); + stream_publisher_body_async( + body, + &mut output, + &mut params, + settings, + integration_registry, + orchestrator, + services, + ) + .await?; + let bytes = output.into_inner(); + response.headers_mut().insert( + http::header::CONTENT_LENGTH, + http::HeaderValue::from(bytes.len() as u64), + ); + *response.body_mut() = EdgeBody::from(bytes); + Ok(response) + } + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + Ok(response) + } + } +} + /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// From 297efcd6f7e06dfaa58e9fefcb37f8f9ee710bba Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 10:42:16 +0530 Subject: [PATCH 127/195] Build the EC consent context from the request on the portability adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auction consent gate (`consent_allows_server_side_auction`) reads jurisdiction and TCF consent from the EC context. The adapters passed `EcContext::default()` to `handle_publisher_request`, leaving jurisdiction Unknown with no consent — so the gate failed closed and no auction ran (empty `tsjs.bids`), even though the slots matched. Build the context via `read_from_request_with_geo` (consent from the request, geo from the platform), mirroring the Fastly entry point, and fall back to default on a parse error. Cloudflare resolves geo from the Workers `cf` object when deployed; Axum and Spin have no-op geo providers, so on those a known non-GDPR jurisdiction requires the request to carry geo or the gate needs a TCF consent signal. --- crates/trusted-server-adapter-axum/src/app.rs | 16 ++++++++++++++- .../src/app.rs | 19 +++++++++++++++++- crates/trusted-server-adapter-spin/src/app.rs | 20 ++++++++++++++++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index ceab9eac4..953a0839f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -168,7 +168,21 @@ async fn dispatch_fallback( // Run the server-side auction with the configured creative-opportunity // slots; `handle_publisher_request` matches them against the request path. - let mut ec_context = EcContext::default(); + // Build the EC context (consent + jurisdiction) from the request like the + // Fastly entry point — `EcContext::default()` leaves jurisdiction Unknown, + // which fails the auction consent gate closed. Geo comes from the platform + // (no-op on the local Axum dev server, so jurisdiction stays Unknown there + // unless the request carries TCF consent). + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let mut ec_context = + EcContext::read_from_request_with_geo(&state.settings, &req, services, geo_info.as_ref()) + .unwrap_or_default(); let slots = state .settings .creative_opportunities diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index e62f3a535..670c09255 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -308,7 +308,24 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - let mut ec_context = EcContext::default(); + // Build the EC context (consent + jurisdiction) from the request + // like the Fastly entry point — `EcContext::default()` leaves + // jurisdiction Unknown and fails the auction consent gate closed. + // Geo comes from the Workers `cf` object when deployed. + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let mut ec_context = EcContext::read_from_request_with_geo( + &state.settings, + &req, + &services, + geo_info.as_ref(), + ) + .unwrap_or_default(); let slots = state .settings .creative_opportunities diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 26852eacb..d8a487d20 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -611,7 +611,25 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - let mut ec_context = EcContext::default(); + // Build the EC context (consent + jurisdiction) from the request + // like the Fastly entry point — `EcContext::default()` leaves + // jurisdiction Unknown and fails the auction consent gate closed. + // Spin's platform geo is a no-op, so jurisdiction stays Unknown + // unless the request carries TCF consent. + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + let mut ec_context = EcContext::read_from_request_with_geo( + &state.settings, + &req, + &services, + geo_info.as_ref(), + ) + .unwrap_or_default(); let slots = state .settings .creative_opportunities From c24cf5271d5236dcb8ebac81d13f80e0a3d237fd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 10:46:13 +0530 Subject: [PATCH 128/195] Log the server-side ad-stack gate inputs at debug When the auction does not run, this pinpoints which gate suppressed it (slots, bot, navigation, consent, or orchestrator kill switch) instead of only seeing `dispatch_auction: None`. Pair with the EC-context jurisdiction log when consent_allows_auction is false. --- crates/trusted-server-core/src/publisher.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4f6f7823d..d0d7ef812 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1322,6 +1322,17 @@ pub async fn handle_publisher_request( auction.orchestrator.is_enabled(), ); let should_run_auction = should_run_ad_stack; + // Diagnostic: shows which gate suppresses the server-side auction. Pair with + // the `EC context: ... jurisdiction=...` line from EC-context construction + // when `consent_allows_auction=false`. + log::debug!( + "server-side ad-stack gate: is_get={is_get} is_navigation={is_navigation} \ + is_prefetch={is_prefetch} is_bot={is_bot} matched_slots={} \ + consent_allows_auction={consent_allows_auction} orchestrator_enabled={} \ + -> should_run_auction={should_run_auction}", + matched_slots.len(), + auction.orchestrator.is_enabled(), + ); if matched_slots.is_empty() && settings.creative_opportunities.is_some() { log::debug!( From 36a6e7ae58f05f7bd0bc4c2b4fbcefb8e3efa3ed Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 15:37:21 +0530 Subject: [PATCH 129/195] Run the server-side auction on the Fastly EdgeZero path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EdgeZero buffered path passed empty slots and finalized via the sync `buffer_publisher_response`, so configured creative-opportunity slots were routed to the legacy path by `edgezero_can_handle_settings`. Now that `buffer_publisher_response_async` collects the dispatched auction, EdgeZero can run the full ad stack: - Pass the configured `creative_opportunities.slot` and finalize via `buffer_publisher_response_async` (the path's `ec.ec_context` already carries consent + platform geo). EID targeting stays off (`registry: None`). - Drop the `edgezero_can_handle_settings` gate, its routing branch, the three tests, and the now-unused test settings helpers — EdgeZero handles configured slots, so the legacy fallback for them is obsolete. --- .../trusted-server-adapter-fastly/src/app.rs | 47 ++++--- .../trusted-server-adapter-fastly/src/main.rs | 122 +----------------- 2 files changed, 31 insertions(+), 138 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 13a348314..465a9fa54 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -114,7 +114,7 @@ use trusted_server_core::proxy::{ AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, BoundedWriter, + buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, BoundedWriter, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -715,19 +715,24 @@ async fn dispatch_fallback( // be opened, matching legacy behavior. match runtime_services_for_consent_route(&state.settings, services) { Ok(publisher_services) => { - // Server-side auction is not yet wired into the EdgeZero buffered - // finalize path (`buffer_publisher_response` runs the - // synchronous pipeline, which does not collect dispatched SSP - // bids). Pass no slots so `handle_publisher_request` dispatches no - // auction and no bid requests are wasted. The legacy path runs the - // full server-side auction; wiring it here is deferred to the - // EdgeZero cutover. + // Run the server-side auction with the configured creative- + // opportunity slots and collect the dispatched bids in the + // buffered finalize (`buffer_publisher_response_async`), matching + // the legacy streaming path. `handle_publisher_request` matches the + // slots against the request path. EID targeting stays off here + // (`registry: None`) until per-platform KV enrichment is wired. + let slots = state + .settings + .creative_opportunities + .as_ref() + .map(|creative_opportunities| creative_opportunities.slot.as_slice()) + .unwrap_or(&[]); let auction = trusted_server_core::publisher::AuctionDispatch { orchestrator: &state.orchestrator, - slots: &[], + slots, registry: None, }; - handle_publisher_request( + match handle_publisher_request( &state.settings, &publisher_services, ec.kv_graph.as_ref(), @@ -736,14 +741,20 @@ async fn dispatch_fallback( req, ) .await - .and_then(|pub_response| { - buffer_publisher_response( - pub_response, - &method, - &state.settings, - &state.registry, - ) - }) + { + Ok(pub_response) => { + buffer_publisher_response_async( + pub_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &publisher_services, + ) + .await + } + Err(e) => Err(e), + } } Err(e) => Err(e), } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 30e91b01f..0d79193a0 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -191,13 +191,6 @@ fn is_edgezero_enabled(config_store: &ConfigStoreHandle) -> Result bool { - settings - .creative_opportunities - .as_ref() - .is_none_or(|creative_opportunities| creative_opportunities.slot.is_empty()) -} - /// Reads `edgezero_rollout_pct` from the config store. /// /// | Config store state | Return value | Effect | @@ -352,24 +345,8 @@ fn main() { }; if route_to_edgezero { - match get_settings() { - Ok(settings) if edgezero_can_handle_settings(&settings) => { - log::debug!("routing request through EdgeZero path"); - edgezero_main(req, edgezero_config_store); - } - Ok(_) => { - log::warn!( - "EdgeZero path does not yet support configured creative_opportunity slots; routing through legacy path" - ); - legacy_main(req); - } - Err(e) => { - log::warn!( - "failed to load settings for EdgeZero compatibility check, falling back to legacy path: {e:?}" - ); - legacy_main(req); - } - } + log::debug!("routing request through EdgeZero path"); + edgezero_main(req, edgezero_config_store); } else { legacy_main(req); } @@ -1513,71 +1490,6 @@ mod tests { .expect("should parse test settings") } - fn test_settings_with_empty_creative_opportunities() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [creative_opportunities] - gam_network_id = "12345" - auction_timeout_ms = 500 - "#, - ) - .expect("should parse test settings with creative opportunities") - } - - fn test_settings_with_configured_creative_opportunities() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [creative_opportunities] - gam_network_id = "12345" - auction_timeout_ms = 500 - - [[creative_opportunities.slot]] - id = "atf" - page_patterns = ["/article/*"] - formats = [{ width = 300, height = 250 }] - "#, - ) - .expect("should parse test settings with configured creative opportunities") - } - #[test] fn parses_true_flag_values() { assert!(parse_edgezero_flag("true"), "should parse 'true'"); @@ -1925,36 +1837,6 @@ mod tests { ); } - #[test] - fn edgezero_accepts_settings_without_creative_opportunities() { - let settings = test_settings(); - - assert!( - edgezero_can_handle_settings(&settings), - "should allow EdgeZero when server-side ad templates are not configured" - ); - } - - #[test] - fn edgezero_accepts_settings_with_empty_creative_opportunities() { - let settings = test_settings_with_empty_creative_opportunities(); - - assert!( - edgezero_can_handle_settings(&settings), - "should allow EdgeZero when server-side ad templates are configured but no slots are enabled" - ); - } - - #[test] - fn edgezero_rejects_settings_with_configured_creative_opportunity_slots() { - let settings = test_settings_with_configured_creative_opportunities(); - - assert!( - !edgezero_can_handle_settings(&settings), - "should route through legacy path while EdgeZero lacks server-side ad-template support" - ); - } - #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); From c65c2967fd9d018b28da75cca7f2e9279336e352 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 30 Jun 2026 15:47:39 +0530 Subject: [PATCH 130/195] Enrich Fastly EdgeZero auction bids with server-side EIDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EdgeZero publisher path dispatched the auction with registry: None, so the bid request carried no KV identity-graph EIDs (only client cookie EIDs). It already passes ec.kv_graph as the identity KV, so wire the matching PartnerRegistry::from_config(settings.ec.partners) into the AuctionDispatch to resolve server-side partner EIDs — matching the legacy auction path. Fastly-only: the sync EC identity graph (KvIdentityGraph/EcKvStore) works on Fastly's sync KV; the async-KV portability adapters are unaffected (they still pass registry: None until the EC graph supports async stores). --- .../trusted-server-adapter-fastly/src/app.rs | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 465a9fa54..8267f55b9 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -719,39 +719,45 @@ async fn dispatch_fallback( // opportunity slots and collect the dispatched bids in the // buffered finalize (`buffer_publisher_response_async`), matching // the legacy streaming path. `handle_publisher_request` matches the - // slots against the request path. EID targeting stays off here - // (`registry: None`) until per-platform KV enrichment is wired. + // slots against the request path. The partner registry plus the + // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with + // server-side EIDs, same as the legacy auction. let slots = state .settings .creative_opportunities .as_ref() .map(|creative_opportunities| creative_opportunities.slot.as_slice()) .unwrap_or(&[]); - let auction = trusted_server_core::publisher::AuctionDispatch { - orchestrator: &state.orchestrator, - slots, - registry: None, - }; - match handle_publisher_request( - &state.settings, - &publisher_services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, - ) - .await - { - Ok(pub_response) => { - buffer_publisher_response_async( - pub_response, - &method, + match PartnerRegistry::from_config(&state.settings.ec.partners) { + Ok(partner_registry) => { + let auction = trusted_server_core::publisher::AuctionDispatch { + orchestrator: &state.orchestrator, + slots, + registry: Some(&partner_registry), + }; + match handle_publisher_request( &state.settings, - &state.registry, - &state.orchestrator, &publisher_services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, ) .await + { + Ok(pub_response) => { + buffer_publisher_response_async( + pub_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &publisher_services, + ) + .await + } + Err(e) => Err(e), + } } Err(e) => Err(e), } From 9c0c4249133029c1ee03fbcb562ba98dbac7cdc3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 1 Jul 2026 11:16:46 +0530 Subject: [PATCH 131/195] Bring the server-side auction to parity across all adapters Resolve the fifth code-review pass. The blocking findings were all cross-adapter parity gaps in the server-side auction: - Build the geo-aware EC context in the /auction handlers on Axum, Cloudflare, and Spin. They passed EcContext::default(), leaving jurisdiction Unknown and failing the consent gate closed even for consented users. A shared per-adapter build_ec_context helper now serves /auction, page-bids, and the publisher fallback, and logs (rather than swallows) a malformed-consent read error. - Wire GET /__ts/page-bids and its OPTIONS->403 CSRF guard on the Fastly EdgeZero path and all three portability adapters, reusing core handle_page_bids and a shared page_bids_preflight_denied() helper. Previously it was Fastly-legacy-only, so SPA re-auction silently fell through to the origin on every other path. - Add trusted_server_core::response_privacy with the Set-Cookie cache-privacy downgrade and the uncacheable-operator-header guard, and call it from every adapter's apply_finalize_headers so a shared cache (Cloudflare) can no longer serve an operator/origin public Cache-Control on a cookie-bearing response. Also address the inline and non-blocking findings: warn on a dropped dispatched auction for bodiless responses, extract build_slot_json shared by the initial-page and page-bids paths, use creative_opportunity_slots() everywhere, drop the PBS id->ad_id fallback, log APS slot-id collisions, align the parallel provider parse with the collect path, remove the dead sync buffer_publisher_response, factor the mediator placeholder request, drop the unused toml dependency, guard MediaType against a future serde(default), and document the Fastly-only KV EID enrichment. JS: dedup win/billing beacons across concurrent renders, add the SSR guard to installSlimPrebidLoader, and short-circuit waitForSlotElements on an already-aborted signal. Add regression tests for the currentPath rollback and the u32::MAX format-dimension rejection. --- Cargo.lock | 1 - crates/trusted-server-adapter-axum/src/app.rs | 94 +++++--- .../src/middleware.rs | 23 +- .../src/app.rs | 81 ++++--- .../src/middleware.rs | 24 +-- .../trusted-server-adapter-fastly/Cargo.toml | 1 - .../trusted-server-adapter-fastly/src/app.rs | 52 ++++- .../src/middleware.rs | 93 ++------ crates/trusted-server-adapter-spin/src/app.rs | 97 ++++++--- .../src/middleware.rs | 23 +- .../src/auction/orchestrator.rs | 9 +- .../trusted-server-core/src/auction/types.rs | 5 + .../src/creative_slot_build_check.rs | 14 ++ .../src/integrations/aps.rs | 15 +- .../src/integrations/prebid.rs | 5 +- crates/trusted-server-core/src/lib.rs | 1 + crates/trusted-server-core/src/publisher.rs | 202 +++++++++--------- .../src/response_privacy.rs | 180 ++++++++++++++++ .../lib/src/integrations/gpt/index.ts | 19 ++ .../test/integrations/gpt/spa_hook.test.ts | 33 +++ trusted-server.toml | 5 +- 21 files changed, 654 insertions(+), 323 deletions(-) create mode 100644 crates/trusted-server-core/src/response_privacy.rs diff --git a/Cargo.lock b/Cargo.lock index e026ef3af..629e5df60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3713,7 +3713,6 @@ dependencies = [ "log-fastly", "serde", "serde_json", - "toml", "trusted-server-core", "url", "urlencoding", diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 953a0839f..18548c631 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -19,7 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -129,6 +130,34 @@ where .unwrap_or_else(|e| http_error(&e))) } +// --------------------------------------------------------------------------- +// EC context +// --------------------------------------------------------------------------- + +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Geo comes from the platform (a no-op on the local Axum dev server, so +/// jurisdiction stays Unknown there unless the request carries TCF consent). A +/// malformed consent string is logged and falls back to the default +/// (fail-closed) context rather than being silently swallowed. +fn build_ec_context(state: &AppState, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(&state.settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Fallback dispatcher (tsjs / integration proxy / publisher) // --------------------------------------------------------------------------- @@ -168,30 +197,10 @@ async fn dispatch_fallback( // Run the server-side auction with the configured creative-opportunity // slots; `handle_publisher_request` matches them against the request path. - // Build the EC context (consent + jurisdiction) from the request like the - // Fastly entry point — `EcContext::default()` leaves jurisdiction Unknown, - // which fails the auction consent gate closed. Geo comes from the platform - // (no-op on the local Axum dev server, so jurisdiction stays Unknown there - // unless the request carries TCF consent). - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - let mut ec_context = - EcContext::read_from_request_with_geo(&state.settings, &req, services, geo_info.as_ref()) - .unwrap_or_default(); - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|co| co.slot.as_slice()) - .unwrap_or(&[]); + let mut ec_context = build_ec_context(state, services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots, + slots: state.settings.creative_opportunity_slots(), registry: None, }; let publisher_response = handle_publisher_request( @@ -242,6 +251,7 @@ enum NamedRouteHandler { /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, Auction, + PageBids, FirstPartyProxy, FirstPartyClick, FirstPartySign, @@ -264,7 +274,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 11] { +fn named_routes() -> [NamedRoute; 12] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -310,6 +320,13 @@ fn named_routes() -> [NamedRoute; 11] { primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, }, + // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS + // preflight guard for this side-effecting endpoint. + NamedRoute { + path: "/__ts/page-bids", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], @@ -368,7 +385,10 @@ fn named_route_handler( } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent + // gate sees the caller's jurisdiction — `EcContext::default()` + // fails it closed for consented users. + let ec_context = build_ec_context(&state, &services, &req); handle_auction( &state.settings, &state.orchestrator, @@ -380,6 +400,30 @@ fn named_route_handler( ) .await } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight + // for this side-effecting GET and is always denied so the + // GET handler's `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + Ok(page_bids_preflight_denied()) + } else { + let ec_context = build_ec_context(&state, &services, &req); + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids( + &state.settings, + &services, + None, + auction, + &ec_context, + req, + ) + .await + } + } NamedRouteHandler::FirstPartyProxy => { handle_first_party_proxy(&state.settings, &services, req).await } diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 8ad362a97..45cbedc2c 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -88,26 +88,19 @@ impl Middleware for AuthMiddleware { /// /// Unlike the Fastly variant, geo is always unavailable so `X-Geo-Info-Available: false` /// is unconditionally emitted. Fastly-specific headers are omitted. -/// Operator-configured `settings.response_headers` are applied last and can override -/// any managed header. +/// Operator-configured `settings.response_headers` are applied last (with the +/// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Response) { response.headers_mut().insert( HEADER_X_GEO_INFO_AVAILABLE, HeaderValue::from_static("false"), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cookie-bearing responses stay private to shared caches and operator + // headers cannot re-enable caching for uncacheable per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 670c09255..1a58f4ef5 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -19,8 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, - handle_tsjs_dynamic, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, + handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -81,6 +81,29 @@ fn build_per_request_services(ctx: &RequestContext) -> RuntimeServices { build_runtime_services(ctx) } +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Geo comes from the Workers `cf` object when deployed. A malformed +/// consent string is logged and falls back to the default (fail-closed) context +/// rather than being silently swallowed. +fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Handler factory // --------------------------------------------------------------------------- @@ -308,33 +331,10 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - // Build the EC context (consent + jurisdiction) from the request - // like the Fastly entry point — `EcContext::default()` leaves - // jurisdiction Unknown and fails the auction consent gate closed. - // Geo comes from the Workers `cf` object when deployed. - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - let mut ec_context = EcContext::read_from_request_with_geo( - &state.settings, - &req, - &services, - geo_info.as_ref(), - ) - .unwrap_or_default(); - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|co| co.slot.as_slice()) - .unwrap_or(&[]); + let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots, + slots: state.settings.creative_opportunity_slots(), registry: None, }; match handle_publisher_request( @@ -413,7 +413,10 @@ fn build_router(state: &Arc) -> RouterService { .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent gate + // sees the caller's jurisdiction — `EcContext::default()` + // fails it closed for consented users. + let ec_context = build_ec_context(&s.settings, &services, &req); handle_auction( &s.settings, &s.orchestrator, @@ -426,6 +429,28 @@ fn build_router(state: &Arc) -> RouterService { .await }), ) + // SPA re-auction endpoint. The OPTIONS preflight for this + // side-effecting GET is denied so the GET handler's `X-TSJS-Page-Bids` + // gate stays trustworthy. + .route( + "/__ts/page-bids", + Method::OPTIONS, + make_handler(Arc::clone(&state), |_s, _services, _req| async move { + Ok(page_bids_preflight_denied()) + }), + ) + .get( + "/__ts/page-bids", + make_handler(Arc::clone(&state), |s, services, req| async move { + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req).await + }), + ) .get( "/first-party/proxy", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 3b60cae3f..5b605bcff 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -96,8 +96,8 @@ impl Middleware for AuthMiddleware { /// /// `geo_available` controls `X-Geo-Info-Available`; pass `true` when /// `cf-ipcountry` was present and non-`XX` in the incoming request. -/// Operator-configured `settings.response_headers` are applied last and can -/// override any managed header. +/// Operator-configured `settings.response_headers` are applied last (with the +/// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_available: bool, @@ -108,18 +108,12 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cloudflare is a real shared cache: cookie-bearing responses must stay + // private and operator headers must not re-enable caching for uncacheable + // per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 91c4a36d9..8547fd519 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -21,7 +21,6 @@ log = { workspace = true } log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -toml = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 8267f55b9..990b20257 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -114,7 +114,8 @@ use trusted_server_core::proxy::{ AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - buffer_publisher_response_async, handle_publisher_request, handle_tsjs_dynamic, BoundedWriter, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, AuctionDispatch, BoundedWriter, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -584,6 +585,38 @@ async fn run_named_route( ) .await } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight for this + // side-effecting GET and is always denied so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + return Ok(page_bids_preflight_denied()); + } + // Like the auction, page-bids reads consent data, so the consent KV + // store must be available — fail closed with 503 when configured but + // unopenable, matching legacy. + let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let registry_ref = if partner_registry.is_empty() { + None + } else { + Some(&partner_registry) + }; + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: registry_ref, + }; + handle_page_bids( + &state.settings, + &consent_services, + ec.kv_graph.as_ref(), + auction, + &ec.ec_context, + req, + ) + .await + } NamedRouteHandler::FirstPartyProxy => { handle_first_party_proxy(&state.settings, services, req).await } @@ -722,15 +755,10 @@ async fn dispatch_fallback( // slots against the request path. The partner registry plus the // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|creative_opportunities| creative_opportunities.slot.as_slice()) - .unwrap_or(&[]); + let slots = state.settings.creative_opportunity_slots(); match PartnerRegistry::from_config(&state.settings.ec.partners) { Ok(partner_registry) => { - let auction = trusted_server_core::publisher::AuctionDispatch { + let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots, registry: Some(&partner_registry), @@ -977,6 +1005,7 @@ enum NamedRouteHandler { SetTester, ClearTester, Auction, + PageBids, FirstPartyProxy, FirstPartyClick, FirstPartySign, @@ -1061,6 +1090,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, }, + // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS + // preflight guard for this side-effecting endpoint. + NamedRoute { + path: "/__ts/page-bids", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 91b7dcdec..298d30416 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use edgezero_adapter_fastly::FastlyRequestContext; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{header, HeaderName, HeaderValue, Response, StatusCode}; +use edgezero_core::http::{HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; use std::net::IpAddr; @@ -223,84 +223,29 @@ pub(crate) fn apply_finalize_headers( } // Any response that sets a per-user cookie (notably the EC identity cookie) - // must never be shared-cached, or a shared cache could replay one user's - // Set-Cookie to others. Skip when the response is already uncacheable so we - // don't clobber a stricter directive (e.g. `no-store`). - enforce_set_cookie_cache_privacy(response); - - // Per-user responses (assembled HTML, page-bids, cookie-bearing navigations) - // carry an uncacheable Cache-Control directive (`private` or `no-store`). - // Operator headers must not re-enable shared caching for them — neither by - // replacing Cache-Control nor by reintroducing the surrogate cache headers - // the privacy paths stripped. - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - - for (key, value) in &settings.response_headers { - if response_is_uncacheable - && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || key.eq_ignore_ascii_case("surrogate-control") - || key.eq_ignore_ascii_case("fastly-surrogate-control")) - { - continue; - } - let header_name = HeaderName::from_bytes(key.as_bytes()) - .expect("should be a valid header name: response_headers validated in prepare_runtime"); - let header_value = HeaderValue::from_str(value).expect( - "should be a valid header value: response_headers validated in prepare_runtime", - ); - response.headers_mut().insert(header_name, header_value); - } + // must never be shared-cached, and per-user responses (assembled HTML, + // page-bids, cookie-bearing navigations) must not have their uncacheable + // Cache-Control re-enabled by operator headers. This shared helper runs + // byte-identically on every adapter so the privacy guarantee can't drift. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } -/// Surrogate cache headers stripped from every cookie-bearing response. A single -/// source of truth so the legacy ([`crate::enforce_set_cookie_cache_privacy`]) -/// and `EdgeZero` copies of the privacy downgrade cannot drift apart. -pub(crate) const SURROGATE_CACHE_HEADERS: &[&str] = - &["surrogate-control", "fastly-surrogate-control"]; +/// Surrogate cache headers stripped from every cookie-bearing response. +/// +/// Re-exported from [`trusted_server_core::response_privacy`] so the legacy +/// [`crate::enforce_set_cookie_cache_privacy`] `FastlyResponse` variant and the +/// shared [`Response`] downgrade cannot drift apart. +pub(crate) use trusted_server_core::response_privacy::SURROGATE_CACHE_HEADERS; /// Forces cookie-bearing responses to stay private to shared caches. /// -/// Mirrors [`crate::enforce_set_cookie_cache_privacy`] for the [`Response`] type -/// from `edgezero_core::http`. The `EdgeZero` entry point re-applies this after +/// Re-exported from [`trusted_server_core::response_privacy`] so the `EdgeZero` +/// entry point (`main.rs`) can re-apply it after /// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) -/// and request-filter effects, because the EC identity `Set-Cookie` is written -/// after [`apply_finalize_headers`] runs and would otherwise reach a shared cache -/// with inherited `public`/surrogate cache headers. -/// -/// Idempotent: a response already marked `private`/`no-store` keeps its stricter -/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a -/// `no-store` cookie response can never retain shared cacheability. -pub(crate) fn enforce_set_cookie_cache_privacy(response: &mut Response) { - if !response.headers().contains_key(header::SET_COOKIE) { - return; - } - // Surrogate cache headers must come off every cookie-bearing response, even - // one already carrying a stricter `no-store`/`private` directive — they are - // independent of Cache-Control and would otherwise let a shared cache store - // and replay one visitor's Set-Cookie. - for name in SURROGATE_CACHE_HEADERS { - response.headers_mut().remove(*name); - } - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); - if !already_uncacheable { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - } -} +/// writes the EC identity `Set-Cookie`, using the single shared implementation. +pub(crate) use trusted_server_core::response_privacy::enforce_set_cookie_cache_privacy; // --------------------------------------------------------------------------- // Tests @@ -317,7 +262,7 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; - use edgezero_core::http::{request_builder, response_builder, Method, StatusCode}; + use edgezero_core::http::{request_builder, response_builder, HeaderName, Method, StatusCode}; use edgezero_core::middleware::Next; use edgezero_core::params::PathParams; use error_stack::Report; diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index d8a487d20..55e97c723 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -20,8 +20,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_publisher_request, - handle_tsjs_dynamic, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, + handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -143,7 +143,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 11] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -152,6 +152,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 11] { ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), + ("/__ts/page-bids", &[Method::GET, Method::OPTIONS]), ("/first-party/proxy", &[Method::GET]), ("/first-party/click", &[Method::GET]), ("/first-party/sign", &[Method::GET, Method::POST]), @@ -322,6 +323,30 @@ fn health_response() -> Response { resp } +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Spin's platform geo is a no-op, so jurisdiction stays Unknown unless +/// the request carries TCF consent. A malformed consent string is logged and +/// falls back to the default (fail-closed) context rather than being silently +/// swallowed. +fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -506,7 +531,10 @@ fn build_router(state: &Arc) -> RouterService { // OpenRTB metadata that auction signing derives from // `RequestInfo::from_request` uses the trusted runtime authority. let req = ctx.into_request(); - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent gate sees + // the caller's jurisdiction — `EcContext::default()` fails it + // closed for consented users. + let ec_context = build_ec_context(&s.settings, &services, &req); Ok(handle_auction( &s.settings, &s.orchestrator, @@ -521,6 +549,33 @@ fn build_router(state: &Arc) -> RouterService { } }; + // GET /__ts/page-bids — SPA re-auction endpoint. + let s = Arc::clone(&state); + let page_bids_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let services = build_runtime_services(&ctx); + let req = ctx.into_request(); + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + Ok( + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) + } + }; + + // OPTIONS /__ts/page-bids — deny the CORS preflight for this + // side-effecting GET so the `X-TSJS-Page-Bids` gate stays trustworthy. + let page_bids_options_handler = |_ctx: RequestContext| async { + Ok::(page_bids_preflight_denied()) + }; + // GET /first-party/proxy let s = Arc::clone(&state); let fp_proxy_handler = move |ctx: RequestContext| { @@ -611,34 +666,10 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - // Build the EC context (consent + jurisdiction) from the request - // like the Fastly entry point — `EcContext::default()` leaves - // jurisdiction Unknown and fails the auction consent gate closed. - // Spin's platform geo is a no-op, so jurisdiction stays Unknown - // unless the request carries TCF consent. - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - let mut ec_context = EcContext::read_from_request_with_geo( - &state.settings, - &req, - &services, - geo_info.as_ref(), - ) - .unwrap_or_default(); - let slots = state - .settings - .creative_opportunities - .as_ref() - .map(|co| co.slot.as_slice()) - .unwrap_or(&[]); + let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, - slots, + slots: state.settings.creative_opportunity_slots(), registry: None, }; match handle_publisher_request( @@ -708,6 +739,12 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/rotate", rotate_handler) .post("/_ts/admin/keys/deactivate", deactivate_handler) .post("/auction", auction_handler) + .get("/__ts/page-bids", page_bids_handler) + .route( + "/__ts/page-bids", + Method::OPTIONS, + page_bids_options_handler, + ) .get("/first-party/proxy", fp_proxy_handler) .get("/first-party/click", fp_click_handler) .get("/first-party/sign", fp_sign_handler) diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 62f83e1ea..1bcede1fc 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -124,8 +124,8 @@ impl Middleware for NormalizeMiddleware { /// /// `geo_available` controls `X-Geo-Info-Available`. Spin passes `false` /// because it has no geo headers. Operator-configured -/// `settings.response_headers` are applied last and can override any managed -/// header. +/// `settings.response_headers` are applied last (with the shared cookie +/// cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_available: bool, @@ -136,18 +136,11 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cookie-bearing responses stay private to shared caches and operator + // headers cannot re-enable caching for uncacheable per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index c29f182e2..1898a628d 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -541,7 +541,14 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; - match provider.parse_response(response, response_time_ms).await { + // Use the context-aware parse so a provider overriding + // `parse_response_with_context` behaves identically on the + // parallel (`/auction`, page-bids) and collect (publisher) + // paths. The default impl delegates to `parse_response`. + match provider + .parse_response_with_context(response, response_time_ms, context) + .await + { Ok(auction_response) => { log::info!( "Provider '{}' returned {} bids (status: {:?}, time: {}ms)", diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index ffe918aa4..14c7713f8 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -53,6 +53,11 @@ pub struct AdFormat { } /// Media type enumeration. +/// +/// `Default` is `Banner` for programmatic construction only. Do **not** add +/// `#[serde(default)]` to any field of this type: it would coerce an +/// unknown/missing media type to `Banner` rather than failing, silently +/// mis-typing video/native slots. Deserialization must stay strict. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum MediaType { diff --git a/crates/trusted-server-core/src/creative_slot_build_check.rs b/crates/trusted-server-core/src/creative_slot_build_check.rs index 066cdde1a..2e763083f 100644 --- a/crates/trusted-server-core/src/creative_slot_build_check.rs +++ b/crates/trusted-server-core/src/creative_slot_build_check.rs @@ -478,6 +478,20 @@ mod tests { assert!(err.contains("positive width and height"), "got: {err}"); } + #[test] + fn rejects_format_dimension_above_u32_range() { + // Runtime dimensions are `u32`; a value above `u32::MAX` would silently + // truncate when parsed into the runtime slot, so it must fail at build. + let slot = json!({ + "id": "atf", + "page_patterns": ["/20**"], + "formats": [{ "width": 5_000_000_000_u64, "height": 250 }] + }); + let err = validate_creative_slot(&slot, "123456789") + .expect_err("width above u32::MAX must fail at build time"); + assert!(err.contains("within u32 range"), "got: {err}"); + } + #[test] fn rejects_empty_page_patterns() { let slot = json!({ diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 2ba1e5149..1d2d4ea4a 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -345,7 +345,20 @@ impl ApsAuctionProvider { .and_then(|v| v.as_str()) .unwrap_or(&slot.id) .to_string(); - slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()); + // Last-write-wins: two slots configuring the same + // `[bidders.aps].slotID` would remap one slot's bids to the + // wrong creative slot. Log the collision so a misconfiguration + // is diagnosable, mirroring the build_bid_index collision log. + if let Some(previous_slot_id) = + slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()) + { + log::debug!( + "APS slot ID '{aps_slot_id}' maps to multiple creative slots \ + ('{previous_slot_id}' overwritten by '{}'); bids for this APS \ + slot will resolve to the last one", + slot.id, + ); + } // Extract sizes from banner formats let sizes: Vec<[u32; 2]> = slot diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 451650611..300d5dd68 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1490,9 +1490,12 @@ impl PrebidAuctionProvider { .map(std::string::ToString::to_string) }; + // `adid` is the creative/ad identifier. The OpenRTB `id` is the bid ID, + // not an ad ID, so it is not used as a fallback: surfacing it as `ad_id` + // (which is exposed raw in the debug bid) would mislead any consumer that + // treats `ad_id` as a creative identifier. Absent `adid`, `ad_id` is None. let ad_id = bid_obj .get("adid") - .or_else(|| bid_obj.get("id")) .and_then(|v| v.as_str()) .map(String::from); diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 3bcb0b652..d4ee8515d 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -62,6 +62,7 @@ pub mod proxy; pub mod publisher; pub mod redacted; pub mod request_signing; +pub mod response_privacy; pub mod rsc_flight; pub(crate) mod s3_sigv4; pub mod settings; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d0d7ef812..21c6f560d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -440,68 +440,23 @@ pub struct OwnedProcessResponseParams { pub(crate) price_granularity: PriceGranularity, } -/// Buffer a [`PublisherResponse`] into a single [`Response`]. +/// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the +/// dispatched server-side auction before buffering. /// /// Handles all three variants: returns [`PublisherResponse::Buffered`] unchanged, -/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into memory, -/// and reattaches [`PublisherResponse::PassThrough`] bodies directly. +/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into +/// memory, and reattaches [`PublisherResponse::PassThrough`] bodies directly. /// /// The buffered size is capped by `settings.publisher.max_buffered_body_bytes` -/// (16 MiB by default), so processable origin responses cannot grow the -/// buffer without bound and exhaust the Wasm heap. +/// (16 MiB by default), so processable origin responses cannot grow the buffer +/// without bound and exhaust the Wasm heap. /// -/// `method` is used to preserve metadata for bodiless responses: `HEAD` and -/// bodiless statuses (204, 304) carry no body but may advertise the `GET` -/// representation's length. `handle_publisher_request` already strips the origin -/// `Content-Length` for processable [`PublisherResponse::Stream`] responses, so -/// rewriting it here to the buffered byte count (`0`) would replace it with a -/// misleading length. Those responses skip the buffer, the length rewrite, and -/// the body replacement, mirroring the asset path's bodiless guard. +/// `method` preserves metadata for bodiless responses: `HEAD` and bodiless +/// statuses (204, 304) carry no body but may advertise the `GET` representation's +/// length, so they skip the buffer and length rewrite. /// -/// # Errors -/// -/// Returns an error if the streaming pipeline fails to process the response -/// body, or if the processed body exceeds the configured buffer cap. -pub fn buffer_publisher_response( - publisher_response: PublisherResponse, - method: &Method, - settings: &Settings, - integration_registry: &IntegrationRegistry, -) -> Result, Report> { - match publisher_response { - PublisherResponse::Buffered(response) => Ok(response), - PublisherResponse::Stream { - mut response, - body, - params, - } => { - if !response_carries_body(method, response.status()) { - return Ok(response); - } - let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); - stream_publisher_body(body, &mut output, ¶ms, settings, integration_registry)?; - let bytes = output.into_inner(); - response.headers_mut().insert( - http::header::CONTENT_LENGTH, - http::HeaderValue::from(bytes.len() as u64), - ); - *response.body_mut() = EdgeBody::from(bytes); - Ok(response) - } - PublisherResponse::PassThrough { mut response, body } => { - *response.body_mut() = body; - Ok(response) - } - } -} - -/// Async variant of [`buffer_publisher_response`] that collects the dispatched -/// server-side auction before buffering. -/// -/// The sync [`buffer_publisher_response`] drives [`stream_publisher_body`], -/// which ignores `params.dispatched_auction`, so its `` injection always -/// falls back to empty `tsjs.bids`. Adapters that finalize on an async runtime -/// (Axum, Cloudflare, Spin) call this instead: it drives +/// Every adapter (Axum, Cloudflare, Spin, and the Fastly `EdgeZero` path) calls +/// this: it drives /// [`stream_publisher_body_async`], which awaits /// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids /// into `ad_bids_state`, and injects them before ``. @@ -526,6 +481,17 @@ pub async fn buffer_publisher_response_async( mut params, } => { if !response_carries_body(method, response.status()) { + if params.dispatched_auction.is_some() { + // A bodiless response (HEAD navigation, 204/304) has no + // `` to inject bids into, so the dispatched SSP + // requests are wasted — surface it for quota observability, + // matching the pass-through / buffered-unmodified arms. + log::warn!( + "Server-side auction dispatched but response is bodiless (method: {}, status: {}); in-flight SSP bid requests will not be collected", + method, + response.status(), + ); + } return Ok(response); } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); @@ -685,10 +651,7 @@ pub async fn stream_publisher_body_async( if !is_html { // Non-HTML: collect auction first, then stream. There is no // to hold, so delaying the entire body until collection is acceptable. - let placeholder = Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(EdgeBody::empty()) - .unwrap_or_else(|_| Request::new(EdgeBody::empty())); + let placeholder = mediator_placeholder_request(); let result = orchestrator .collect_dispatched_auction( dispatched, @@ -736,6 +699,20 @@ pub async fn stream_publisher_body_async( .await } +/// Builds the canonical mediator placeholder [`Request`] passed to the collect +/// phase via [`make_collect_context`]. +/// +/// The URI is the compile-time constant +/// [`MEDIATOR_PLACEHOLDER_URL`](crate::auction::types::MEDIATOR_PLACEHOLDER_URL), +/// so the builder is infallible; a default-URI fallback would trip +/// [`make_collect_context`]'s `debug_assert_eq!`. +fn mediator_placeholder_request() -> Request { + Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(EdgeBody::empty()) + .expect("MEDIATOR_PLACEHOLDER_URL should be a valid URI") +} + /// Build a minimal [`AuctionContext`] for the collect phase. /// /// See [`AuctionContext::request`]: the orchestrator's collect path runs @@ -1127,10 +1104,7 @@ async fn collect_stream_auction( settings: &Settings, ) { log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); - let placeholder = Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(EdgeBody::empty()) - .unwrap_or_else(|_| Request::new(EdgeBody::empty())); + let placeholder = mediator_placeholder_request(); let collect_ctx = make_collect_context(settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) @@ -1831,6 +1805,38 @@ pub(crate) fn build_empty_bids_script() -> String { build_bids_script(&serde_json::Map::new()) } +/// Builds the client-facing JSON wire shape for one creative-opportunity slot. +/// +/// Shared verbatim by [`build_ad_slots_script`] (initial page render) and +/// [`handle_page_bids`] (SPA navigation) so the slot wire shape has a single +/// definition and the two paths cannot silently diverge. Property names match +/// what the client-side TSJS bundle expects: `gam_unit_path`, `div_id`, +/// `formats`, and `targeting`. +fn build_slot_json( + slot: &crate::creative_opportunities::CreativeOpportunitySlot, + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, +) -> serde_json::Value { + let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + let div_id = slot.resolved_div_id(); + let formats: Vec = slot + .formats + .iter() + .map(|f| serde_json::json!([f.width, f.height])) + .collect(); + let targeting: serde_json::Map = slot + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + serde_json::json!({ + "id": slot.id, + "gam_unit_path": gam_path, + "div_id": div_id, + "formats": formats, + "targeting": targeting, + }) +} + /// Build the `tsjs.adSlots` ` + + +"#; + + #[test] + fn collects_gpt_slot_from_local_fixture() { + if !chrome_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + aps_slot_ids: Vec::new(), + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } +} diff --git a/crates/trusted-server-cli/src/audit/collector.rs b/crates/trusted-server-cli/src/audit/collector.rs index 314ae54fc..4a774c9c7 100644 --- a/crates/trusted-server-cli/src/audit/collector.rs +++ b/crates/trusted-server-cli/src/audit/collector.rs @@ -1,41 +1,184 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::Args; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of `')).toBeUndefined(); + expect(safeAdmIframeSrc('blob:https://example.com/uuid')).toBeUndefined(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 6d52c368f..9ad7945c4 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1559,3 +1559,47 @@ describe('prebid/client-side bidders', () => { errorSpy.mockRestore(); }); }); + +describe('prebid self-init user ID module timing', () => { + const userSyncCallCount = () => + mockSetConfig.mock.calls.filter(([arg]) => arg && typeof arg === 'object' && 'userSync' in arg) + .length; + + const setReadyState = (value: DocumentReadyState) => { + Object.defineProperty(document, 'readyState', { value, configurable: true }); + }; + + beforeEach(() => { + vi.resetModules(); + mockSetConfig.mockClear(); + }); + + afterEach(() => { + setReadyState('complete'); + }); + + it('installs user ID modules immediately when the bundle loads after window load', async () => { + // The GPT slim loader appends this bundle from a window.load handler, so + // the document is already complete — a load listener would never fire. + setReadyState('complete'); + + await import('../../../src/integrations/prebid/index'); + + expect(userSyncCallCount()).toBeGreaterThan(0); + }); + + it('defers user ID modules to window load when the document is still loading', async () => { + setReadyState('loading'); + + await import('../../../src/integrations/prebid/index'); + + expect(userSyncCallCount()).toBe(0); + + window.dispatchEvent(new Event('load')); + expect(userSyncCallCount()).toBe(1); + + // { once: true } — a second load event must not reinstall. + window.dispatchEvent(new Event('load')); + expect(userSyncCallCount()).toBe(1); + }); +}); From fe198d988b6b6382f5fb242c2815d8efae793613 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 6 Jul 2026 22:00:02 +0530 Subject: [PATCH 134/195] Add ts audit ad-templates generate with multi-page slot merge Reconstruct [creative_opportunities] slots from a live page's GPT registry and gampad/ads requests, and write them into an existing trusted-server.toml in place, preserving all other sections. Slots merge across runs: --page-pattern unions patterns into a re-seen slot, existing slots are preserved, and --replace wipes. Ephemeral div-id noise (React hashes, -container, hex UUIDs) is normalized to stable prefixes so verify matches across renders, and TOML keys/strings are escaped defensively. Add --cookie to ad-templates generate and verify so a valid bot-protection clearance cookie can carry the browser audit past an origin challenge. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/audit/ad_templates.rs | 4 + .../trusted-server-cli/src/audit/browser.rs | 21 +- .../trusted-server-cli/src/audit/collector.rs | 6 + .../src/audit/generate/analyzer.rs | 6 + .../src/audit/generate/browser_collector.rs | 65 +- .../src/audit/generate/collector.rs | 27 +- .../src/audit/generate/gpt_slots.rs | 585 ++++++++++++ .../src/audit/generate/mod.rs | 875 +++++++++++++++++- crates/trusted-server-cli/src/audit/mod.rs | 106 +++ crates/trusted-server-cli/src/audit/page.rs | 1 + 10 files changed, 1679 insertions(+), 17 deletions(-) create mode 100644 crates/trusted-server-cli/src/audit/generate/gpt_slots.rs diff --git a/crates/trusted-server-cli/src/audit/ad_templates.rs b/crates/trusted-server-cli/src/audit/ad_templates.rs index ce9855ff7..c7a9416c5 100644 --- a/crates/trusted-server-cli/src/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/audit/ad_templates.rs @@ -43,6 +43,7 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String &args.urls, args.strict, args.scroll, + &args.cookies, ); let stdout = io::stdout(); @@ -71,6 +72,7 @@ fn build_report( urls: &[url::Url], strict: bool, scroll: bool, + cookies: &[(String, String)], ) -> VerificationReport { let init_script = build_init_script(creative); @@ -84,6 +86,7 @@ fn build_report( init_scripts: init_script.clone().into_iter().collect(), scroll, collect_ad_evidence: true, + cookies: cookies.to_vec(), }; match collector.collect_page(request) { @@ -459,6 +462,7 @@ mod tests { &parsed, strict, false, + &[], ) } diff --git a/crates/trusted-server-cli/src/audit/browser.rs b/crates/trusted-server-cli/src/audit/browser.rs index 1f6eb2584..d4d1d068f 100644 --- a/crates/trusted-server-cli/src/audit/browser.rs +++ b/crates/trusted-server-cli/src/audit/browser.rs @@ -1,13 +1,16 @@ //! Chrome/Chromium-backed implementation of [`AuditCollector`] using //! `chromiumoxide` (CDP). //! -//! The collector is read-only: it installs optional pre-navigation init scripts, -//! navigates, waits for the page to settle, optionally scrolls, and reads back a -//! bounded set of evidence. It never captures page HTML, cookies, or storage. +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. use std::time::Duration; use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; use chromiumoxide::page::Page; use futures::StreamExt as _; @@ -251,6 +254,17 @@ async fn collect_with_browser( .map_err(|error| format!("failed to install init script: {error}"))?; } + // Set operator-supplied cookies on the context before navigating so the + // origin sees an authenticated session on the first request. Scoping each to + // the request URL lets Chrome infer domain/path. + for (name, value) in &request.cookies { + let mut cookie = CookieParam::new(name.clone(), value.clone()); + cookie.url = Some(request.url.to_string()); + page.set_cookie(cookie) + .await + .map_err(|error| format!("failed to set cookie `{name}`: {error}"))?; + } + page.goto(request.url.as_str()) .await .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; @@ -470,6 +484,7 @@ mod tests { init_scripts: vec![script], scroll: false, collect_ad_evidence: true, + cookies: Vec::new(), }) .expect("should collect fixture page"); diff --git a/crates/trusted-server-cli/src/audit/collector.rs b/crates/trusted-server-cli/src/audit/collector.rs index 4a774c9c7..aca6814ca 100644 --- a/crates/trusted-server-cli/src/audit/collector.rs +++ b/crates/trusted-server-cli/src/audit/collector.rs @@ -42,6 +42,12 @@ pub struct BrowserCollectRequest { pub scroll: bool, /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, } /// The result of collecting a single page. diff --git a/crates/trusted-server-cli/src/audit/generate/analyzer.rs b/crates/trusted-server-cli/src/audit/generate/analyzer.rs index 2a13a27bc..e55952e23 100644 --- a/crates/trusted-server-cli/src/audit/generate/analyzer.rs +++ b/crates/trusted-server-cli/src/audit/generate/analyzer.rs @@ -283,6 +283,7 @@ mod tests { url: "https://cdn.example.com/dynamic.js".to_string(), resource_type: Some("Script".to_string()), }], + gpt_slots: Vec::new(), warnings: vec!["partial settle".to_string()], }; @@ -319,6 +320,7 @@ mod tests { html: "HTML Title".to_string(), script_tags: Vec::new(), network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -336,6 +338,7 @@ mod tests { html: "HTML Title".to_string(), script_tags: Vec::new(), network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -360,6 +363,7 @@ mod tests { url: "https://cdn.example.com/prebid.js".to_string(), resource_type: Some("script".to_string()), }], + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -394,6 +398,7 @@ mod tests { }, ], network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; @@ -424,6 +429,7 @@ mod tests { html: "".to_string(), script_tags: Vec::new(), network_requests: Vec::new(), + gpt_slots: Vec::new(), warnings: Vec::new(), }; diff --git a/crates/trusted-server-cli/src/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/audit/generate/browser_collector.rs index c26421b88..8ec83ba0c 100644 --- a/crates/trusted-server-cli/src/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/audit/generate/browser_collector.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; use chromiumoxide::ArcHttpRequest; use futures::StreamExt as _; use serde::Deserialize; @@ -12,7 +13,7 @@ use url::Url; use which::which; use crate::audit::generate::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, + AuditCollector, CollectedGptSlot, CollectedPage, CollectedRequest, CollectedScriptTag, }; use crate::error::{report_error, CliResult}; @@ -29,7 +30,11 @@ const RESOURCE_TIMING_BUFFER_WARNING: &str = pub(crate) struct BrowserAuditCollector; impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult { let runtime = Builder::new_current_thread() .enable_all() .build() @@ -39,11 +44,14 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(collect_page_via_browser_async(target_url)) + runtime.block_on(collect_page_via_browser_async(target_url, cookies)) } } -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { +async fn collect_page_via_browser_async( + target_url: &Url, + cookies: &[(String, String)], +) -> CliResult { let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { report_error(format!( @@ -75,7 +83,7 @@ async fn collect_page_via_browser_async(target_url: &Url) -> CliResult CliResult CliResult { let page = browser.new_page("about:blank").await.map_err(|error| { report_error(format!("failed to create browser page for audit: {error}")) })?; + // Set operator-supplied cookies before navigating so the origin sees an + // authenticated session on the first request. Scoping each to the target URL + // lets Chrome infer domain/path. + for (name, value) in cookies { + let mut cookie = CookieParam::new(name.clone(), value.clone()); + cookie.url = Some(target_url.to_string()); + page.set_cookie(cookie) + .await + .map_err(|error| report_error(format!("failed to set cookie `{name}`: {error}")))?; + } + timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) .await .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? @@ -187,6 +207,14 @@ async fn collect_page_from_browser( warnings.push(warning.to_string()); } + // Best-effort read of the live GPT slot registry. This is the authoritative + // source for slot path/div/size, so a failure here downgrades to empty + // rather than failing the whole audit. + let gpt_slots: Vec = match page.evaluate(GPT_SLOTS_SCRIPT).await { + Ok(result) => result.into_value().unwrap_or_default(), + Err(_) => Vec::new(), + }; + Ok(CollectedPage { requested_url: target_url.to_string(), final_url, @@ -206,10 +234,37 @@ async fn collect_page_from_browser( resource_type: entry.initiator_type, }) .collect(), + gpt_slots, warnings, }) } +/// Reads the live GPT slot registry into `{gam_unit_path, div_id, sizes}` rows. +/// +/// Mirrors the ad-template verifier's `getSlots()` scrape: it defends against a +/// missing or partially-initialized `googletag`, keeps only numeric sizes, and +/// drops slots without a path or div id. +const GPT_SLOTS_SCRIPT: &str = r#"() => { + try { + if (!window.googletag || typeof googletag.pubads !== 'function') return []; + const pubads = googletag.pubads(); + if (typeof pubads.getSlots !== 'function') return []; + return pubads.getSlots().map((slot) => { + const path = typeof slot.getAdUnitPath === 'function' ? slot.getAdUnitPath() : ''; + const div = typeof slot.getSlotElementId === 'function' ? slot.getSlotElementId() : ''; + const rawSizes = typeof slot.getSizes === 'function' ? (slot.getSizes() || []) : []; + const sizes = rawSizes.map((size) => + (size && typeof size.getWidth === 'function' && typeof size.getHeight === 'function') + ? [size.getWidth(), size.getHeight()] + : null + ).filter(Boolean); + return { gam_unit_path: path, div_id: div, sizes }; + }).filter((slot) => slot.gam_unit_path && slot.div_id); + } catch (error) { + return []; + } +}"#; + async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { let mut elapsed = Duration::ZERO; let mut previous_count = None; diff --git a/crates/trusted-server-cli/src/audit/generate/collector.rs b/crates/trusted-server-cli/src/audit/generate/collector.rs index 314ae54fc..2a31c763b 100644 --- a/crates/trusted-server-cli/src/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/audit/generate/collector.rs @@ -4,7 +4,15 @@ use url::Url; use crate::error::CliResult; pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; + /// Collects a live page. `cookies` are `(name, value)` pairs set on the + /// browser context before navigation (scoped to `target_url`) so an existing + /// session — e.g. a valid bot-protection clearance cookie — can carry the + /// audit past an origin challenge. + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult; } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -15,9 +23,26 @@ pub(crate) struct CollectedPage { pub(crate) html: String, pub(crate) script_tags: Vec, pub(crate) network_requests: Vec, + /// Slots read from the live GPT registry (`googletag.pubads().getSlots()`). + /// + /// Populated at `defineSlot` time, so this captures configured slots even + /// when the ad request never fires (consent-gated or iframe-issued). + #[serde(default)] + pub(crate) gpt_slots: Vec, pub(crate) warnings: Vec, } +/// A single slot read from the page's live GPT registry. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedGptSlot { + /// The GAM ad-unit path (`slot.getAdUnitPath()`). + pub(crate) gam_unit_path: String, + /// The slot's div element id (`slot.getSlotElementId()`). + pub(crate) div_id: String, + /// Numeric `[width, height]` sizes (`slot.getSizes()`, fluid entries dropped). + pub(crate) sizes: Vec<(u32, u32)>, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct CollectedScriptTag { pub(crate) src: Option, diff --git a/crates/trusted-server-cli/src/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/audit/generate/gpt_slots.rs new file mode 100644 index 000000000..8ee09f71b --- /dev/null +++ b/crates/trusted-server-cli/src/audit/generate/gpt_slots.rs @@ -0,0 +1,585 @@ +//! Reconstructs `[creative_opportunities]` slots from a live page's GPT state. +//! +//! Two complementary sources feed the reconstruction: +//! +//! 1. The **live GPT registry** (`googletag.pubads().getSlots()`) is the primary +//! source. It exposes each defined slot's ad-unit path, div id, and sizes +//! directly, and is populated at `defineSlot` time — so it captures slots even +//! when the ad request never fires (consent-gated stacks, iframe-issued +//! requests). It carries no per-slot header-bidding signal, so Prebid is +//! inferred from page-level detection. +//! 2. Captured **`gampad/ads` requests** are a fallback for any div the registry +//! did not report. Each request URL encodes the ad-unit path (`iu_parts`), div +//! id (`dids`), sizes (`prev_iu_szs`), and targeting (`prev_scp`, which does +//! carry a per-slot Prebid signal). +//! +//! Neither source executes the page's ad-stack logic ourselves; both read state +//! the page's own GPT/Prebid setup produced. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; +use url::Url; + +use crate::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; + +/// A hyphen-delimited hex hash *segment* (16+ hex chars bounded by `-` or end), +/// e.g. the UUID GPT embeds in `ad-in_content--in_content-0`. Marks the +/// start of ephemeral div-id noise, like the React `_R_` hash. The trailing +/// boundary avoids truncating a legit token that merely starts with hex-like +/// characters (only `start()` of the match is used). +static HEX_HASH_SEGMENT: LazyLock = + LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); + +/// Hosts that serve GPT `gampad/ads` requests. +const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; + +/// Common GPT div-id prefix stripped when deriving a slot id. +const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; + +/// Minimum width/height for a format to be treated as a real creative size. +/// +/// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside +/// pixel sizes in `prev_iu_szs`; those are not banner dimensions, so they are +/// dropped from the drafted `formats`. +const MIN_FORMAT_DIMENSION: u32 = 50; + +/// A slot reconstructed from a single GPT ad request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiscoveredSlot { + /// Slot id derived from the div id (GPT prefix stripped). + pub(crate) id: String, + /// The HTML div id that holds the creative. + pub(crate) div_id: String, + /// The full GAM ad-unit path (e.g. `/123/desktop/homepage/leaderboard`). + pub(crate) gam_unit_path: String, + /// Candidate creative sizes as `(width, height)` pixel pairs. + pub(crate) formats: Vec<(u32, u32)>, + /// Whether the slot's targeting shows Prebid/header-bidding signals. + pub(crate) has_prebid: bool, +} + +/// The result of scanning captured requests for GPT slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct DiscoveredSlots { + /// GAM network id shared by the discovered slots, if any were found. + pub(crate) gam_network_id: Option, + /// The reconstructed slots, deduplicated by div id in first-seen order. + pub(crate) slots: Vec, +} + +/// Reconstructs GPT slots from the page's live registry and ad requests. +/// +/// The live registry (`googletag.pubads().getSlots()`) is the primary source: it +/// carries the authoritative path/div/size for every defined slot and is present +/// even when the ad request never fires. Captured `gampad/ads` requests are a +/// fallback for any div the registry did not report, and also supply per-slot +/// Prebid signals. Slots are deduplicated by div id in first-seen order. +/// +/// `page_has_prebid` marks registry slots as Prebid-enabled when the page as a +/// whole was detected running Prebid (the registry alone carries no such signal). +pub(crate) fn discover_gpt_slots( + registry: &[CollectedGptSlot], + requests: &[CollectedRequest], + page_has_prebid: bool, +) -> DiscoveredSlots { + let mut slots = Vec::new(); + let mut gam_network_id = None; + let mut seen_divs = BTreeSet::new(); + + for entry in registry { + let Some(slot) = slot_from_registry(entry, page_has_prebid) else { + continue; + }; + if !seen_divs.insert(slot.div_id.clone()) { + continue; + } + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&slot.gam_unit_path); + } + slots.push(slot); + } + + for request in requests { + let Some((network_id, slot)) = parse_gampad_request(&request.url) else { + continue; + }; + if !seen_divs.insert(slot.div_id.clone()) { + continue; + } + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } + slots.push(slot); + } + + DiscoveredSlots { + gam_network_id, + slots, + } +} + +/// Converts a live-registry slot into a [`DiscoveredSlot`]. +/// +/// Returns `None` when the slot has no usable pixel size or its div id is a +/// multi-slot (SRA) concatenation rather than a single element. +fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option { + if is_multi_slot_div(&entry.div_id) { + return None; + } + let formats: Vec<(u32, u32)> = entry + .sizes + .iter() + .copied() + .filter(|(width, height)| *width >= MIN_FORMAT_DIMENSION && *height >= MIN_FORMAT_DIMENSION) + .collect(); + if formats.is_empty() { + return None; + } + let div_stem = normalize_div_stem(&entry.div_id); + Some(DiscoveredSlot { + id: slot_id_from_div(&div_stem), + div_id: div_stem, + gam_unit_path: entry.gam_unit_path.clone(), + formats, + has_prebid: page_has_prebid, + }) +} + +/// Whether a div id is a GPT single-request (SRA) concatenation of multiple +/// slots (joined with `~`) rather than one element. +fn is_multi_slot_div(div_id: &str) -> bool { + div_id.contains('~') +} + +/// Strips ephemeral GPT div-id noise so the stored id is stable across renders. +/// +/// Removes a trailing `-container` wrapper, then truncates at the first ephemeral +/// marker — a React SSR hash (`_R_`) or a hex-UUID segment — since both +/// change on every page load. Truncating (rather than excising) keeps the result +/// a valid **prefix** of the live div id, which is how verify matches slots. +/// +/// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` +/// → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` → `ad-in_content`. +fn normalize_div_stem(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut cut = stem.len(); + if let Some(pos) = stem.find("_R_") { + cut = cut.min(pos); + } + if let Some(matched) = HEX_HASH_SEGMENT.find(stem) { + cut = cut.min(matched.start()); + } + stem[..cut].trim_end_matches('-').to_string() +} + +/// Extracts the leading network id from a GAM ad-unit path (`//...`). +fn network_id_from_unit_path(path: &str) -> Option { + let segment = path.trim_start_matches('/').split('/').next()?; + (!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| segment.to_string()) +} + +/// Parses a single `gampad/ads` request URL into `(network_id, slot)`. +/// +/// Returns `None` when the URL is not a GPT ad request or is missing the fields +/// needed to describe a slot (ad-unit path, div id, and at least one size). +fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { + let url = Url::parse(raw_url).ok()?; + let host = url.host_str()?; + if !GAMPAD_HOSTS.contains(&host) || !url.path().ends_with("/gampad/ads") { + return None; + } + + let mut iu_parts = None; + let mut dids = None; + let mut sizes_raw = None; + let mut fallback_sizes_raw = None; + let mut scp = None; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "iu_parts" => iu_parts = Some(value.into_owned()), + "dids" => dids = Some(value.into_owned()), + "prev_iu_szs" => sizes_raw = Some(value.into_owned()), + "pb_szs" => fallback_sizes_raw = Some(value.into_owned()), + "prev_scp" => scp = Some(value.into_owned()), + _ => {} + } + } + + let iu_parts = iu_parts?; + let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); + let network_id = parts.next()?.to_string(); + let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + // A usable unit path needs the network id plus at least one path segment. + parts.next()?; + + let raw_div = dids? + .split(',') + .map(str::trim) + .find(|did| !did.is_empty())? + .to_string(); + if is_multi_slot_div(&raw_div) { + return None; + } + let div_id = normalize_div_stem(&raw_div); + + let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); + if formats.is_empty() { + return None; + } + + let id = slot_id_from_div(&div_id); + let has_prebid = scp.as_deref().is_some_and(scp_shows_prebid); + + Some(( + network_id, + DiscoveredSlot { + id, + div_id, + gam_unit_path, + formats, + has_prebid, + }, + )) +} + +/// Parses a GPT size list (e.g. `970x250|4x1|620x366`) into pixel pairs. +/// +/// Accepts `|` or `,` separators, ignores non-`WxH` tokens, and drops +/// fluid/native ratio markers below [`MIN_FORMAT_DIMENSION`]. +fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { + let mut sizes = Vec::new(); + for token in raw.split(['|', ',']) { + let Some((width, height)) = token.trim().split_once('x') else { + continue; + }; + let (Ok(width), Ok(height)) = (width.parse::(), height.parse::()) else { + continue; + }; + if width < MIN_FORMAT_DIMENSION || height < MIN_FORMAT_DIMENSION { + continue; + } + if !sizes.contains(&(width, height)) { + sizes.push((width, height)); + } + } + sizes +} + +/// Derives a slot id from a div id by stripping the common GPT prefix. +fn slot_id_from_div(div_id: &str) -> String { + div_id + .strip_prefix(GPT_DIV_PREFIX) + .unwrap_or(div_id) + .to_string() +} + +/// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. +fn scp_shows_prebid(scp: &str) -> bool { + let scp = scp.to_ascii_lowercase(); + scp.contains("test=prebid") || scp.contains("tude=true") || scp.contains("prebid") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sample GPT leaderboard ad request (truncated to the fields the + /// parser reads; values are otherwise unmodified live output). + const SAMPLE_LEADERBOARD: &str = "https://securepubads.g.doubleclick.net/gampad/ads?\ + gdfp_req=1&iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C8x1%7C620x366%7C325x508%7C325x204\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=ad-loc%3Dleaderboard-1%26baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid%26tude%3Dtrue\ + &pb_szs=970x250%7C620x366"; + + fn request(url: &str) -> CollectedRequest { + CollectedRequest { + url: url.to_string(), + resource_type: Some("fetch".to_string()), + } + } + + /// Discovers slots from ad requests only (no live registry). + fn from_requests(requests: &[CollectedRequest]) -> DiscoveredSlots { + discover_gpt_slots(&[], requests, false) + } + + #[test] + fn parses_leaderboard_slot() { + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD)]); + + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_eq!(discovered.slots.len(), 1, "should find one slot"); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1", "should strip the GPT div prefix"); + assert_eq!(slot.div_id, "div-gpt-ad-leaderboard-1"); + assert_eq!( + slot.gam_unit_path, + "/123456789/desktop/homepage/leaderboard1" + ); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366), (325, 508), (325, 204)], + "should keep pixel sizes and drop 4x1/8x1 fluid markers" + ); + assert!(slot.has_prebid, "prev_scp test=prebid should flag prebid"); + } + + #[test] + fn deduplicates_refreshed_slot_requests() { + // GPT refreshes the same slot; a second identical request must not + // produce a duplicate slot. + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD), request(SAMPLE_LEADERBOARD)]); + + assert_eq!( + discovered.slots.len(), + 1, + "repeat requests for the same div should collapse" + ); + } + + #[test] + fn ignores_non_gampad_requests() { + let discovered = from_requests(&[ + request("https://securepubads.g.doubleclick.net/tag/js/gpt.js"), + request("https://cdn.example.com/app.js"), + request("https://analytics.example.com/collect?iu_parts=1%2Cfoo&dids=x"), + ]); + + assert!( + discovered.slots.is_empty(), + "only doubleclick gampad/ads requests should yield slots" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn skips_requests_missing_sizes() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x", + )]); + + assert!( + discovered.slots.is_empty(), + "a slot with no usable size should be skipped" + ); + } + + #[test] + fn skips_requests_with_only_network_id() { + // iu_parts with just the network id yields no unit path segment. + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a bare network id is not a usable ad-unit path" + ); + } + + #[test] + fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x&pb_szs=300x250%7C728x90", + )]); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!(discovered.slots[0].formats, vec![(300, 250), (728, 90)]); + } + + fn registry_slot(path: &str, div: &str, sizes: &[(u32, u32)]) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: path.to_string(), + div_id: div.to_string(), + sizes: sizes.to_vec(), + } + } + + #[test] + fn reads_slots_from_live_registry() { + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250), (1, 1), (620, 366)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], true); + + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "network id should come from the unit path" + ); + assert_eq!(discovered.slots.len(), 1); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1"); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366)], + "should drop the 1x1 out-of-page marker" + ); + assert!( + slot.has_prebid, + "page-level prebid should mark registry slots" + ); + } + + #[test] + fn registry_wins_and_requests_fill_gaps() { + // The registry reports the leaderboard; a gampad request reports a + // different div that the registry missed. Both should appear once. + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250)], + )]; + let requests = vec![ + // Same div as the registry — must not duplicate. + request(SAMPLE_LEADERBOARD), + // A div the registry did not report — must be added. + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Cdesktop%2Chomepage%2Csidebar1&dids=div-gpt-ad-sidebar-1&prev_iu_szs=300x600", + ), + ]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + let ids: Vec<&str> = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["leaderboard-1", "sidebar-1"], + "registry slot kept, request fills the missing div, no duplicate" + ); + } + + #[test] + fn registry_slot_without_pixel_sizes_is_skipped() { + let registry = vec![registry_slot("/123/fluid", "div-gpt-ad-fluid", &[(1, 1)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a registry slot with only fluid markers is not usable" + ); + } + + #[test] + fn normalizes_ephemeral_hash_and_container_and_dedups() { + // A framework-hashed div: the same placement appears as a hashed inner div, + // a `-container` wrapper, and re-rendered with a different hash. All must + // collapse to one stable stem. + let registry = vec![ + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_", + &[(728, 90)], + ), + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_-container", + &[(728, 90)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hash + container variants collapse" + ); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "ephemeral React hash and -container are stripped to a stable stem" + ); + assert_eq!(discovered.slots[0].id, "ad-header-0"); + } + + #[test] + fn drops_sra_multi_slot_concatenations() { + let registry = vec![registry_slot( + "/987654321/homepage/header-0/fixed_bottom-0", + "ad-header-0-_R_9slin~ad-fixed_bottom-0-_R_ainp", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "tilde-joined SRA multi-slot divs are not real single elements" + ); + } + + #[test] + fn leaves_clean_div_ids_unchanged() { + assert_eq!( + normalize_div_stem("div-gpt-ad-leaderboard-1"), + "div-gpt-ad-leaderboard-1" + ); + } + + #[test] + fn normalizes_react_and_hex_hashes_to_stable_prefixes() { + assert_eq!( + normalize_div_stem("ad-header-0-_R_9slinpflik6lb_-container"), + "ad-header-0" + ); + let stem = + normalize_div_stem("ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0"); + assert_eq!(stem, "ad-in_content"); + assert!( + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0".starts_with(&stem), + "stem must prefix-match any re-rendered hex variant" + ); + } + + #[test] + fn hex_hash_truncation_requires_a_segment_boundary() { + // Hex UUID bounded by `-` → truncated to the stem. + assert_eq!( + normalize_div_stem("ad-x-de669245b2ea4b05826dc96f07a36272-y"), + "ad-x" + ); + // A token that merely starts with 16 hex chars (no boundary) is left intact. + assert_eq!( + normalize_div_stem("ad-de669245b2ea4b05z"), + "ad-de669245b2ea4b05z" + ); + } + + #[test] + fn hex_normalized_in_content_slots_dedup() { + // Same in_content placement, different per-render hex — one stable slot. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hex variants collapse to one slot" + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + } +} diff --git a/crates/trusted-server-cli/src/audit/generate/mod.rs b/crates/trusted-server-cli/src/audit/generate/mod.rs index 6d06e8698..74453f6ba 100644 --- a/crates/trusted-server-cli/src/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/audit/generate/mod.rs @@ -1,13 +1,18 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; +mod gpt_slots; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use serde::Serialize; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; use url::Url; use crate::audit::generate::collector::AuditCollector; @@ -37,6 +42,11 @@ pub(crate) struct GenerateArgs { /// Overwrite existing output files. #[arg(long)] pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::audit::parse_cookie)] + pub(crate) cookies: Vec<(String, String)>, } const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; @@ -82,6 +92,7 @@ pub(crate) struct AuditOutputs { pub(crate) artifact: AuditArtifact, pub(crate) js_assets_toml: String, pub(crate) draft_config_toml: String, + pub(crate) ad_slot_count: usize, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -97,7 +108,7 @@ pub(crate) fn run_generate( ) -> CliResult<()> { let target_url = parse_audit_url(&args.url)?; let plan = resolve_output_plan(args)?; - let collected = collector.collect_page(&target_url)?; + let collected = collector.collect_page(&target_url, &args.cookies)?; let outputs = build_audit_outputs(&collected)?; let wrote_config = plan.config_path.is_some(); let written = write_audit_outputs(&outputs, &plan)?; @@ -175,12 +186,23 @@ fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult"), outputs.artifact.js_asset_count, outputs.artifact.third_party_asset_count, + outputs.ad_slot_count, if integrations.is_empty() { "none".to_string() } else { @@ -269,7 +292,11 @@ fn write_success_summary( .map_err(|error| report_error(format!("failed to write command output: {error}"))) } -fn build_draft_config(target_url: &Url, artifact: &AuditArtifact) -> CliResult { +fn build_draft_config( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, +) -> CliResult { let host = target_url .host_str() .ok_or_else(|| report_error("audited URL is missing a host"))?; @@ -353,9 +380,526 @@ fn build_draft_config(target_url: &Url, artifact: &AuditArtifact) -> CliResult String { + let path = target_url.path(); + let page_pattern = if path.is_empty() { "/" } else { path }; + + let mut out = String::from( + "\n# Slots discovered from live GPT ad requests during the audit.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in &slots.slots { + let formats = slot + .formats + .iter() + .map(|(width, height)| format!("{{ width = {width}, height = {height} }}")) + .collect::>() + .join(", "); + out.push_str(&format!( + "\n[[creative_opportunities.slot]]\n\ + id = \"{id}\"\n\ + div_id = \"{div_id}\"\n\ + gam_unit_path = \"{gam_unit_path}\"\n\ + page_patterns = [\"{page_pattern}\"]\n\ + formats = [{formats}]\n", + id = slot.id, + div_id = slot.div_id, + gam_unit_path = slot.gam_unit_path, + )); + if slot.has_prebid { + out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); + } + } + out +} + +/// Runs `ts audit ad-templates generate`: scrape the live page's GPT slots and +/// rewrite only the `[creative_opportunities]` slot array in `config_path` in +/// place, preserving every other section and comment. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the page cannot be +/// collected, no slots are discovered, or the config has no +/// `[creative_opportunities]` section to update. +#[allow(clippy::too_many_arguments, reason = "cohesive one-shot command entry")] +pub(crate) fn run_update_slots( + url: &str, + config_path: &Path, + existing_creative: Option<&CreativeOpportunitiesConfig>, + page_patterns: &[String], + replace: bool, + cookies: &[(String, String)], + dry_run: bool, + collector: &dyn AuditCollector, + out: &mut dyn Write, +) -> CliResult<()> { + let target_url = parse_audit_url(url)?; + let existing = fs::read_to_string(config_path).map_err(|error| { + report_error(format!( + "failed to read config {}: {error}", + config_path.display() + )) + })?; + + let collected = collector.collect_page(&target_url, cookies)?; + let artifact = analyze_collected_page(&collected)?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + if discovered.slots.is_empty() { + return cli_error("no ad-template slots were discovered on the page"); + } + + // Patterns for slots seen on this run: the `--page-pattern` values, or the + // audited path when none are given (preserving single-page behavior). + let run_patterns: Vec = if page_patterns.is_empty() { + vec![default_page_pattern(&target_url)] + } else { + page_patterns.to_vec() + }; + + let merged = merge_slots(existing_creative, &discovered, &run_patterns, replace); + let network_id = resolve_network_id( + existing_creative, + discovered.gam_network_id.as_deref(), + replace, + ); + let rendered_slots = render_slots(&merged); + let updated = splice_creative_slots(&existing, network_id.as_deref(), &rendered_slots)?; + + if dry_run { + writeln!(out, "{updated}") + .map_err(|error| report_error(format!("failed to write preview: {error}")))?; + return Ok(()); + } + fs::write(config_path, &updated).map_err(|error| { + report_error(format!( + "failed to write config {}: {error}", + config_path.display() + )) + })?; + writeln!( + out, + "Wrote {} slot(s) to {} ({} discovered this run)", + merged.len(), + config_path.display(), + discovered.slots.len(), + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// [`merge_slots`], which returns discovered-only in those cases. +fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +/// The default page pattern for a scraped URL: its path, or `/` for the root. +fn default_page_pattern(target_url: &Url) -> String { + let path = target_url.path(); + if path.is_empty() { + "/".to_string() + } else { + path.to_string() + } +} + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable identity used to match slots across runs: the div id (or slot + /// id), with any trailing `-` trimmed so hand-authored stems still match. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has `run_patterns` unioned into its +/// `page_patterns`; slots seen only this run are appended. +fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return discovered_slots; + } + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + for slot in discovered_slots { + let key = slot.key(); + if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + } else { + merged.push(slot); + } + } + merged +} + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = String::from( + "\n# Slots managed by `ts audit ad-templates generate`.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + let patterns = slot + .page_patterns + .iter() + .map(|pattern| toml_string(pattern)) + .collect::>() + .join(", "); + out.push_str(&format!("page_patterns = [{patterns}]\n")); + let formats = slot + .formats + .iter() + .map(|(width, height, media_type)| match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }) + .collect::>() + .join(", "); + out.push_str(&format!("formats = [{formats}]\n")); + if let Some(floor) = slot.floor_price { + out.push_str(&format!("floor_price = {floor}\n")); + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + control if (control as u32) < 0x20 => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// Rewrites the `[creative_opportunities]` slot array of `existing` with the +/// pre-rendered `rendered_slots` text, updating `gam_network_id` and preserving +/// all other sections and comments. +/// +/// If the config has no `[creative_opportunities]` section, a fresh one is +/// appended so `generate` works against a config that omits it. +fn splice_creative_slots( + existing: &str, + network_id: Option<&str>, + rendered_slots: &str, +) -> CliResult { + let rendered = rendered_slots.trim_matches('\n'); + + // No section yet — append a fresh one with the network id and slots. + if !existing + .lines() + .any(|line| line.trim() == "[creative_opportunities]") + { + let mut result = existing.to_string(); + if !result.is_empty() && !result.ends_with('\n') { + result.push('\n'); + } + result.push_str("\n[creative_opportunities]\n"); + if let Some(network_id) = network_id { + result.push_str(&format!("gam_network_id = \"{network_id}\"\n")); + } + result.push_str(rendered); + result.push('\n'); + return Ok(result); + } + + // Section exists — update `gam_network_id` (best-effort) and replace slots. + let mut document = existing.to_string(); + if let Some(network_id) = network_id { + if let Ok(updated) = replace_key_in_section( + &document, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = \"{network_id}\""), + ) { + document = updated; + } + } + + let lines: Vec<&str> = document.lines().collect(); + let header = lines + .iter() + .position(|line| line.trim() == "[creative_opportunities]") + .ok_or_else(|| { + report_error("target config has no [creative_opportunities] section to update") + })?; + + let is_slot_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with("[[creative_opportunities.slot]]") + || trimmed.starts_with("[creative_opportunities.slot.") + }; + let is_unrelated_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" + }; + + // Where the existing slot array begins (first slot table after the header), + // else the end of the scalar block (first unrelated table, or EOF). + let existing_start = lines[header + 1..] + .iter() + .position(|line| is_slot_table(line)) + .map(|offset| header + 1 + offset); + let start = existing_start.unwrap_or_else(|| { + lines[header + 1..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| header + 1 + offset) + }); + // Where the slot array ends: first unrelated top-level table, or EOF. + let end = lines[start..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| start + offset); + + let mut result = lines[..start].join("\n"); + if !result.is_empty() { + result.push('\n'); + } + result.push_str(rendered); + result.push('\n'); + let tail = lines[end..].join("\n"); + if !tail.is_empty() { + result.push('\n'); + result.push_str(&tail); + } + if existing.ends_with('\n') && !result.ends_with('\n') { + result.push('\n'); + } + Ok(result) +} + fn replace_key_in_section( document: &str, section: &str, @@ -432,7 +976,11 @@ mod tests { } impl AuditCollector for FakeCollector { - fn collect_page(&self, _target_url: &Url) -> CliResult { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { self.calls.set(self.calls.get() + 1); Ok(self.collected.clone()) } @@ -458,6 +1006,7 @@ mod tests { url: "https://cdn.publisher.example/app.js".to_string(), resource_type: Some("script".to_string()), }], + gpt_slots: Vec::new(), warnings: Vec::new(), } } @@ -470,6 +1019,7 @@ mod tests { no_js_assets: false, no_config: false, force: false, + cookies: Vec::new(), } } @@ -552,6 +1102,7 @@ mod tests { no_js_assets: false, no_config: false, force: false, + cookies: Vec::new(), }; let collector = FakeCollector::new(collected_page()); let mut out = Vec::new(); @@ -675,7 +1226,8 @@ mod tests { warnings: Vec::new(), }; - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); assert!(draft.contains("domain = \"www.publisher.example\"")); assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); @@ -703,9 +1255,316 @@ mod tests { warnings: Vec::new(), }; - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); assert!(draft.contains("Detected google_tag_manager")); } + + #[test] + fn build_audit_outputs_reconstructs_creative_opportunity_slots() { + let collected = CollectedPage { + requested_url: "https://example.com/".to_string(), + final_url: "https://example.com/".to_string(), + page_title: Some("Example Publisher".to_string()), + html: "".to_string(), + script_tags: Vec::new(), + network_requests: vec![CollectedRequest { + url: "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C620x366\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid" + .to_string(), + resource_type: Some("fetch".to_string()), + }], + gpt_slots: Vec::new(), + warnings: Vec::new(), + }; + + let outputs = build_audit_outputs(&collected).expect("should build outputs"); + assert_eq!(outputs.ad_slot_count, 1, "should discover one slot"); + + // The drafted config must be valid TOML with the reconstructed slot. + let value = + toml::from_str::(&outputs.draft_config_toml).expect("draft parses"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("123456789")); + let slot = &creative["slot"][0]; + assert_eq!(slot["id"].as_str(), Some("leaderboard-1")); + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/123456789/desktop/homepage/leaderboard1") + ); + assert_eq!( + slot["formats"][0]["width"].as_integer(), + Some(970), + "should keep the 970x250 pixel size" + ); + assert!( + slot["providers"]["prebid"].is_table(), + "prev_scp test=prebid should emit a prebid provider" + ); + } + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn default_page_pattern_uses_path_or_root() { + assert_eq!( + default_page_pattern(&Url::parse("https://x/news/story").expect("url")), + "/news/story" + ); + assert_eq!( + default_page_pattern(&Url::parse("https://x/").expect("url")), + "/" + ); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } } diff --git a/crates/trusted-server-cli/src/audit/mod.rs b/crates/trusted-server-cli/src/audit/mod.rs index 3bcbd6924..ba1378ca5 100644 --- a/crates/trusted-server-cli/src/audit/mod.rs +++ b/crates/trusted-server-cli/src/audit/mod.rs @@ -32,6 +32,24 @@ pub(crate) fn parse_http_url(raw: &str) -> Result { } } +/// Parses a `name=value` cookie argument into its `(name, value)` parts. +/// +/// Splits on the first `=` so cookie values may themselves contain `=`. The name +/// must be non-empty; the value may be empty. +/// +/// # Errors +/// +/// Returns a user-facing string when the input has no `=` or an empty name. +pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { + let (name, value) = raw + .split_once('=') + .ok_or_else(|| format!("invalid cookie `{raw}` (expected NAME=VALUE)"))?; + if name.is_empty() { + return Err(format!("invalid cookie `{raw}` (empty name)")); + } + Ok((name.to_string(), value.to_string())) +} + /// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. #[derive(Debug, Args)] pub(crate) struct AuditArgs { @@ -57,10 +75,39 @@ pub(crate) enum AuditSubcommand { /// `ts audit ad-templates` subcommands. #[derive(Debug, Subcommand)] pub(crate) enum AuditAdTemplatesCommand { + /// Scrape a live page's GPT slots and update the config's + /// `[creative_opportunities]` slots in place. + Generate(AuditAdTemplatesGenerateArgs), /// Verify ad-template slots for one or more live URLs. Verify(AuditAdTemplatesVerifyArgs), } +/// Arguments for `ts audit ad-templates generate `. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesGenerateArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page URL to scrape for GPT slots (http or https). + #[arg(value_parser = parse_http_url)] + pub url: url::Url, + /// Glob applied to every slot discovered this run (e.g. `/`, `/news/*`). + /// Repeatable. Defaults to the scraped URL's path. Re-running with a + /// different pattern unions it into slots already in the config. + #[arg(long = "page-pattern", value_name = "GLOB")] + pub page_patterns: Vec, + /// Replace all existing slots instead of merging this run into them. + #[arg(long)] + pub replace: bool, + /// Preview the updated config on stdout instead of writing it. + #[arg(long)] + pub dry_run: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, +} + /// Arguments for `ts audit ad-templates verify ...`. #[derive(Debug, Args)] pub(crate) struct AuditAdTemplatesVerifyArgs { @@ -78,6 +125,11 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// Perform a deterministic scroll pass after the initial settle. #[arg(long)] pub scroll: bool, + /// Cookie to send with each page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, #[command(flatten)] pub browser: BrowserOpts, } @@ -94,6 +146,23 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { match &args.command { Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { + let loaded = crate::app_config::load_settings(&gen_args.config)?; + let collector = generate::browser_collector::BrowserAuditCollector; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + generate::run_update_slots( + gen_args.url.as_str(), + &loaded.app_config_path, + loaded.settings.creative_opportunities.as_ref(), + &gen_args.page_patterns, + gen_args.replace, + &gen_args.cookies, + gen_args.dry_run, + &collector, + &mut out, + ) + } Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { ad_templates::run_verify(verify_args) } @@ -109,3 +178,40 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { }, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_cookie_splits_on_first_equals() { + let (name, value) = parse_cookie("datadome=abc=def~ghi").expect("should parse cookie"); + assert_eq!(name, "datadome", "name should be the pre-`=` portion"); + assert_eq!( + value, "abc=def~ghi", + "value should keep later `=` characters" + ); + } + + #[test] + fn parse_cookie_allows_empty_value() { + let (name, value) = parse_cookie("session=").expect("should parse empty value"); + assert_eq!(name, "session"); + assert!(value.is_empty(), "empty value should be allowed"); + } + + #[test] + fn parse_cookie_rejects_missing_equals() { + let err = parse_cookie("datadome").expect_err("should reject missing `=`"); + assert!( + err.contains("NAME=VALUE"), + "error should show expected form" + ); + } + + #[test] + fn parse_cookie_rejects_empty_name() { + let err = parse_cookie("=value").expect_err("should reject empty name"); + assert!(err.contains("empty name"), "error should name the problem"); + } +} diff --git a/crates/trusted-server-cli/src/audit/page.rs b/crates/trusted-server-cli/src/audit/page.rs index 648a09f5d..cda9970dd 100644 --- a/crates/trusted-server-cli/src/audit/page.rs +++ b/crates/trusted-server-cli/src/audit/page.rs @@ -53,6 +53,7 @@ fn run_with_collector( init_scripts: Vec::new(), scroll, collect_ad_evidence: false, + cookies: Vec::new(), })?; let stdout = io::stdout(); From b3374ae5972caca8e24900671c0d685e6484da3f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 7 Jul 2026 13:22:26 +0530 Subject: [PATCH 135/195] Apply optional follow-ups from server-side ad-template review - Roll SPA page-bids navigation `currentPath` back to the last applied path instead of the immediately-previous one, so an aborted-then-failed navigation can no longer strand a route behind the no-op guard; add a regression test. - Add a concurrent render-bridge test: two same-adId messages before the cache fetch resolves must collapse to one fetch (in-flight gate), two beacons. - Assert `OPTIONS /__ts/page-bids` is denied with 403 on every adapter (Axum/Cloudflare/Spin) in cross-adapter parity. - Add a `u32::MAX` banner-format test covering the imp-drop branch when all formats exceed `i32::MAX`. - Dedup the page-bids GET 403 into `page_bids_preflight_denied()`. - Fix stale comments/docs: `buffer_publisher_response_async`, soften the oversized-body comment, and correct the `firedBeacons` key doc. --- .../trusted-server-adapter-fastly/src/app.rs | 2 +- .../src/auction/endpoints.rs | 2 +- .../src/integrations/prebid.rs | 32 +++++++++ crates/trusted-server-core/src/publisher.rs | 14 ++-- .../tests/parity.rs | 68 +++++++++++++++++++ .../trusted-server-js/lib/src/core/types.ts | 2 +- .../lib/src/integrations/gpt/index.ts | 14 +++- .../lib/test/integrations/gpt/ad_init.test.ts | 53 +++++++++++++++ .../test/integrations/gpt/spa_hook.test.ts | 53 +++++++++++++++ 9 files changed, 225 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 27fb18918..5321c32cd 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -718,7 +718,7 @@ async fn dispatch_fallback( } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by publisher.max_buffered_body_bytes. // Only the handle_publisher_request branch below routes through - // buffer_publisher_response. Integration responses are small in practice + // buffer_publisher_response_async. Integration responses are small in practice // and the EdgeZero flag is off by default; extend the cap here if that changes. state .registry diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index c642c9ec3..56c104ab6 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -110,7 +110,7 @@ pub async fn handle_auction( services: &RuntimeServices, req: Request, ) -> Result, Report> { - // Reject oversized bodies before any allocation. The Content-Length + // Reject oversized bodies before core buffers/parses them. The Content-Length // pre-check stops well-behaved clients early; the post-read check defends // against clients that lie about (or omit) the header. let content_length_exceeded = req diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index dd73dabae..596eb7d33 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3472,6 +3472,38 @@ server_url = "https://prebid.example" assert_eq!(formats[0].h, Some(250), "should preserve valid height"); } + #[test] + fn to_openrtb_drops_imp_when_all_banner_formats_exceed_i32_max() { + // The build-time bound: every banner format's u32 dimensions pass through + // `to_openrtb_i32`, which omits any value above i32::MAX. When a slot's + // only format is out of range (here u32::MAX), no valid formats remain, so + // the whole imp must be dropped rather than emitted with an empty format + // list — a sizeless imp is unbiddable and would only waste an SSP call. + let provider = PrebidAuctionProvider::new(base_config()); + let mut auction_request = create_test_auction_request(); + auction_request.slots[0].formats = vec![AdFormat { + media_type: MediaType::Banner, + width: u32::MAX, + height: u32::MAX, + }]; + + let settings = make_settings(); + let request = build_test_request(); + let context = create_test_auction_context(&settings, &request); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert!( + openrtb.imp.is_empty(), + "should drop the imp entirely when every banner format exceeds i32::MAX" + ); + } + #[test] fn to_openrtb_sets_site_ref_from_referer_header() { let provider = PrebidAuctionProvider::new(base_config()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d9944ad95..7482de88a 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1915,8 +1915,10 @@ fn page_bids_request_allowed(req: &Request) -> bool { } } -/// Builds the `403 Forbidden` returned for a CORS preflight (`OPTIONS`) to the -/// side-effecting `/__ts/page-bids` endpoint. +/// Builds the `403 Forbidden` returned when the side-effecting +/// `/__ts/page-bids` endpoint refuses a request — both the CORS preflight +/// (`OPTIONS`) and the GET cross-site gate ([`page_bids_request_allowed`]) +/// return this single denial shape. /// /// The GET handler's [`page_bids_request_allowed`] gate trusts the /// `X-TSJS-Page-Bids` header precisely because this endpoint never grants a @@ -1985,13 +1987,7 @@ pub async fn handle_page_bids( .and_then(|v| v.to_str().ok()), req.headers().contains_key("x-tsjs-page-bids") ); - let mut response = Response::new(EdgeBody::from("Forbidden")); - *response.status_mut() = StatusCode::FORBIDDEN; - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - return Ok(response); + return Ok(page_bids_preflight_denied()); } let path_param = req diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index a5f32b275..e85b1d8d1 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -265,6 +265,48 @@ async fn spin_authorized_json(method: &str, uri: &str, body: &str) -> (u16, Head (resp.status().as_u16(), resp.headers().clone()) } +/// Send an OPTIONS request to the Axum adapter and return (status, headers). +async fn axum_options(uri: &str) -> (u16, HeaderMap) { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("OPTIONS") + .uri(uri) + .body(AxumBody::empty()) + .expect("should build OPTIONS request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + +/// Send an OPTIONS request to the Cloudflare adapter and return (status, headers). +async fn cf_options(uri: &str) -> (u16, HeaderMap) { + let router = cf_router(); + let req = request_builder() + .method("OPTIONS") + .uri(uri) + .body(edgezero_core::body::Body::empty()) + .expect("should build OPTIONS request"); + let resp = router.oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + +/// Send an OPTIONS request to the Spin adapter and return (status, headers). +async fn spin_options(uri: &str) -> (u16, HeaderMap) { + let router = spin_router(); + let req = request_builder() + .method("OPTIONS") + .uri(uri) + .body(edgezero_core::body::Body::empty()) + .expect("should build OPTIONS request"); + let resp = router.oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + // --------------------------------------------------------------------------- // Route parity: same route → same status on all adapters // --------------------------------------------------------------------------- @@ -652,6 +694,32 @@ async fn auction_not_challenged_by_auth_parity() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn page_bids_options_preflight_denied_parity() { + // OPTIONS /__ts/page-bids is a CORS preflight to a side-effecting endpoint. + // Every adapter must refuse it with 403 rather than proxy it to the origin: + // a permissive origin preflight would let a cross-site page defeat the GET + // handler's `X-TSJS-Page-Bids` gate and trigger real auctions in a visitor's + // browser. The denial is unconditional (independent of creative-opportunity + // configuration), so all adapters must agree on 403. + let (axum_status, _) = axum_options("/__ts/page-bids").await; + let (cf_status, _) = cf_options("/__ts/page-bids").await; + let (spin_status, _) = spin_options("/__ts/page-bids").await; + + assert_eq!( + axum_status, 403, + "Axum OPTIONS /__ts/page-bids must be denied with 403, got {axum_status}" + ); + assert_eq!( + cf_status, 403, + "Cloudflare OPTIONS /__ts/page-bids must be denied with 403, got {cf_status}" + ); + assert_eq!( + spin_status, 403, + "Spin OPTIONS /__ts/page-bids must be denied with 403, got {spin_status}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn spin_auction_ignores_spoofed_forwarded_headers() { // POST /auction feeds prebid request signing via `RequestInfo::from_request`, diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index ec2882efb..360e2aa49 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -99,7 +99,7 @@ export interface TsjsApi { /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; /** - * Win/billing beacons already fired, keyed by `slotId|bidIdentity`. + * Win/billing beacons already fired, keyed by `slotId|bidIdentity|kind|url`. * Used by the GPT render bridge so a bid's nurl/burl fire at most once even * across repeated Prebid Universal Creative requests for the same adId. */ diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8b0b8d529..ca4689684 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -713,10 +713,15 @@ export function installSpaAuctionHook(): void { // can be called with the current URL, so guard every entry point against // re-requesting impressions for a path we already loaded. let currentPath = location.pathname; + // Last path whose slots/bids were actually applied — the initial SSR page + // counts. A failed navigation rolls `currentPath` back to this rather than to + // the immediately-previous committed value: on rapid A→B where A was aborted + // mid-flight and B then fails, rolling back to A (never loaded) would strand + // it behind the no-op guard, so we roll back to the last applied route instead. + let lastAppliedPath = location.pathname; async function onNavigate(path: string): Promise { if (path === currentPath) return; - const previousPath = currentPath; currentPath = path; inflight?.abort(); const controller = new AbortController(); @@ -737,7 +742,7 @@ export function installSpaAuctionHook(): void { // committed path back so a later navigation here retries instead of // being skipped by the no-op guard at the top. Only roll back when no // newer navigation has already advanced currentPath. - if (inflight === controller) currentPath = previousPath; + if (inflight === controller) currentPath = lastAppliedPath; return; } const data = (await res.json()) as PageBidsResponse; @@ -748,6 +753,9 @@ export function installSpaAuctionHook(): void { if (inflight !== controller) return; ts.adSlots = data.slots; ts.bids = data.bids; + // This route is now the committed, loaded state — a later failed + // navigation rolls back here, and a return trip no-ops correctly. + lastAppliedPath = path; // An empty page-bids response (auction kill switch or consent gate) carries // no TS slots. Only run adInit() when there are slots to apply or prior TS // state to sweep — otherwise a consent-denied or kill-switched navigation @@ -761,7 +769,7 @@ export function installSpaAuctionHook(): void { } } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; - if (inflight === controller) currentPath = previousPath; + if (inflight === controller) currentPath = lastAppliedPath; log.warn('SPA auction hook: fetch failed', err); } } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index b82542695..4a6368768 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -986,6 +986,59 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('fetches PBS Cache once when two same-adId messages race before the fetch resolves', async () => { + // Concurrent render double-fire guard: two 'Prebid Request' messages for the + // same adId can arrive before the first cache fetch settles. The in-flight + // `renderingAdIds` gate must collapse them to a single fetch — the persistent + // firedBeacons dedup only engages after a fetch resolves, so it cannot stop + // the second fetch on its own. Deferring the fetch keeps both messages in the + // window where only the in-flight gate can prevent the duplicate. + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const mockAd = '
Test Creative
'; + let resolveFetch: (value: Response) => void = () => {}; + fetchStub.mockReturnValue( + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + + const bridgeListener = await captureBridgeListener(); + + const stopSpy = vi.fn(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + + const dispatch = (): unknown => + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [fakePort], + source, + stopImmediatePropagation: stopSpy, + }) as unknown as MessageEvent + ); + + // Both messages dispatched before the deferred fetch resolves. + dispatch(); + dispatch(); + + // The second message hit the in-flight gate — only one fetch launched. + expect(fetchStub).toHaveBeenCalledTimes(1); + + // Resolve the single fetch and flush its .then chain. + resolveFetch({ ok: true, text: () => Promise.resolve(mockAd) } as Response); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(fetchStub).toHaveBeenCalledTimes(1); + expect(portMessages).toHaveLength(1); + // A single render still fires both win and billing beacons exactly once. + expect(beaconSpy).toHaveBeenCalledWith('https://ssp.example/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://ssp.example/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + it('responds with adm without fetching PBS Cache when debug adm is available', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const debugAdm = '
Debug Creative
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 2ef3a1746..9a08defcb 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -353,6 +353,59 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('does not strand a path that was aborted mid-flight then failed on the next nav', async () => { + // Rapid A→B where A is aborted mid-flight and B then fails must roll + // `currentPath` back to the last *applied* path (here the initial route), + // not to A. Rolling back to A — which never loaded — would leave it behind + // the no-op guard so a later real navigation to A never re-fetches. + document.body.innerHTML = '
'; + let resolveA: ((value: unknown) => void) | undefined; + fetchStub + // A: still in flight when B starts (aborted, never settles on its own). + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveA = resolve; + }) + ) + // B: fails. + .mockResolvedValueOnce({ ok: false, status: 500 }) + // A retried: succeeds. + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + slots: [{ id: 'a', div_id: 'div-a' }], + bids: { a: { hb_pb: '1.00' } }, + }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + // A starts (left in flight), then B aborts A and fails. + history.pushState({}, '', '/a'); + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.adSlots).toBeUndefined(); + + // Navigate back to /a. With the rollback keyed to the last applied path + // (the initial route) instead of B's previous path (/a), this is NOT + // swallowed by the no-op guard and re-fetches. + history.pushState({}, '', '/a'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledTimes(3); + expect(ts.adSlots).toEqual([{ id: 'a', div_id: 'div-a' }]); + expect(adInit).toHaveBeenCalledTimes(1); + + // The original aborted A fetch resolving late must not clobber the retry. + resolveA?.({ ok: true, json: async () => ({ slots: [{ id: 'stale' }], bids: {} }) }); + await flushAsync(); + expect(ts.adSlots).toEqual([{ id: 'a', div_id: 'div-a' }]); + }); + it('is idempotent — repeated install calls do not double-fetch a navigation', async () => { fetchStub.mockResolvedValue({ ok: true, From f444a15d03ba670d7bc24511a84858562080ebfd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 7 Jul 2026 21:57:09 +0530 Subject: [PATCH 136/195] Harden ad-template audit against hostile page data - Escape page-controlled slot fields and validate the gampad network id before splicing scraped values into trusted-server.toml - Add navigation and teardown timeouts to the verify browser collector - Snapshot evidence before the scroll pass so load-time entries keep phase initial_load; add a Chrome-gated regression fixture - Cap collector evidence lists in the injected script and after decode - Preserve CRLF line endings and render non-finite floor_price as valid TOML when updating configs in place - Cover all 128 gate combinations in the core ad-stack mirror test - Extract slot TOML rendering/merging/splicing into slot_toml.rs --- .../commands/audit/ad_template_collector.js | 27 +- .../src/commands/audit/browser.rs | 91 ++- .../src/commands/audit/generate/gpt_slots.rs | 23 +- .../src/commands/audit/generate/mod.rs | 714 +--------------- .../src/commands/audit/generate/slot_toml.rs | 762 ++++++++++++++++++ .../src/creative_opportunities.rs | 4 +- 6 files changed, 920 insertions(+), 701 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index bc83772d8..c01074376 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -18,16 +18,24 @@ const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence | const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 1024 +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) list.push(entry) +} + function __ts_normalize_sizes(sizes) { const out = [] if (!Array.isArray(sizes)) return out // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. const pairs = typeof sizes[0] === "number" ? [sizes] : sizes for (const size of pairs) { + if (out.length >= __ts_max_entries) break if (Array.isArray(size) && typeof size[0] === "number" && typeof size[1] === "number") { out.push([size[0], size[1]]) } else { - __ts_ev.warnings.push({ + __ts_push(__ts_ev.warnings, { code: "fluid_size_ignored", message: "non-numeric GPT size ignored", }) @@ -37,7 +45,7 @@ function __ts_normalize_sizes(sizes) { } function __ts_record_define_slot(adUnitPath, sizes, divId) { - __ts_ev.gpt_slots.push({ + __ts_push(__ts_ev.gpt_slots, { gam_unit_path: String(adUnitPath), div_id: String(divId), sizes: __ts_normalize_sizes(sizes), @@ -63,7 +71,7 @@ function __ts_wrap_googletag(googletag) { try { __ts_record_define_slot(adUnitPath, sizes, divId) } catch (error) { - __ts_ev.warnings.push({ code: "define_slot_capture_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "define_slot_capture_failed", message: String(error) }) } return slot } @@ -80,14 +88,14 @@ function __ts_wrap_apstag(apstag) { try { const slots = (config && config.slots) || [] for (const slot of slots) { - __ts_ev.aps_calls.push({ + __ts_push(__ts_ev.aps_calls, { slot_id: String(slot.slotID || slot.slotName || ""), sizes: __ts_normalize_sizes(slot.sizes), phase: __ts_phase(), }) } } catch (error) { - __ts_ev.warnings.push({ code: "aps_capture_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "aps_capture_failed", message: String(error) }) } return originalFetchBids.apply(this, arguments) } @@ -124,7 +132,7 @@ window.__tsCollectAdTemplateEvidence = function () { const id = element.id if (id.endsWith("-container")) continue if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { - __ts_ev.dom_ids.push({ dom_id: id, phase: __ts_phase() }) + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) seen.add(id) } } @@ -139,6 +147,7 @@ window.__tsCollectAdTemplateEvidence = function () { const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] const sizes = [] for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break if (size && typeof size.getWidth === "function") { sizes.push([size.getWidth(), size.getHeight()]) } else if (Array.isArray(size) && typeof size[0] === "number") { @@ -149,7 +158,7 @@ window.__tsCollectAdTemplateEvidence = function () { (entry) => entry.gam_unit_path === String(path) && entry.div_id === String(divId) ) if (!exists) { - __ts_ev.gpt_slots.push({ + __ts_push(__ts_ev.gpt_slots, { gam_unit_path: String(path), div_id: String(divId), sizes, @@ -157,12 +166,12 @@ window.__tsCollectAdTemplateEvidence = function () { }) } } catch (error) { - __ts_ev.warnings.push({ code: "gpt_scrape_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "gpt_scrape_failed", message: String(error) }) } } } } catch (error) { - __ts_ev.warnings.push({ code: "collect_failed", message: String(error) }) + __ts_push(__ts_ev.warnings, { code: "collect_failed", message: String(error) }) } return __ts_ev } diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index e302b74d2..9f6aca1e6 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -31,6 +31,13 @@ const CHROME_NAMES: &[&str] = &[ /// Poll interval while waiting for the page network to settle, in milliseconds. const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Hard cap per decoded evidence list, mirroring the collector script's +/// `__ts_max_entries`, so a hostile page cannot inflate CLI memory. +const MAX_EVIDENCE_ENTRIES: usize = 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); /// Default quiet window (no new resources) marking the page settled. const DEFAULT_SETTLE_QUIET_MS: u64 = 750; /// Default hard cap on settling so slow/ad-heavy pages still terminate. @@ -228,9 +235,10 @@ async fn collect( let result = collect_with_browser(&browser, request, settle_config).await; - // Best-effort teardown; ignore errors since we already have a result. - let _ = browser.close().await; - let _ = browser.wait().await; + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; handler_task.abort(); result @@ -267,16 +275,29 @@ async fn collect_with_browser( .map_err(|error| format!("failed to set cookie `{name}`: {error}"))?; } - page.goto(request.url.as_str()) + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) .await + .map_err(|_| format!("navigation to {} timed out", request.url))? .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; - page.wait_for_navigation() + tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()) .await + .map_err(|_| format!("navigation to {} timed out", request.url))? .map_err(|error| format!("failed to read main document navigation response: {error}"))?; settle(&page, settle_config).await; if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + let _ = page + .evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ) + .await; + } scroll_page(&page).await; settle(&page, settle_config).await; } @@ -386,7 +407,15 @@ async fn extract_ad_evidence( None } Some(value) => match serde_json::from_value::(value) { - Ok(evidence) => Some(evidence), + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } Err(error) => { warnings.push(Warning { code: "ad_evidence_decode_failed".to_string(), @@ -505,4 +534,54 @@ mod tests { "should capture the configured-prefix DOM id" ); } + + #[test] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !chrome_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + aps_slot_ids: Vec::new(), + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index 14172ff0c..b7d595dbc 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -210,7 +210,13 @@ fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { let iu_parts = iu_parts?; let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); - let network_id = parts.next()?.to_string(); + // Mirror the registry path's validation: a GAM network id is digits only. + // The percent-decoded query value is page-controlled and gets spliced into + // generated TOML, so reject anything else. + let network_id = parts + .next() + .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? + .to_string(); let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); // A usable unit path needs the network id plus at least one path segment. parts.next()?; @@ -381,6 +387,21 @@ mod tests { ); } + #[test] + fn skips_requests_with_non_numeric_network_id() { + // A page-controlled iu_parts value must not smuggle a non-numeric + // network id (it gets spliced into generated TOML). + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%22evil%2Cslot&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a non-numeric network id should be rejected" + ); + assert_eq!(discovered.gam_network_id, None); + } + #[test] fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { let discovered = from_requests(&[request( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index bf1262697..a6f2546e6 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -2,20 +2,22 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; mod gpt_slots; +mod slot_toml; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use serde::Serialize; -use trusted_server_core::auction::types::MediaType; -use trusted_server_core::creative_opportunities::{ - CreativeOpportunitiesConfig, CreativeOpportunitySlot, -}; +use trusted_server_core::creative_opportunities::CreativeOpportunitiesConfig; use url::Url; use crate::commands::audit::generate::collector::AuditCollector; +use crate::commands::audit::generate::slot_toml::{ + merge_slots, render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, + toml_string, +}; use crate::commands::config::init::EXAMPLE_CONFIG; use crate::error::{CliResult, cli_error, report_error}; @@ -386,7 +388,7 @@ fn build_draft_config( &draft, "creative_opportunities", "gam_network_id", - &format!("gam_network_id = \"{network_id}\""), + &format!("gam_network_id = {}", toml_string(network_id)), )?; } draft.push_str(&render_discovered_slots(target_url, slots)); @@ -414,14 +416,15 @@ fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) .join(", "); out.push_str(&format!( "\n[[creative_opportunities.slot]]\n\ - id = \"{id}\"\n\ - div_id = \"{div_id}\"\n\ - gam_unit_path = \"{gam_unit_path}\"\n\ - page_patterns = [\"{page_pattern}\"]\n\ + id = {id}\n\ + div_id = {div_id}\n\ + gam_unit_path = {gam_unit_path}\n\ + page_patterns = [{page_pattern}]\n\ formats = [{formats}]\n", - id = slot.id, - div_id = slot.div_id, - gam_unit_path = slot.gam_unit_path, + id = toml_string(&slot.id), + div_id = toml_string(&slot.div_id), + gam_unit_path = toml_string(&slot.gam_unit_path), + page_pattern = toml_string(page_pattern), )); if slot.has_prebid { out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); @@ -511,29 +514,6 @@ pub(crate) fn run_update_slots( ) .map_err(|error| report_error(format!("failed to write command output: {error}"))) } - -/// Chooses the `gam_network_id` to write. -/// -/// The existing id is kept only when a real merge preserves existing slots. -/// On `--replace`, or when the config had no slots (e.g. a placeholder -/// `[creative_opportunities]` section), the discovered id wins — mirroring -/// [`merge_slots`], which returns discovered-only in those cases. -fn resolve_network_id( - existing: Option<&CreativeOpportunitiesConfig>, - discovered_network_id: Option<&str>, - replace: bool, -) -> Option { - let existing_network_id = existing.map(|config| config.gam_network_id.clone()); - let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); - if preserving_existing { - existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) - } else { - discovered_network_id - .map(str::to_string) - .or(existing_network_id) - } -} - /// The default page pattern for a scraped URL: its path, or `/` for the root. fn default_page_pattern(target_url: &Url) -> String { let path = target_url.path(); @@ -544,414 +524,6 @@ fn default_page_pattern(target_url: &Url) -> String { } } -/// A slot ready to render — the union of discovered and existing fields, without -/// the core type's `pub(crate)` compiled-pattern cache. -#[derive(Debug, Clone)] -struct RenderSlot { - id: String, - div_id: Option, - gam_unit_path: Option, - page_patterns: Vec, - /// `(width, height, non-banner media type)`. - formats: Vec<(u32, u32, Option<&'static str>)>, - floor_price: Option, - targeting: BTreeMap, - aps_slot_id: Option, - /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). - prebid_bidders: Option>, -} - -impl RenderSlot { - /// The stable identity used to match slots across runs: the div id (or slot - /// id), with any trailing `-` trimmed so hand-authored stems still match. - fn key(&self) -> String { - self.div_id - .as_deref() - .unwrap_or(&self.id) - .trim_end_matches('-') - .to_string() - } - - fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { - Self { - id: slot.id.clone(), - div_id: Some(slot.div_id.clone()), - gam_unit_path: Some(slot.gam_unit_path.clone()), - page_patterns: patterns.to_vec(), - formats: slot - .formats - .iter() - .map(|&(width, height)| (width, height, None)) - .collect(), - floor_price: None, - targeting: BTreeMap::new(), - aps_slot_id: None, - prebid_bidders: slot.has_prebid.then(BTreeMap::new), - } - } - - fn from_existing(slot: &CreativeOpportunitySlot) -> Self { - Self { - id: slot.id.clone(), - div_id: slot.div_id.clone(), - gam_unit_path: slot.gam_unit_path.clone(), - page_patterns: slot.page_patterns.clone(), - formats: slot - .formats - .iter() - .map(|format| { - ( - format.width, - format.height, - media_type_label(&format.media_type), - ) - }) - .collect(), - floor_price: slot.floor_price, - targeting: slot - .targeting - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect(), - aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), - prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { - prebid - .bidders - .iter() - .map(|(name, params)| (name.clone(), params.clone())) - .collect() - }), - } - } -} - -/// The non-default (non-banner) media-type label to emit, or `None` for banner. -fn media_type_label(media_type: &MediaType) -> Option<&'static str> { - match media_type { - MediaType::Banner => None, - MediaType::Video => Some("video"), - MediaType::Native => Some("native"), - } -} - -/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. -/// -/// - `--replace` (or no existing slots): the result is exactly the discovered set. -/// - Otherwise existing slots are preserved (covering other pages / hand-tuned -/// fields); a slot re-seen this run has `run_patterns` unioned into its -/// `page_patterns`; slots seen only this run are appended. -fn merge_slots( - existing: Option<&CreativeOpportunitiesConfig>, - discovered: &gpt_slots::DiscoveredSlots, - run_patterns: &[String], - replace: bool, -) -> Vec { - let discovered_slots: Vec = discovered - .slots - .iter() - .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) - .collect(); - - let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); - if replace || existing_slots.is_empty() { - return discovered_slots; - } - - let mut merged: Vec = existing_slots - .iter() - .map(RenderSlot::from_existing) - .collect(); - for slot in discovered_slots { - let key = slot.key(); - if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { - for pattern in &slot.page_patterns { - if !present.page_patterns.contains(pattern) { - present.page_patterns.push(pattern.clone()); - } - } - } else { - merged.push(slot); - } - } - merged -} - -/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. -fn render_slots(slots: &[RenderSlot]) -> String { - let mut out = String::from( - "\n# Slots managed by `ts audit ad-templates generate`.\n\ - # Review page_patterns and formats before validating/pushing.\n", - ); - for slot in slots { - out.push_str("\n[[creative_opportunities.slot]]\n"); - out.push_str(&format!("id = {}\n", toml_string(&slot.id))); - if let Some(div_id) = &slot.div_id { - out.push_str(&format!("div_id = {}\n", toml_string(div_id))); - } - if let Some(path) = &slot.gam_unit_path { - out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); - } - let patterns = slot - .page_patterns - .iter() - .map(|pattern| toml_string(pattern)) - .collect::>() - .join(", "); - out.push_str(&format!("page_patterns = [{patterns}]\n")); - let formats = slot - .formats - .iter() - .map(|(width, height, media_type)| match media_type { - Some(kind) => { - format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") - } - None => format!("{{ width = {width}, height = {height} }}"), - }) - .collect::>() - .join(", "); - out.push_str(&format!("formats = [{formats}]\n")); - if let Some(floor) = slot.floor_price { - out.push_str(&format!("floor_price = {floor}\n")); - } - if !slot.targeting.is_empty() { - let pairs = slot - .targeting - .iter() - .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) - .collect::>() - .join(", "); - out.push_str(&format!("targeting = {{ {pairs} }}\n")); - } - if let Some(slot_id) = &slot.aps_slot_id { - out.push_str("[creative_opportunities.slot.providers.aps]\n"); - out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); - } - if let Some(bidders) = &slot.prebid_bidders { - out.push_str("[creative_opportunities.slot.providers.prebid]\n"); - let rendered = bidders - .iter() - .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) - .collect::>() - .join(", "); - if rendered.is_empty() { - out.push_str("bidders = {}\n"); - } else { - out.push_str(&format!("bidders = {{ {rendered} }}\n")); - } - } - } - out -} - -/// Quotes and escapes a string as a TOML basic string, including control chars. -fn toml_string(value: &str) -> String { - let mut out = String::with_capacity(value.len() + 2); - out.push('"'); - for ch in value.chars() { - match ch { - '"' => out.push_str("\\\""), - '\\' => out.push_str("\\\\"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - control if (control as u32) < 0x20 => { - out.push_str(&format!("\\u{:04X}", control as u32)); - } - other => out.push(other), - } - } - out.push('"'); - out -} - -/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. -fn toml_key(key: &str) -> String { - let is_bare = !key.is_empty() - && key - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); - if is_bare { - key.to_string() - } else { - toml_string(key) - } -} - -/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). -fn toml_inline_value(value: &serde_json::Value) -> String { - match value { - serde_json::Value::Null => "{}".to_string(), - serde_json::Value::Bool(bool) => bool.to_string(), - serde_json::Value::Number(number) => number.to_string(), - serde_json::Value::String(string) => toml_string(string), - serde_json::Value::Array(items) => { - let rendered = items - .iter() - .map(toml_inline_value) - .collect::>() - .join(", "); - format!("[{rendered}]") - } - serde_json::Value::Object(map) => { - let rendered = map - .iter() - .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) - .collect::>() - .join(", "); - format!("{{ {rendered} }}") - } - } -} - -/// Rewrites the `[creative_opportunities]` slot array of `existing` with the -/// pre-rendered `rendered_slots` text, updating `gam_network_id` and preserving -/// all other sections and comments. -/// -/// If the config has no `[creative_opportunities]` section, a fresh one is -/// appended so `generate` works against a config that omits it. -fn splice_creative_slots( - existing: &str, - network_id: Option<&str>, - rendered_slots: &str, -) -> CliResult { - let rendered = rendered_slots.trim_matches('\n'); - - // No section yet — append a fresh one with the network id and slots. - if !existing - .lines() - .any(|line| line.trim() == "[creative_opportunities]") - { - let mut result = existing.to_string(); - if !result.is_empty() && !result.ends_with('\n') { - result.push('\n'); - } - result.push_str("\n[creative_opportunities]\n"); - if let Some(network_id) = network_id { - result.push_str(&format!("gam_network_id = \"{network_id}\"\n")); - } - result.push_str(rendered); - result.push('\n'); - return Ok(result); - } - - // Section exists — update `gam_network_id` (best-effort) and replace slots. - let mut document = existing.to_string(); - if let Some(network_id) = network_id - && let Ok(updated) = replace_key_in_section( - &document, - "creative_opportunities", - "gam_network_id", - &format!("gam_network_id = \"{network_id}\""), - ) - { - document = updated; - } - - let lines: Vec<&str> = document.lines().collect(); - let header = lines - .iter() - .position(|line| line.trim() == "[creative_opportunities]") - .ok_or_else(|| { - report_error("target config has no [creative_opportunities] section to update") - })?; - - let is_slot_table = |line: &str| { - let trimmed = line.trim_start(); - trimmed.starts_with("[[creative_opportunities.slot]]") - || trimmed.starts_with("[creative_opportunities.slot.") - }; - let is_unrelated_table = |line: &str| { - let trimmed = line.trim_start(); - trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" - }; - - // Where the existing slot array begins (first slot table after the header), - // else the end of the scalar block (first unrelated table, or EOF). - let existing_start = lines[header + 1..] - .iter() - .position(|line| is_slot_table(line)) - .map(|offset| header + 1 + offset); - let start = existing_start.unwrap_or_else(|| { - lines[header + 1..] - .iter() - .position(|line| is_unrelated_table(line)) - .map_or(lines.len(), |offset| header + 1 + offset) - }); - // Where the slot array ends: first unrelated top-level table, or EOF. - let end = lines[start..] - .iter() - .position(|line| is_unrelated_table(line)) - .map_or(lines.len(), |offset| start + offset); - - let mut result = lines[..start].join("\n"); - if !result.is_empty() { - result.push('\n'); - } - result.push_str(rendered); - result.push('\n'); - let tail = lines[end..].join("\n"); - if !tail.is_empty() { - result.push('\n'); - result.push_str(&tail); - } - if existing.ends_with('\n') && !result.ends_with('\n') { - result.push('\n'); - } - Ok(result) -} - -fn replace_key_in_section( - document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - let section_header = format!("[{section}]"); - let mut in_section = false; - let mut replaced = false; - let mut saw_section = false; - let mut lines = Vec::new(); - - for line in document.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; - saw_section |= in_section; - } - - if in_section && !replaced && is_key_line(trimmed, key) { - lines.push(replacement_line.to_string()); - replaced = true; - } else { - lines.push(line.to_string()); - } - } - - if !saw_section { - return cli_error(format!( - "failed to update starter config because section `{section_header}` was not found" - )); - } - if !replaced { - return cli_error(format!( - "failed to update starter config because key `{key}` was not found in `{section_header}`" - )); - } - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); - } - Ok(output) -} - -fn is_key_line(trimmed_line: &str, key: &str) -> bool { - trimmed_line - .strip_prefix(key) - .and_then(|remaining| remaining.trim_start().strip_prefix('=')) - .is_some() -} - #[cfg(test)] mod tests { use std::cell::Cell; @@ -1310,185 +882,30 @@ mod tests { ); } - fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + #[test] + fn render_discovered_slots_escapes_page_controlled_strings() { + // Slot fields scraped from the live page must be escaped so a quote + // cannot inject TOML into the drafted config. let registry = vec![collector::CollectedGptSlot { - gam_unit_path: "/222/homepage/header".to_string(), - div_id: "div-gpt-ad-header".to_string(), + gam_unit_path: "/222/homepage/head\"er".to_string(), + div_id: "div-gpt-ad-head\"er".to_string(), sizes: vec![(728, 90)], }]; - gpt_slots::discover_gpt_slots(®istry, &[], false) - } - - /// Rendered slot text for the discovered header slot, patterns = `/`. - fn header_rendered() -> String { - let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); - render_slots(&merged) - } - - fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { - toml::from_str::(toml_str).expect("valid creative config") - } - - #[test] - fn splice_replaces_slots_and_preserves_other_sections() { - let existing = "[publisher]\ndomain = \"x\"\n\n\ - [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ - [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ - gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 300, height = 250 }]\n\n\ - [auction]\nenabled = true\n"; - - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) - .expect("should splice"); - - assert!( - out.contains("gam_network_id = \"222\""), - "network id updated" - ); - assert!(!out.contains("id = \"old\""), "old slot removed"); - assert!( - out.contains("gam_unit_path = \"/222/homepage/header\""), - "new slot written" - ); - assert!( - out.contains("[publisher]") && out.contains("domain = \"x\""), - "publisher section preserved" - ); - assert!( - out.contains("[auction]") && out.contains("enabled = true"), - "trailing auction section preserved" - ); - toml::from_str::(&out).expect("spliced config is valid TOML"); - } - - #[test] - fn splice_creates_section_when_absent() { - // Config with no [creative_opportunities] at all — generate should append it. - let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; - - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) - .expect("should splice"); - - let value = toml::from_str::(&out).expect("valid TOML"); - assert_eq!( - value["creative_opportunities"]["gam_network_id"].as_str(), - Some("222"), - "appended section carries the discovered network id" - ); - assert_eq!( - value["creative_opportunities"]["slot"][0]["id"].as_str(), - Some("header") - ); - assert!( - value["publisher"]["domain"].as_str() == Some("x") - && value["auction"]["enabled"].as_bool() == Some(true), - "existing sections preserved when appending" - ); - } - - #[test] - fn splice_inserts_when_no_existing_slots() { - let existing = - "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + let slots = gpt_slots::discover_gpt_slots(®istry, &[], false); + let url = Url::parse("https://publisher.example/").expect("should parse URL"); - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) - .expect("should splice"); + let rendered = render_discovered_slots(&url, &slots); - let value = toml::from_str::(&out).expect("valid TOML"); + let value = toml::from_str::(&rendered) + .expect("should render valid TOML despite embedded quotes"); + let slot = &value["creative_opportunities"]["slot"][0]; assert_eq!( - value["creative_opportunities"]["slot"][0]["id"].as_str(), - Some("header"), - "inserted slot id strips the div-gpt-ad- prefix" - ); - assert_eq!( - value["creative_opportunities"]["slot"][0]["div_id"].as_str(), - Some("div-gpt-ad-header"), - "div_id keeps the stable stem" - ); - assert!( - value["auction"]["enabled"].as_bool() == Some(true), - "auction section preserved after inserted slots" + slot["div_id"].as_str(), + Some("div-gpt-ad-head\"er"), + "should keep the quote as data, not TOML syntax" ); } - #[test] - fn merge_second_run_unions_page_patterns() { - // Existing slot on "/"; re-discovered this run with "/news/*". - let existing = existing_config( - "gam_network_id = \"222\"\n\n\ - [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ - gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 728, height = 90 }]\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/news/*".to_string()], - false, - ); - - assert_eq!(merged.len(), 1, "same slot is not duplicated"); - assert_eq!( - merged[0].page_patterns, - vec!["/".to_string(), "/news/*".to_string()], - "this run's pattern is unioned into the existing slot" - ); - } - - #[test] - fn merge_keeps_existing_only_slots() { - // Existing has header + sidebar; this run re-sees only header. - let existing = existing_config( - "gam_network_id = \"222\"\n\n\ - [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ - gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 728, height = 90 }]\n\n\ - [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ - gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ - formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/".to_string()], - false, - ); - - let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); - assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); - let sidebar = merged - .iter() - .find(|slot| slot.id == "sidebar") - .expect("sidebar"); - assert_eq!( - sidebar.floor_price, - Some(0.5), - "hand-tuned fields preserved" - ); - } - - #[test] - fn merge_replace_wipes_existing() { - let existing = existing_config( - "gam_network_id = \"222\"\n\n\ - [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ - gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 300, height = 250 }]\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/".to_string()], - true, - ); - - let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); - assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); - } - #[test] fn default_page_pattern_uses_path_or_root() { assert_eq!( @@ -1500,73 +917,4 @@ mod tests { "/" ); } - - #[test] - fn resolve_network_id_prefers_discovered_unless_preserving_existing() { - let with_slots = existing_config( - "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ - gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ - formats = [{ width = 300, height = 250 }]\n", - ); - let empty = existing_config("gam_network_id = \"111\"\n"); - - // Real merge → keep existing. - assert_eq!( - resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), - Some("111") - ); - // Placeholder section with no slots → discovered wins. - assert_eq!( - resolve_network_id(Some(&empty), Some("222"), false).as_deref(), - Some("222") - ); - // --replace → discovered wins. - assert_eq!( - resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), - Some("222") - ); - // No existing config → discovered. - assert_eq!( - resolve_network_id(None, Some("222"), false).as_deref(), - Some("222") - ); - } - - #[test] - fn toml_key_quotes_only_non_bare_keys() { - assert_eq!(toml_key("zone"), "zone"); - assert_eq!(toml_key("ad-loc"), "ad-loc"); - assert_eq!(toml_key("a.b"), "\"a.b\""); - assert_eq!(toml_key("with space"), "\"with space\""); - assert_eq!(toml_key(""), "\"\""); - } - - #[test] - fn toml_string_escapes_quotes_backslashes_and_controls() { - assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); - assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); - } - - #[test] - fn render_quotes_exotic_targeting_keys_to_valid_toml() { - let existing = existing_config( - "gam_network_id = \"1\"\n\n\ - [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ - page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ - targeting = { \"a.b\" = \"x\" }\n", - ); - - let merged = merge_slots( - Some(&existing), - &discovered_header_slot(), - &["/".to_string()], - false, - ); - let doc = format!( - "[creative_opportunities]\ngam_network_id = \"1\"\n{}", - render_slots(&merged) - ); - - toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); - } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs new file mode 100644 index 000000000..9c24ec9d6 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -0,0 +1,762 @@ +//! TOML-side slot config: the [`RenderSlot`] model, run merging, rendering, +//! and in-place `[creative_opportunities]` splicing for `ts audit ad-templates +//! generate`. + +use std::collections::BTreeMap; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; + +use crate::commands::audit::generate::gpt_slots; +use crate::error::{CliResult, cli_error, report_error}; + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +pub(super) struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable identity used to match slots across runs: the div id (or slot + /// id), with any trailing `-` trimmed so hand-authored stems still match. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has `run_patterns` unioned into its +/// `page_patterns`; slots seen only this run are appended. +pub(super) fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return discovered_slots; + } + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + for slot in discovered_slots { + let key = slot.key(); + if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + } else { + merged.push(slot); + } + } + merged +} + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +pub(super) fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = String::from( + "\n# Slots managed by `ts audit ad-templates generate`.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + let patterns = slot + .page_patterns + .iter() + .map(|pattern| toml_string(pattern)) + .collect::>() + .join(", "); + out.push_str(&format!("page_patterns = [{patterns}]\n")); + let formats = slot + .formats + .iter() + .map(|(width, height, media_type)| match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }) + .collect::>() + .join(", "); + out.push_str(&format!("formats = [{formats}]\n")); + if let Some(floor) = slot.floor_price { + // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); + // normalize non-finite values so the spliced config stays parseable. + if floor.is_finite() { + out.push_str(&format!("floor_price = {floor}\n")); + } else if floor.is_nan() { + out.push_str("floor_price = nan\n"); + } else if floor.is_sign_positive() { + out.push_str("floor_price = inf\n"); + } else { + out.push_str("floor_price = -inf\n"); + } + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +pub(super) fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + control if (control as u32) < 0x20 => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// Rewrites the `[creative_opportunities]` slot array of `existing` with the +/// pre-rendered `rendered_slots` text, updating `gam_network_id` and preserving +/// all other sections and comments. +/// +/// If the config has no `[creative_opportunities]` section, a fresh one is +/// appended so `generate` works against a config that omits it. +pub(super) fn splice_creative_slots( + existing: &str, + network_id: Option<&str>, + rendered_slots: &str, +) -> CliResult { + let rendered = rendered_slots.trim_matches('\n'); + + // No section yet — append a fresh one with the network id and slots. + if !existing + .lines() + .any(|line| line.trim() == "[creative_opportunities]") + { + let mut result = existing.to_string(); + if !result.is_empty() && !result.ends_with('\n') { + result.push('\n'); + } + result.push_str("\n[creative_opportunities]\n"); + if let Some(network_id) = network_id { + result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); + } + result.push_str(rendered); + result.push('\n'); + return Ok(result); + } + + // Section exists — update `gam_network_id` (best-effort) and replace slots. + let mut document = existing.to_string(); + if let Some(network_id) = network_id + && let Ok(updated) = replace_key_in_section( + &document, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = {}", toml_string(network_id)), + ) + { + document = updated; + } + + let lines: Vec<&str> = document.lines().collect(); + let header = lines + .iter() + .position(|line| line.trim() == "[creative_opportunities]") + .ok_or_else(|| { + report_error("target config has no [creative_opportunities] section to update") + })?; + + let is_slot_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with("[[creative_opportunities.slot]]") + || trimmed.starts_with("[creative_opportunities.slot.") + }; + let is_unrelated_table = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" + }; + + // Where the existing slot array begins (first slot table after the header), + // else the end of the scalar block (first unrelated table, or EOF). + let existing_start = lines[header + 1..] + .iter() + .position(|line| is_slot_table(line)) + .map(|offset| header + 1 + offset); + let start = existing_start.unwrap_or_else(|| { + lines[header + 1..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| header + 1 + offset) + }); + // Where the slot array ends: first unrelated top-level table, or EOF. + let end = lines[start..] + .iter() + .position(|line| is_unrelated_table(line)) + .map_or(lines.len(), |offset| start + offset); + + let mut result = lines[..start].join("\n"); + if !result.is_empty() { + result.push('\n'); + } + result.push_str(rendered); + result.push('\n'); + let tail = lines[end..].join("\n"); + if !tail.is_empty() { + result.push('\n'); + result.push_str(&tail); + } + if existing.ends_with('\n') && !result.ends_with('\n') { + result.push('\n'); + } + if uses_crlf(existing) { + result = result.replace('\n', "\r\n"); + } + Ok(result) +} + +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + document.contains("\r\n") +} + +pub(super) fn replace_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + let section_header = format!("[{section}]"); + let mut in_section = false; + let mut replaced = false; + let mut saw_section = false; + let mut lines = Vec::new(); + + for line in document.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('[') && trimmed.ends_with(']') { + in_section = trimmed == section_header; + saw_section |= in_section; + } + + if in_section && !replaced && is_key_line(trimmed, key) { + lines.push(replacement_line.to_string()); + replaced = true; + } else { + lines.push(line.to_string()); + } + } + + if !saw_section { + return cli_error(format!( + "failed to update starter config because section `{section_header}` was not found" + )); + } + if !replaced { + return cli_error(format!( + "failed to update starter config because key `{key}` was not found in `{section_header}`" + )); + } + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + // `lines()` stripped the `\r`s; restore the document's CRLF endings. + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + +fn is_key_line(trimmed_line: &str, key: &str) -> bool { + trimmed_line + .strip_prefix(key) + .and_then(|remaining| remaining.trim_start().strip_prefix('=')) + .is_some() +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// [`merge_slots`], which returns discovered-only in those cases. +pub(super) fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector; + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_preserves_crlf_line_endings() { + let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ + [auction]\r\nenabled = true\r\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "every line ending should stay CRLF" + ); + let value = toml::from_str::(&out).expect("spliced CRLF config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated in CRLF config" + ); + } + + #[test] + fn render_slots_writes_non_finite_floor_price_as_valid_toml() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string()], + formats: vec![(728, 90, None)], + floor_price: Some(f64::NAN), + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("floor_price = nan"), + "NaN should render as TOML `nan`, not Rust `NaN`" + ); + toml::from_str::(&rendered).expect("rendered slots are valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } +} diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 047261153..a85ab9a44 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -618,7 +618,7 @@ mod tests { // input combination, `expected == Yes` must equal the legacy all-AND boolean. #[test] fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { - for bits in 0u8..64 { + for bits in 0u8..128 { let input = AdStackGateInput { method_get: bits & 1 != 0, navigation: bits & 2 != 0, @@ -626,7 +626,7 @@ mod tests { bot: bits & 8 != 0, matched_slots: bits & 16 != 0, consent_allows_auction: Some(bits & 32 != 0), - auction_enabled: bits & 1 == 0, + auction_enabled: bits & 64 != 0, }; // Legacy semantics: all positive gates true, both negative gates false. let legacy = input.method_get From db302cbae4aec2ab544875a0c8aed9f2ea8e78db Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 8 Jul 2026 11:23:47 +0530 Subject: [PATCH 137/195] Address ad-template generate review findings - Recognize [creative_opportunities] table headers carrying inline comments in the in-place splice and replace_key_in_section, so a valid operator config is updated instead of gaining a duplicate section - Default generated page_patterns from the recorded post-redirect final URL instead of the requested URL, falling back to the requested URL when the recorded final URL is invalid - Escape DEL (U+007F) in toml_string, which TOML basic strings reject alongside chars below U+0020 Each fix carries a parse-backed regression test. --- .../src/commands/audit/generate/mod.rs | 51 ++++++++- .../src/commands/audit/generate/slot_toml.rs | 104 +++++++++++++++++- 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index a6f2546e6..5f58104cc 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -478,9 +478,13 @@ pub(crate) fn run_update_slots( } // Patterns for slots seen on this run: the `--page-pattern` values, or the - // audited path when none are given (preserving single-page behavior). + // audited path when none are given (preserving single-page behavior). The + // default uses the recorded post-redirect URL so it matches the page that + // was actually audited, falling back to the requested URL when the + // recorded final URL is invalid. let run_patterns: Vec = if page_patterns.is_empty() { - vec![default_page_pattern(&target_url)] + let audited_url = collected.final_url().unwrap_or_else(|_| target_url.clone()); + vec![default_page_pattern(&audited_url)] } else { page_patterns.to_vec() }; @@ -906,6 +910,49 @@ mod tests { ); } + #[test] + fn update_slots_defaults_pattern_to_final_url_after_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + // The requested URL redirects; slots are scraped from the final page. + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/news/story".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &[], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/news/story"), + "default pattern should use the post-redirect path, not the requested one" + ); + } + #[test] fn default_page_pattern_uses_path_or_root() { assert_eq!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 9c24ec9d6..0fc85fcc1 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -232,7 +232,8 @@ pub(super) fn toml_string(value: &str) -> String { '\n' => out.push_str("\\n"), '\r' => out.push_str("\\r"), '\t' => out.push_str("\\t"), - control if (control as u32) < 0x20 => { + // TOML basic strings reject U+0000..U+001F and DEL (U+007F). + control if (control as u32) < 0x20 || control == '\u{7f}' => { out.push_str(&format!("\\u{:04X}", control as u32)); } other => out.push(other), @@ -297,7 +298,7 @@ pub(super) fn splice_creative_slots( // No section yet — append a fresh one with the network id and slots. if !existing .lines() - .any(|line| line.trim() == "[creative_opportunities]") + .any(|line| is_table_header(line, "[creative_opportunities]")) { let mut result = existing.to_string(); if !result.is_empty() && !result.ends_with('\n') { @@ -328,7 +329,7 @@ pub(super) fn splice_creative_slots( let lines: Vec<&str> = document.lines().collect(); let header = lines .iter() - .position(|line| line.trim() == "[creative_opportunities]") + .position(|line| is_table_header(line, "[creative_opportunities]")) .ok_or_else(|| { report_error("target config has no [creative_opportunities] section to update") })?; @@ -340,7 +341,9 @@ pub(super) fn splice_creative_slots( }; let is_unrelated_table = |line: &str| { let trimmed = line.trim_start(); - trimmed.starts_with('[') && !is_slot_table(line) && trimmed != "[creative_opportunities]" + trimmed.starts_with('[') + && !is_slot_table(line) + && !is_table_header(line, "[creative_opportunities]") }; // Where the existing slot array begins (first slot table after the header), @@ -386,6 +389,25 @@ fn uses_crlf(document: &str) -> bool { document.contains("\r\n") } +/// Strips a trailing inline `# comment` from a candidate table-header line. +/// +/// Only valid on header candidates: header lines cannot contain `#` before the +/// closing bracket unless it is inside a quoted key, which the configs this +/// updater manages never use. +fn strip_inline_comment(line: &str) -> &str { + match line.find('#') { + Some(position) => line[..position].trim_end(), + None => line, + } +} + +/// Whether `line` is exactly the `section_header` table header (for example +/// `[creative_opportunities]`), tolerating surrounding whitespace and a +/// trailing inline `# comment` — both valid TOML. +fn is_table_header(line: &str, section_header: &str) -> bool { + strip_inline_comment(line.trim()) == section_header +} + pub(super) fn replace_key_in_section( document: &str, section: &str, @@ -400,8 +422,9 @@ pub(super) fn replace_key_in_section( for line in document.lines() { let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; + let header_candidate = strip_inline_comment(trimmed); + if header_candidate.starts_with('[') && header_candidate.ends_with(']') { + in_section = header_candidate == section_header; saw_section |= in_section; } @@ -588,6 +611,40 @@ mod tests { ); } + #[test] + fn splice_recognizes_inline_commented_section_header() { + // `[creative_opportunities] # comment` is valid TOML; the splice must + // update it in place instead of appending a duplicate section. + let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should splice"); + + assert_eq!( + out.lines() + .filter(|line| is_table_header(line, "[creative_opportunities]")) + .count(), + 1, + "commented header must not be duplicated" + ); + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated under a commented header" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "commented trailing section preserved" + ); + } + #[test] fn splice_inserts_when_no_existing_slots() { let existing = @@ -737,6 +794,41 @@ mod tests { assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); } + #[test] + fn toml_string_escapes_del_control_char() { + assert_eq!(toml_string("a\u{7f}b"), "\"a\\u007Fb\""); + let doc = format!("value = {}", toml_string("a\u{7f}b")); + let value = toml::from_str::(&doc).expect("DEL escapes to valid TOML"); + assert_eq!( + value["value"].as_str(), + Some("a\u{7f}b"), + "escaped DEL round-trips as data" + ); + } + + #[test] + fn replace_key_handles_inline_commented_headers() { + let document = "[creative_opportunities] # managed\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let updated = replace_key_in_section( + document, + "creative_opportunities", + "gam_network_id", + "gam_network_id = \"222\"", + ) + .expect("should find the commented section header"); + + assert!( + updated.contains("gam_network_id = \"222\""), + "key replaced under a commented header" + ); + assert!( + updated.contains("enabled = true"), + "later commented section left untouched" + ); + } + #[test] fn render_quotes_exotic_targeting_keys_to_valid_toml() { let existing = existing_config( From ccccfa7b7d7aa91812524d5a516e9cdde73375de Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 9 Jul 2026 16:04:42 +0530 Subject: [PATCH 138/195] Resolve ad-template CLI review findings --- Cargo.lock | 1 + README.md | 2 +- crates/trusted-server-cli/Cargo.toml | 1 + crates/trusted-server-cli/src/app_config.rs | 23 +- .../src/commands/audit/generate/gpt_slots.rs | 113 +++++++++- .../src/commands/audit/generate/mod.rs | 95 ++++++++ .../src/commands/audit/generate/slot_toml.rs | 205 +++++++++++++++++- .../src/commands/audit/mod.rs | 98 ++++++++- .../src/commands/audit/page.rs | 10 - crates/trusted-server-cli/src/run.rs | 32 ++- docs/guide/cli.md | 11 +- docs/guide/getting-started.md | 2 +- ...6-26-server-side-ad-template-cli-design.md | 16 +- 13 files changed, 563 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6036b24f8..a20139cb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5261,6 +5261,7 @@ dependencies = [ "scraper", "serde", "serde_json", + "temp-env", "tempfile", "time", "tokio", diff --git a/README.md b/README.md index e56937e89..405822061 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 276b4fd95..4a643d950 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -61,4 +61,5 @@ webpki-roots = { workspace = true } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index b84d16747..fadf58cde 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -49,6 +49,27 @@ pub struct LoadedSettings { /// explicit `--app-config` path is given and is missing, the error names that /// exact path rather than silently falling back. pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +pub fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { let manifest_loader = ManifestLoader::from_path(&args.manifest) .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { @@ -61,7 +82,7 @@ pub fn load_settings(args: &AppConfigArgs) -> Result { resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); let mut opts = AppConfigLoadOptions::default(); - opts.env_overlay = !args.no_env; + opts.env_overlay = env_overlay; let app_config = app_config::deserialize_app_config_with_options::( &app_config_path, &app_name, diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index b7d595dbc..fea34dd1f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -20,6 +20,7 @@ use std::collections::BTreeSet; use std::sync::LazyLock; use regex::Regex; +use trusted_server_core::creative_opportunities::validate_slot_id; use url::Url; use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; @@ -113,6 +114,7 @@ pub(crate) fn discover_gpt_slots( } slots.push(slot); } + make_slot_ids_unique(&mut slots); DiscoveredSlots { gam_network_id, @@ -274,12 +276,56 @@ fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { sizes } -/// Derives a slot id from a div id by stripping the common GPT prefix. +/// Derives a runtime-safe slot id from a div id. +/// +/// The common GPT prefix is stripped, invalid character runs become one +/// hyphen, and an all-invalid value falls back to `slot`. fn slot_id_from_div(div_id: &str) -> String { - div_id - .strip_prefix(GPT_DIV_PREFIX) - .unwrap_or(div_id) - .to_string() + let candidate = div_id.strip_prefix(GPT_DIV_PREFIX).unwrap_or(div_id); + let mut id = String::with_capacity(candidate.len()); + let mut previous_was_hyphen = false; + for character in candidate.chars() { + if character.is_ascii_alphanumeric() || character == '_' { + id.push(character); + previous_was_hyphen = false; + } else if !id.is_empty() && !previous_was_hyphen { + id.push('-'); + previous_was_hyphen = true; + } + } + while id.ends_with('-') { + id.pop(); + } + if id.is_empty() { + id.push_str("slot"); + } + + if validate_slot_id(&id).is_ok() { + id + } else { + "slot".to_string() + } +} + +/// Adds deterministic numeric suffixes when sanitization produces duplicate ids. +fn make_slot_ids_unique(slots: &mut [DiscoveredSlot]) { + let mut used = BTreeSet::new(); + for slot in slots { + if used.insert(slot.id.clone()) { + continue; + } + + let base = slot.id.clone(); + let mut suffix = 2_usize; + loop { + let candidate = format!("{base}-{suffix}"); + if used.insert(candidate.clone()) { + slot.id = candidate; + break; + } + suffix += 1; + } + } } /// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. @@ -549,6 +595,63 @@ mod tests { ); } + #[test] + fn sanitizes_page_controlled_div_ids_for_runtime_slot_ids() { + let registry = vec![registry_slot( + "/123456789/homepage/header", + "div-gpt-ad-header.main: 1", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "header-main-1"); + assert_eq!( + discovered.slots[0].div_id, "div-gpt-ad-header.main: 1", + "matching should retain the original normalized div stem" + ); + trusted_server_core::creative_opportunities::validate_slot_id(&discovered.slots[0].id) + .expect("generated id should pass runtime validation"); + } + + #[test] + fn uses_fallback_for_div_id_without_safe_slot_id_characters() { + let registry = vec![registry_slot( + "/123456789/homepage/fallback", + "div-gpt-ad-...", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "slot"); + } + + #[test] + fn makes_colliding_sanitized_slot_ids_unique() { + let registry = vec![ + registry_slot( + "/123456789/homepage/dotted", + "div-gpt-ad-header.main", + &[(728, 90)], + ), + registry_slot( + "/123456789/homepage/colon", + "div-gpt-ad-header:main", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + let ids = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + #[test] fn normalizes_react_and_hex_hashes_to_stable_prefixes() { assert_eq!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 5f58104cc..43a8e7f8f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -535,9 +535,11 @@ mod tests { use tempfile::TempDir; use super::*; + use crate::app_config::AppConfigArgs; use crate::commands::audit::generate::collector::{ CollectedPage, CollectedRequest, CollectedScriptTag, }; + use crate::commands::config::init::EXAMPLE_CONFIG; struct FakeCollector { collected: CollectedPage, @@ -953,6 +955,99 @@ mod tests { ); } + #[test] + fn update_slots_dry_run_does_not_persist_environment_overlay_config() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let config = EXAMPLE_CONFIG + .replace( + "replace-with-admin-password-32-bytes", + "test-admin-password-32-bytes-minimum", + ) + .replace( + "trusted-server-placeholder-secret", + "test-ec-passphrase-32-bytes-minimum", + ) + .replace( + "change-me-proxy-secret", + "test-proxy-secret-32-bytes-minimum", + ); + let config = format!( + "{config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"file-only\"\n\ + div_id = \"div-gpt-ad-file\"\n\ + gam_unit_path = \"/123456789/homepage/file\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n" + ); + fs::write(&config_path, config).expect("should write config"); + let args = AppConfigArgs { + app_config: Some(config_path.clone()), + manifest: manifest_path, + no_env: false, + }; + + temp_env::with_var( + "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__GAM_NETWORK_ID", + Some("987654321"), + || { + let effective = crate::app_config::load_settings(&args) + .expect("should load effective settings"); + assert_eq!( + effective + .settings + .creative_opportunities + .as_ref() + .expect("should have creative config") + .gam_network_id, + "987654321", + "test environment should override the network id" + ); + let loaded = crate::app_config::load_file_settings(&args) + .expect("should load file-only settings"); + let mut collected = collected_page(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/123456789/homepage/file".to_string(), + div_id: "div-gpt-ad-file".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &loaded.app_config_path, + loaded.settings.creative_opportunities.as_ref(), + &[], + false, + &[], + true, + &collector, + &mut out, + ) + .expect("should render dry-run update"); + + let output = String::from_utf8(out).expect("output should be UTF-8"); + assert!( + output.contains("id = \"file-only\""), + "dry run should preserve the file-backed slot" + ); + assert!( + output.contains("gam_network_id = \"123456789\""), + "dry run should preserve the file-backed network id" + ); + assert!( + !output.contains("987654321"), + "dry run must not persist environment-only config" + ); + }, + ); + } + #[test] fn default_page_pattern_uses_path_or_root() { assert_eq!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 0fc85fcc1..b15e6ed9f 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; +use toml_edit::{DocumentMut, Item}; use trusted_server_core::auction::types::MediaType; use trusted_server_core::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunitySlot, @@ -30,8 +31,8 @@ pub(super) struct RenderSlot { } impl RenderSlot { - /// The stable identity used to match slots across runs: the div id (or slot - /// id), with any trailing `-` trimmed so hand-authored stems still match. + /// The stable exact identity fallback used when no configured div prefix + /// matches a discovered slot. fn key(&self) -> String { self.div_id .as_deref() @@ -129,21 +130,64 @@ pub(super) fn merge_slots( .iter() .map(RenderSlot::from_existing) .collect(); - for slot in discovered_slots { - let key = slot.key(); - if let Some(present) = merged.iter_mut().find(|existing| existing.key() == key) { + for mut slot in discovered_slots { + if let Some(index) = matching_slot_index(&merged, &slot) { + let present = &mut merged[index]; for pattern in &slot.page_patterns { if !present.page_patterns.contains(pattern) { present.page_patterns.push(pattern.clone()); } } } else { + slot.id = unique_slot_id(&slot.id, &merged); merged.push(slot); } } merged } +fn unique_slot_id(candidate: &str, existing: &[RenderSlot]) -> String { + if existing.iter().all(|slot| slot.id != candidate) { + return candidate.to_string(); + } + + let mut suffix = 2_usize; + loop { + let unique = format!("{candidate}-{suffix}"); + if existing.iter().all(|slot| slot.id != unique) { + return unique; + } + suffix += 1; + } +} + +/// Finds the most specific configured slot matching a discovered live div. +/// +/// Configured `div_id` values are runtime prefixes. Exact matches naturally +/// win because they are the longest possible prefix; equal-length ties retain +/// config order. The prior exact key behavior remains as a fallback. +fn matching_slot_index(existing: &[RenderSlot], discovered: &RenderSlot) -> Option { + if let Some(discovered_div) = discovered.div_id.as_deref() { + let mut best = None; + let mut best_length = 0; + for (index, slot) in existing.iter().enumerate() { + let Some(prefix) = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty()) else { + continue; + }; + if discovered_div.starts_with(prefix) && prefix.len() > best_length { + best = Some(index); + best_length = prefix.len(); + } + } + if best.is_some() { + return best; + } + } + + let key = discovered.key(); + existing.iter().position(|slot| slot.key() == key) +} + /// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. pub(super) fn render_slots(slots: &[RenderSlot]) -> String { let mut out = String::from( @@ -294,13 +338,14 @@ pub(super) fn splice_creative_slots( rendered_slots: &str, ) -> CliResult { let rendered = rendered_slots.trim_matches('\n'); + let existing = remove_inline_slot_value(existing)?; // No section yet — append a fresh one with the network id and slots. if !existing .lines() .any(|line| is_table_header(line, "[creative_opportunities]")) { - let mut result = existing.to_string(); + let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); } @@ -314,7 +359,7 @@ pub(super) fn splice_creative_slots( } // Section exists — update `gam_network_id` (best-effort) and replace slots. - let mut document = existing.to_string(); + let mut document = existing.clone(); if let Some(network_id) = network_id && let Ok(updated) = replace_key_in_section( &document, @@ -378,12 +423,37 @@ pub(super) fn splice_creative_slots( if existing.ends_with('\n') && !result.ends_with('\n') { result.push('\n'); } - if uses_crlf(existing) { + if uses_crlf(&existing) { result = result.replace('\n', "\r\n"); } Ok(result) } +/// Removes a scalar `creative_opportunities.slot` value so it can be replaced +/// with the generated array-of-tables representation. +fn remove_inline_slot_value(document: &str) -> CliResult { + let mut parsed = document.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + let Some(creative) = parsed.get_mut("creative_opportunities") else { + return Ok(document.to_string()); + }; + let Some(table) = creative.as_table_like_mut() else { + return Ok(document.to_string()); + }; + let has_inline_slot = table + .get("slot") + .is_some_and(|slot| matches!(slot, Item::Value(_))); + if !has_inline_slot { + return Ok(document.to_string()); + } + + table.remove("slot"); + Ok(parsed.to_string()) +} + /// Whether `document` uses CRLF line endings (so edits preserve them). fn uses_crlf(document: &str) -> bool { document.contains("\r\n") @@ -670,6 +740,46 @@ mod tests { ); } + #[test] + fn splice_replaces_inline_slot_array() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should replace inline slot array"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "unrelated tables should be preserved" + ); + } + + #[test] + fn splice_replaces_inline_slot_map() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should replace inline slot map"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + } + #[test] fn merge_second_run_unions_page_patterns() { // Existing slot on "/"; re-discovered this run with "/news/*". @@ -695,6 +805,85 @@ mod tests { ); } + #[test] + fn merge_uses_longest_existing_div_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/broad/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"atf\"\ndiv_id = \"ad-atf-\"\n\ + gam_unit_path = \"/222/atf\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 2, + "prefix match should not append a duplicate" + ); + let broad = merged + .iter() + .find(|slot| slot.id == "broad") + .expect("should keep broad slot"); + assert_eq!( + broad.page_patterns, + ["/broad/*"], + "shorter prefix should not claim the discovered div" + ); + let atf = merged + .iter() + .find(|slot| slot.id == "atf") + .expect("should keep specific slot"); + assert_eq!( + atf.page_patterns, + ["/", "/news/*"], + "longest matching prefix should receive this run's pattern" + ); + } + + #[test] + fn merge_renames_new_slot_id_that_collides_with_existing_config() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header-main\"\ndiv_id = \"legacy-header\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "div-gpt-ad-header.main".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + let ids = merged + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + #[test] fn merge_keeps_existing_only_slots() { // Existing has header + sidebar; this run re-sees only header. diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index b7ed91f1e..1a33b890a 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -3,7 +3,7 @@ //! `ts audit page ` is the generic page audit; `ts audit ad-templates verify //! ...` is the ad-template verifier; `ts audit generate ` bootstraps a //! draft config from a live page (issue #800). `ts audit ` is a hidden -//! compatibility alias for `ts audit page `. +//! compatibility alias for `ts audit generate `. pub mod ad_templates; pub mod browser; @@ -55,9 +55,40 @@ pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { pub(crate) struct AuditArgs { #[command(subcommand)] pub(crate) command: Option, - /// Hidden compatibility alias: `ts audit ` behaves like `ts audit page `. + /// Hidden compatibility alias: `ts audit ` behaves like `ts audit generate `. #[arg(value_parser = parse_http_url, hide = true)] pub(crate) legacy_url: Option, + #[command(flatten)] + pub(crate) legacy_generate: LegacyGenerateArgs, +} + +/// Hidden generation flags retained for the legacy `ts audit ` form. +#[derive(Debug, Default, Args)] +pub(crate) struct LegacyGenerateArgs { + /// JavaScript asset audit output path. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) js_assets: Option, + /// Draft Trusted Server config output path. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) config: Option, + /// Do not write the JavaScript asset audit file. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_js_assets: bool, + /// Do not write the draft Trusted Server config file. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_config: bool, + /// Overwrite existing output files. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + #[arg( + long = "cookie", + value_name = "NAME=VALUE", + value_parser = parse_cookie, + hide = true, + requires = "legacy_url" + )] + pub(crate) cookies: Vec<(String, String)>, } /// `ts audit` subcommands. @@ -136,8 +167,8 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// Dispatches a `ts audit` invocation. /// -/// `legacy_url` (if present) and the `page` subcommand both route to the generic -/// page audit; `ad-templates verify` routes to the verifier. +/// `legacy_url` (if present) routes to artifact generation, while the `page` +/// subcommand routes to the generic read-only page audit. /// /// # Errors /// @@ -147,7 +178,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { match &args.command { Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { - let loaded = crate::app_config::load_settings(&gen_args.config)?; + let loaded = crate::app_config::load_file_settings(&gen_args.config)?; let collector = generate::browser_collector::BrowserAuditCollector; let stdout = std::io::stdout(); let mut out = stdout.lock(); @@ -173,12 +204,32 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { generate::run_generate(generate_args, &collector, &mut out) } None => match &args.legacy_url { - Some(url) => page::run_page_url(url, false), + Some(_) => { + let generate_args = legacy_generate_args(args) + .expect("should build generation args when legacy URL is present"); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector; + generate::run_generate(&generate_args, &collector, &mut out) + } None => Err("provide a URL or a subcommand (`page`, `ad-templates`)".to_string()), }, } } +fn legacy_generate_args(args: &AuditArgs) -> Option { + let url = args.legacy_url.as_ref()?; + Some(generate::GenerateArgs { + url: url.to_string(), + js_assets: args.legacy_generate.js_assets.clone(), + config: args.legacy_generate.config.clone(), + no_js_assets: args.legacy_generate.no_js_assets, + no_config: args.legacy_generate.no_config, + force: args.legacy_generate.force, + cookies: args.legacy_generate.cookies.clone(), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -214,4 +265,39 @@ mod tests { let err = parse_cookie("=value").expect_err("should reject empty name"); assert!(err.contains("empty name"), "error should name the problem"); } + + #[test] + fn legacy_url_builds_artifact_generation_args() { + let args = AuditArgs { + command: None, + legacy_url: Some( + url::Url::parse("https://www.example.com/").expect("should parse URL"), + ), + legacy_generate: LegacyGenerateArgs { + js_assets: Some("audit/assets.toml".into()), + config: Some("audit/config.toml".into()), + no_js_assets: false, + no_config: false, + force: true, + cookies: vec![("session".to_string(), "example".to_string())], + }, + }; + + let generate = legacy_generate_args(&args).expect("should build generation args"); + + assert_eq!(generate.url, "https://www.example.com/"); + assert_eq!( + generate.js_assets.as_deref(), + Some(std::path::Path::new("audit/assets.toml")) + ); + assert_eq!( + generate.config.as_deref(), + Some(std::path::Path::new("audit/config.toml")) + ); + assert!(generate.force); + assert_eq!( + generate.cookies, + [("session".to_string(), "example".to_string())] + ); + } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs index 3b511edc1..af0144fe9 100644 --- a/crates/trusted-server-cli/src/commands/audit/page.rs +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -35,16 +35,6 @@ pub(crate) fn run_page(args: &PageAuditArgs) -> Result<(), String> { ) } -/// Runs the generic page audit for a single URL with default browser options -/// (the legacy `ts audit ` alias entry point). -/// -/// # Errors -/// -/// Returns a user-facing string when the browser cannot collect the page. -pub(crate) fn run_page_url(url: &url::Url, scroll: bool) -> Result<(), String> { - run_with_collector(&BrowserCollector::new(), url, scroll) -} - fn run_with_collector( collector: &BrowserCollector, url: &url::Url, diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 8164533f4..b98ed3b6a 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -286,9 +286,35 @@ mod tests { } #[test] - fn audit_legacy_url_parses_as_page_alias() { - let args = parse(&["ts", "audit", "https://www.example.com/"]); - assert!(matches!(args.command, Command::Audit(_))); + fn audit_legacy_url_parses_with_artifact_generation_flags() { + let args = parse(&[ + "ts", + "audit", + "https://www.example.com/", + "--js-assets", + "audit/assets.toml", + "--config", + "audit/config.toml", + "--force", + "--cookie", + "session=example", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + assert_eq!( + audit.legacy_generate.js_assets, + Some(PathBuf::from("audit/assets.toml")) + ); + assert_eq!( + audit.legacy_generate.config, + Some(PathBuf::from("audit/config.toml")) + ); + assert!(audit.legacy_generate.force); + assert_eq!( + audit.legacy_generate.cookies, + [("session".to_string(), "example".to_string())] + ); } #[test] diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 3ef29fcec..e39afe88a 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -79,7 +79,7 @@ Chrome or Chromium must be installed locally. The command checks common PATH names and standard macOS/Linux install locations. ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` By default, the command writes: @@ -99,13 +99,13 @@ ts config validate If a config already exists, avoid overwriting it: ```bash -ts audit https://publisher.example --no-config +ts audit generate https://publisher.example --no-config ``` Use custom output paths when reviewing artifacts first: ```bash -ts audit https://publisher.example \ +ts audit generate https://publisher.example \ --js-assets audit/js-assets.toml \ --config audit/trusted-server.toml ``` @@ -113,9 +113,12 @@ ts audit https://publisher.example \ Use `--force` only when replacing existing output files is intentional: ```bash -ts audit https://publisher.example --force +ts audit generate https://publisher.example --force ``` +The legacy `ts audit ` form remains a compatibility alias for artifact +generation. New automation should use `ts audit generate `. + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 893c5b18f..0d4ab708c 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -113,7 +113,7 @@ ts config init To bootstrap from a public publisher page, run an audit first: ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` The audit command writes `js-assets.toml` plus a draft `trusted-server.toml`. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md index db8197760..018f7f685 100644 --- a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -792,21 +792,23 @@ should become a subcommand namespace: ```bash ts audit page +ts audit generate ts audit ad-templates verify ... ``` The existing #800 `ts audit ` behavior should be preserved as a -compatibility alias for `ts audit page ` during the transition. New -ad-template work should use the nested namespace only. This avoids routing -ambiguity in Clap and keeps generic page audit behavior separate from -ad-template verification. +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. Parsing contract: - `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. - `ts audit ad-templates verify ...` is the canonical ad-template verifier. -- `ts audit ` is a hidden compatibility alias for `ts audit page ` and - is accepted only when `` parses as `http` or `https`. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. - `ts audit ad-templates` must never be treated as a legacy URL positional. - `ts audit page` without a URL must fail with the normal Clap missing-argument error. @@ -834,7 +836,7 @@ If Clap cannot enforce the optional-subcommand plus hidden positional contract cleanly, implement a small custom dispatcher for the `audit` argv tail and test it directly. Required parser tests: -- `ts audit https://www.example.com/` dispatches to page audit; +- `ts audit https://www.example.com/` dispatches to artifact generation; - `ts audit page https://www.example.com/` dispatches to page audit; - `ts audit ad-templates verify https://www.example.com/` dispatches to ad-template verification; From c0da7e7fa58e6be53d9e26f373586567e65995d3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 9 Jul 2026 16:22:09 +0530 Subject: [PATCH 139/195] Box audit CLI arguments --- crates/trusted-server-cli/src/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index b98ed3b6a..9aade197e 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -24,7 +24,7 @@ enum Command { /// Sign in / out / status against an `EdgeZero` adapter. Auth(AuthArgs), /// Browser-backed page and ad-template audits. - Audit(AuditArgs), + Audit(Box), /// Build the project for a target adapter. Build(BuildArgs), /// Trusted Server app-config commands. From 39d22ca3c669f303493b8dceaf84f27fc6a746cd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 9 Jul 2026 16:58:59 +0530 Subject: [PATCH 140/195] Run browser fixture tests serially --- .github/workflows/test.yml | 3 +-- crates/trusted-server-cli/src/commands/audit/browser.rs | 4 ++-- scripts/test-cli.sh | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f2717dcb..8ed0a920b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -210,8 +210,7 @@ jobs: cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh test-typescript: name: vitest diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index 9f6aca1e6..a67591b59 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -489,12 +489,12 @@ mod tests { "#; #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] fn collects_gpt_slot_from_local_fixture() { if !chrome_available() { // Browser fixture test requires a local Chrome/Chromium; skipping. return; } - let mut fixture = tempfile::Builder::new() .suffix(".html") .tempfile() @@ -536,12 +536,12 @@ mod tests { } #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { if !chrome_available() { // Browser fixture test requires a local Chrome/Chromium; skipping. return; } - let mut fixture = tempfile::Builder::new() .suffix(".html") .tempfile() diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..379771675 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,5 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + commands::audit::browser::tests:: -- --ignored --test-threads=1 From 671a742eac0d011fe431b9052baa08680ecf3260 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 15 Jul 2026 16:22:31 +0530 Subject: [PATCH 141/195] Make ad-template audit collector resilient to non-loading pages Ad-heavy publisher pages (video players, continuous ad refresh, anti-bot scripts) may never fire the `load` event, so `page.goto` would block until the navigation timeout and the audit failed before scraping any slots. Article pages consistently timed out this way while lighter listing pages succeeded. Navigate without hard-failing on the load wait: a load-wait or main-document-response timeout is downgraded to a "results may be partial" warning, and the existing settle loop is the real readiness signal. The settle loop now also accepts `interactive` readyState, since these pages define their GPT slots before (or without ever) reaching `complete`. Load wait is bounded separately at 12s and the settle cap is raised to 12s so lazily-defined slots are captured. --- .../audit/generate/browser_collector.rs | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 1f0694bf4..b1446933c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -19,8 +19,13 @@ use crate::error::{CliResult, report_error}; const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +const SETTLE_MAX_WAIT: Duration = Duration::from_secs(12); +/// How long to wait for the navigation `load` event (and, separately, the main +/// document response) before falling through to the settle loop. Ad-heavy pages +/// (video players, continuous ad refresh) may never fire `load`, so this is a +/// soft bound: the settle loop is the real readiness signal and the scrape reads +/// whatever rendered by then. +const NAVIGATION_LOAD_TIMEOUT: Duration = Duration::from_secs(12); const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; const RESOURCE_TIMING_BUFFER_WARNING: &str = @@ -124,28 +129,41 @@ async fn collect_page_from_browser( .map_err(|error| report_error(format!("failed to set cookie `{name}`: {error}")))?; } - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; + let mut warnings = Vec::new(); - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; + // Navigate, but don't hard-fail when the `load` event never fires. Ad-heavy + // pages (video players, continuous ad refresh, anti-bot scripts) can keep + // the frame "loading" indefinitely, so a load-wait timeout is downgraded to + // a warning: the settle loop below is the real readiness signal and the + // scrape reads whatever rendered by then. + match timeout(NAVIGATION_LOAD_TIMEOUT, page.goto(target_url.as_str())).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(format!( + "navigation to `{target_url}` did not complete cleanly ({error}); results may be partial" + )), + Err(_) => warnings.push(format!( + "navigation to `{target_url}` did not fire `load` within {}s; results may be partial", + NAVIGATION_LOAD_TIMEOUT.as_secs() + )), + } - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); + // Best-effort read of the main-document response for status validation. When + // the load wait above times out the response is usually already buffered, so + // this returns promptly; tolerate a miss rather than failing the audit. + match timeout(NAVIGATION_LOAD_TIMEOUT, page.wait_for_navigation_response()).await { + Ok(Ok(navigation_response)) => { + if let Some(warning) = validate_navigation_response(navigation_response)? { + warnings.push(warning); + } + } + Ok(Err(error)) => warnings.push(format!( + "could not read the main document response from `{target_url}` ({error}); results may be partial" + )), + Err(_) => warnings.push(format!( + "timed out reading the main document response from `{target_url}`; results may be partial" + )), } + if !wait_for_page_settle(&page).await? { warnings.push( "browser audit timed out while waiting for the page to settle; results may be partial" @@ -286,7 +304,11 @@ async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { .into_value() .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - if ready_state == "complete" { + // Accept `interactive` as well as `complete`: ad-heavy pages often never + // reach `complete` (the `load` event never fires), but their GPT slots + // are defined once the DOM is interactive, so a quiet network period at + // `interactive` is a valid settle signal for the slot scrape. + if ready_state == "complete" || ready_state == "interactive" { if previous_count == Some(resource_count) { stable_for += SETTLE_POLL_INTERVAL; } else { From 1d852a7c6115b42978f5a6d1ceeb83c607f2ca28 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 15 Jul 2026 17:22:53 +0530 Subject: [PATCH 142/195] Keep one managed-slot header comment across generate re-runs `render_slots` prepends a `# Slots managed by ...` header, but the in-place splice preserved the previous copy in the scalar block and inserted a fresh one, so each `ts audit ad-templates generate` run against an already-managed config appended another duplicate comment block. Extract the two header lines to constants and strip any prior copy (and the blank lines it leaves) from the preserved head before re-inserting the rendered slots, so repeated runs keep exactly one header. Add a regression test that splices three times and asserts a single header. --- .../src/commands/audit/generate/slot_toml.rs | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index b15e6ed9f..63f81b50c 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -188,12 +188,17 @@ fn matching_slot_index(existing: &[RenderSlot], discovered: &RenderSlot) -> Opti existing.iter().position(|slot| slot.key() == key) } +/// Header comment emitted above the managed slot array. Stripped from the +/// preserved scalar block on re-splice (see [`is_managed_comment_line`]) so +/// repeated `generate` runs don't accumulate duplicate copies. +const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; +/// Second line of the managed-slot header comment. +const MANAGED_SLOTS_REVIEW_COMMENT: &str = + "# Review page_patterns and formats before validating/pushing."; + /// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. pub(super) fn render_slots(slots: &[RenderSlot]) -> String { - let mut out = String::from( - "\n# Slots managed by `ts audit ad-templates generate`.\n\ - # Review page_patterns and formats before validating/pushing.\n", - ); + let mut out = format!("\n{MANAGED_SLOTS_COMMENT}\n{MANAGED_SLOTS_REVIEW_COMMENT}\n"); for slot in slots { out.push_str("\n[[creative_opportunities.slot]]\n"); out.push_str(&format!("id = {}\n", toml_string(&slot.id))); @@ -409,9 +414,20 @@ pub(super) fn splice_creative_slots( .position(|line| is_unrelated_table(line)) .map_or(lines.len(), |offset| start + offset); - let mut result = lines[..start].join("\n"); + // Preserve everything before the slot array, but drop any prior managed + // header comment (and the blank lines it leaves behind): `rendered` re-emits + // it, so keeping the old copy would duplicate it on every re-splice. + let mut head_lines: Vec<&str> = lines[..start] + .iter() + .copied() + .filter(|line| !is_managed_comment_line(line)) + .collect(); + while head_lines.last().is_some_and(|line| line.trim().is_empty()) { + head_lines.pop(); + } + let mut result = head_lines.join("\n"); if !result.is_empty() { - result.push('\n'); + result.push_str("\n\n"); } result.push_str(rendered); result.push('\n'); @@ -478,6 +494,14 @@ fn is_table_header(line: &str, section_header: &str) -> bool { strip_inline_comment(line.trim()) == section_header } +/// Whether `line` is one of the managed header comment lines emitted by +/// [`render_slots`]. Used to strip the prior copy on re-splice so repeated +/// `generate` runs keep exactly one header comment. +fn is_managed_comment_line(line: &str) -> bool { + let trimmed = line.trim(); + trimmed == MANAGED_SLOTS_COMMENT || trimmed == MANAGED_SLOTS_REVIEW_COMMENT +} + pub(super) fn replace_key_in_section( document: &str, section: &str, @@ -681,6 +705,32 @@ mod tests { ); } + #[test] + fn resplice_does_not_accumulate_managed_comment() { + // A re-run splices into a config that already carries the managed + // header comment; it must keep exactly one copy, not append another. + let first = splice_creative_slots( + "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", + Some("222"), + &header_rendered(), + ) + .expect("first splice"); + let second = + splice_creative_slots(&first, Some("222"), &header_rendered()).expect("second splice"); + let third = + splice_creative_slots(&second, Some("222"), &header_rendered()).expect("third splice"); + + assert_eq!( + third + .lines() + .filter(|line| line.trim() == MANAGED_SLOTS_COMMENT) + .count(), + 1, + "managed header comment must not accumulate across re-splices" + ); + toml::from_str::(&third).expect("re-spliced config stays valid TOML"); + } + #[test] fn splice_recognizes_inline_commented_section_header() { // `[creative_opportunities] # comment` is valid TOML; the splice must From e081e2c166ec3c8743e09cb790010b40e18f9ae2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 15 Jul 2026 17:36:29 +0530 Subject: [PATCH 143/195] Wrap assert! in ad-stack gate test to satisfy CI rustfmt CI's rustfmt wraps the single method-chain argument of this assert! onto its own lines; the compact form the merge brought in passed locally but failed the format gate. Match CI's canonical form. --- crates/trusted-server-core/src/creative_opportunities.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index b643bb914..e55c5676f 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -594,9 +594,11 @@ mod tests { }); assert_eq!(result.expected, RuntimeAdStackExpected::No); - assert!(result - .blocking_gates() - .contains(&AdStackGateName::AuctionEnabled)); + assert!( + result + .blocking_gates() + .contains(&AdStackGateName::AuctionEnabled) + ); } #[test] From d5be3d96b86f4d6c693d219af54d5fe6b6e674a9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 17 Jul 2026 18:01:32 +0530 Subject: [PATCH 144/195] Add admin endpoint to look up EC entries by id Adds GET /_ts/admin/ec/{id} (explicit EC ID) and GET /_ts/admin/ec (EC ID from the caller's ts-ec cookie) so operators can inspect EC identity graph entries and debug KV-to-auction EID propagation. The core handler returns the stored KvEntry verbatim (including raw consent strings and partner UIDs), the KV metadata mirror, the store generation marker, and a derived auction view showing exactly which EIDs the auction would attach and why each stored partner ID was skipped (empty_uid, not_in_registry, bidstream_disabled). Corrupt entries are returned with the parse error and raw body via the new KvIdentityGraph::lookup_raw instead of failing closed. The routes join Settings::ADMIN_ENDPOINTS so startup validation rejects configs whose basic-auth handler regex does not cover them. The EC identity graph is Fastly KV backed, so the Axum, Cloudflare, and Spin adapters register the routes to local 501 responses, keeping them off the publisher fallback that would forward the Authorization header to the origin. Closes #921 --- crates/trusted-server-adapter-axum/src/app.rs | 31 +- .../tests/routes.rs | 34 + .../src/app.rs | 25 + .../tests/routes.rs | 28 + .../trusted-server-adapter-fastly/src/app.rs | 44 ++ crates/trusted-server-adapter-spin/src/app.rs | 29 +- .../tests/routes.rs | 26 + crates/trusted-server-core/src/ec/admin.rs | 626 ++++++++++++++++++ crates/trusted-server-core/src/ec/kv.rs | 21 +- crates/trusted-server-core/src/ec/mod.rs | 1 + crates/trusted-server-core/src/settings.rs | 28 +- 11 files changed, 885 insertions(+), 8 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/admin.rs diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2f4329574..12acbfc68 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -252,6 +252,7 @@ enum NamedRouteHandler { TrustedServerDiscovery, VerifySignature, AdminNotSupported, + AdminEcNotSupported, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -279,7 +280,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 12] { +fn named_routes() -> [NamedRoute; 14] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -304,6 +305,19 @@ fn named_routes() -> [NamedRoute; 12] { primary_methods: &[Method::POST], handler: NamedRouteHandler::AdminNotSupported, }, + // Admin EC lookup routes. Registered explicitly (like the key routes + // above) so they never fall through to the publisher fallback, and + // they match `Settings::ADMIN_ENDPOINTS` for auth coverage. + NamedRoute { + path: "/_ts/admin/ec", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcNotSupported, + }, + NamedRoute { + path: "/_ts/admin/ec/{id}", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcNotSupported, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with // a 404, matching the Fastly and Cloudflare adapters: the production // basic-auth handler regex `^/_ts/admin` does not match them, and letting @@ -388,6 +402,21 @@ fn named_route_handler( ); Ok(resp) } + NamedRouteHandler::AdminEcNotSupported => { + // The EC identity graph is Fastly KV backed; the Axum + // dev server has no store to read. + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on the Axum dev server.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut resp = Response::new(body); + *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; + resp.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + Ok(resp) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index c4bf7d990..7c20e2dd2 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -74,6 +74,8 @@ fn all_explicit_routes_are_registered() { ("POST", "/verify-signature"), ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), + ("GET", "/_ts/admin/ec"), + ("GET", "/_ts/admin/ec/{id}"), ("POST", "/admin/keys/rotate"), ("POST", "/admin/keys/deactivate"), ("POST", "/auction"), @@ -256,6 +258,38 @@ async fn admin_route_without_credentials_returns_401() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so the Axum dev server + // answers the admin EC lookup routes locally with 501 instead of letting + // them fall through to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Axum EC lookup is unsupported" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: the production basic-auth regex diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c931360f6..85eb09f1b 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -242,6 +242,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn admin_ec_lookup_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on Cloudflare Workers.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases on the Cloudflare adapter. /// @@ -461,6 +475,17 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/deactivate", |_ctx: RequestContext| async { Ok::(admin_key_management_not_supported()) }) + // Admin EC lookup routes. Registered explicitly (like the key + // routes above) so they never fall through to the publisher + // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth + // coverage. The EC identity graph is Fastly KV backed, so this + // adapter has no store to read. + .get("/_ts/admin/ec", |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }) + .get("/_ts/admin/ec/{id}", |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index df2781945..7b048833b 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -215,6 +215,8 @@ fn all_explicit_routes_are_registered() { ("POST", "/verify-signature"), ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), + ("GET", "/_ts/admin/ec"), + ("GET", "/_ts/admin/ec/{id}"), ("POST", "/auction"), ("GET", "/first-party/proxy"), ("GET", "/first-party/click"), @@ -264,6 +266,32 @@ async fn authenticated_admin_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so Cloudflare answers the + // admin EC lookup routes locally with 501 instead of letting them fall + // through to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let req = request_builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Cloudflare EC lookup is unsupported" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 955ff235b..71d906d52 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -22,6 +22,8 @@ //! | POST | `/verify-signature` | [`handle_verify_signature`] | //! | POST | `/_ts/admin/keys/rotate` | [`handle_rotate_key`] | //! | POST | `/_ts/admin/keys/deactivate` | [`handle_deactivate_key`] | +//! | GET | `/_ts/admin/ec` | [`handle_admin_ec_lookup`] | +//! | GET | `/_ts/admin/ec/{id}` | [`handle_admin_ec_lookup`] | //! | POST | `/_ts/api/v1/batch-sync` | [`handle_batch_sync`] | //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | @@ -98,6 +100,7 @@ use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_ec_lookup; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -565,6 +568,10 @@ async fn run_named_route( } NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), + NamedRouteHandler::AdminEcLookup => { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_ec_lookup(ec.kv_graph.as_ref(), &partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. @@ -987,6 +994,7 @@ enum NamedRouteHandler { VerifySignature, RotateKey, DeactivateKey, + AdminEcLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -1039,6 +1047,18 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, }, + // Admin EC lookup: the bare route reads the EC ID from the caller's + // `ts-ec` cookie; the parameterized route takes an explicit EC ID. + NamedRoute { + path: "/_ts/admin/ec", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcLookup, + }, + NamedRoute { + path: "/_ts/admin/ec/{id}", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEcLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler // regex `^/_ts/admin` does not match them, and letting them fall through to @@ -1624,6 +1644,30 @@ mod tests { } } + #[test] + fn admin_ec_lookup_routes_are_registered() { + // Both lookup shapes must be explicitly routed to the admin EC + // handler: the bare cookie-based route and the parameterized route. + // Leaving either unrouted would fall through to the publisher + // fallback, forwarding the caller's `Authorization` header to the + // origin. + for path in ["/_ts/admin/ec", "/_ts/admin/ec/{id}"] { + let route = NAMED_ROUTES + .iter() + .find(|route| route.path == path) + .unwrap_or_else(|| panic!("{path} must be a named route")); + assert!( + matches!(route.handler, NamedRouteHandler::AdminEcLookup), + "{path} must map to the admin EC lookup handler" + ); + assert_eq!( + route.primary_methods, + &[Method::GET], + "{path} must have GET as its only primary method" + ); + } + } + #[test] fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: with a production-shaped diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 2291fce74..29ca574ff 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -141,12 +141,14 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), ("/_ts/admin/keys/rotate", &[Method::POST]), ("/_ts/admin/keys/deactivate", &[Method::POST]), + ("/_ts/admin/ec", &[Method::GET]), + ("/_ts/admin/ec/{id}", &[Method::GET]), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), @@ -359,6 +361,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn admin_ec_lookup_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Admin EC lookup is not supported on Fermyon Spin.\n\ + Use the Fastly adapter (via Viceroy or deployed) to inspect EC entries.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -511,6 +527,10 @@ fn build_router(state: &Arc) -> RouterService { Ok::(admin_key_management_not_supported()) }; + let admin_ec_not_supported_handler = |_ctx: RequestContext| async { + Ok::(admin_ec_lookup_not_supported()) + }; + // /auction let s = Arc::clone(&state); let auction_handler = move |ctx: RequestContext| { @@ -730,6 +750,13 @@ fn build_router(state: &Arc) -> RouterService { // credentials and key-management payloads to the origin. .post("/_ts/admin/keys/rotate", admin_not_supported_handler) .post("/_ts/admin/keys/deactivate", admin_not_supported_handler) + // Admin EC lookup routes. Registered explicitly (like the key + // routes above) so they never fall through to the publisher + // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth + // coverage. The EC identity graph is Fastly KV backed, so this + // adapter has no store to read. + .get("/_ts/admin/ec", admin_ec_not_supported_handler) + .get("/_ts/admin/ec/{id}", admin_ec_not_supported_handler) .post("/auction", auction_handler) .get("/__ts/page-bids", page_bids_handler) .route( diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 9b96dbd70..4194baea4 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -113,6 +113,32 @@ async fn authenticated_admin_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_ec_routes_return_501() { + // The EC identity graph is Fastly KV backed, so Spin answers the admin + // EC lookup routes locally with 501 instead of letting them fall through + // to the publisher fallback. + let sample_ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin/ec".to_owned(), + format!("/_ts/admin/ec/{sample_ec_id}"), + ] { + let req = request_builder() + .method("GET") + .uri(&path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 501, + "{path} should report that Spin EC lookup is unsupported" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_route_returns_ok() { // Parity with the Fastly/Axum adapters: GET /health is a cheap liveness probe diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs new file mode 100644 index 000000000..54599d59d --- /dev/null +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -0,0 +1,626 @@ +//! Admin endpoint for inspecting EC identity graph entries. +//! +//! Serves `GET /_ts/admin/ec` (EC ID taken from the request's `ts-ec` +//! cookie) and `GET /_ts/admin/ec/{id}` (explicit EC ID). Returns the raw +//! stored [`KvEntry`] plus a derived view of the EIDs the auction would +//! attach, so operators can debug KV-to-auction propagation without KV +//! console access. +//! +//! Authentication is enforced by the `^/_ts/admin` basic-auth handler +//! configuration; startup validation rejects configs that leave these paths +//! uncovered (see `Settings::ADMIN_ENDPOINTS`). Because the endpoint is +//! auth-gated and operator-facing, responses intentionally include full +//! internal detail (raw consent strings, partner UIDs, parse errors). + +use http::{Request, Response, StatusCode, header}; +use serde::Serialize; +use serde_json::Value as JsonValue; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; + +use crate::constants::COOKIE_TS_EC; +use crate::error::TrustedServerError; +use crate::openrtb::Eid; + +use super::eids::{resolve_partner_ids, to_eids}; +use super::generation::is_valid_ec_id; +use super::kv::KvIdentityGraph; +use super::kv_backend::EcKvLookup; +use super::kv_types::{KvEntry, KvMetadata}; +use super::log_id; +use super::registry::PartnerRegistry; + +/// Route prefix shared by the cookie-based and explicit-ID lookup routes. +const ADMIN_EC_PATH: &str = "/_ts/admin/ec"; + +/// Successful admin EC lookup payload. +#[derive(Debug, Serialize)] +struct AdminEcLookupResponse { + /// The EC ID that was looked up. + ec_id: String, + /// Platform KV store name the entry was read from. + store: String, + /// Store generation marker for the entry. + generation: u64, + /// `true` when the entry is a consent-withdrawal tombstone + /// (`consent.ok = false`). Absent when the body failed to parse. + #[serde(skip_serializing_if = "Option::is_none")] + tombstone: Option, + /// The stored entry, re-serialized verbatim. Absent when the body + /// failed to deserialize (see `entry_error` / `raw_body`). + #[serde(skip_serializing_if = "Option::is_none")] + entry: Option, + /// Deserialization or validation failure detail for the entry body. + #[serde(skip_serializing_if = "Option::is_none")] + entry_error: Option, + /// Raw entry body (lossy UTF-8) when it could not be deserialized. + #[serde(skip_serializing_if = "Option::is_none")] + raw_body: Option, + /// The stored KV metadata mirror, when present and parseable. + #[serde(skip_serializing_if = "Option::is_none")] + metadata: Option, + /// Deserialization failure detail for the metadata, including its raw + /// value. + #[serde(skip_serializing_if = "Option::is_none")] + metadata_error: Option, + /// Derived auction view. Present only when the entry deserializes and + /// validates — the same precondition the auction read path applies, so + /// its absence means the auction would attach no KV-derived EIDs. + /// Live requests additionally gate on per-request consent, which is not + /// reproducible here. + #[serde(skip_serializing_if = "Option::is_none")] + auction: Option, +} + +/// What the auction EID decoration would produce for this entry. +#[derive(Debug, Serialize)] +struct AuctionEidsView { + /// EIDs the auction would attach to `user.eids`, exactly as produced by + /// the auction resolution path. + eids: Vec, + /// Stored partner IDs that the auction resolution filters out, with the + /// reason each was skipped. + skipped: Vec, +} + +/// A stored partner ID excluded from auction EIDs. +#[derive(Debug, Serialize)] +struct SkippedPartnerId { + /// Partner namespace key in the entry's `ids` map. + source_domain: String, + /// Why the auction resolution skips it: `empty_uid`, `not_in_registry`, + /// or `bidstream_disabled`. + reason: &'static str, +} + +/// Handles `GET /_ts/admin/ec` and `GET /_ts/admin/ec/{id}`. +/// +/// Resolves the EC ID from the path when present, falling back to the +/// request's `ts-ec` cookie for the bare route. Responds: +/// +/// - `200 OK` with an [`AdminEcLookupResponse`] JSON body when the key +/// exists (including corrupt entries, which are reported with +/// `entry_error` and `raw_body` instead of failing closed); +/// - `400 Bad Request` when the resolved ID is not a valid EC ID; +/// - `404 Not Found` when the key does not exist, or the bare route was +/// called without a `ts-ec` cookie; +/// - `501 Not Implemented` when no EC identity graph is configured. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::KvStore`] when the store open or read +/// fails. +pub fn handle_admin_ec_lookup( + kv: Option<&KvIdentityGraph>, + registry: &PartnerRegistry, + req: &Request, +) -> Result, Report> { + let Some(kv) = kv else { + return Ok(json_error( + StatusCode::NOT_IMPLEMENTED, + "EC identity graph is not configured on this deployment", + )); + }; + + let ec_id = match requested_ec_id(req) { + Ok(ec_id) => ec_id, + Err(response) => return Ok(*response), + }; + + let Some(lookup) = kv.lookup_raw(&ec_id)? else { + log::info!("Admin EC lookup: no entry for '{}'", log_id(&ec_id)); + return Ok(json_error( + StatusCode::NOT_FOUND, + "EC entry not found (KV reads are eventually consistent; a very \ + recent entry may not be visible yet)", + )); + }; + + log::info!("Admin EC lookup: returning entry for '{}'", log_id(&ec_id)); + let payload = build_lookup_response(registry, kv.store_name(), ec_id, &lookup); + let body = + serde_json::to_string(&payload).change_context(TrustedServerError::Configuration { + message: "failed to serialize admin EC lookup response".to_owned(), + })?; + Ok(json_response(StatusCode::OK, body)) +} + +/// Resolves the EC ID to look up from the path or the `ts-ec` cookie. +/// +/// Returns the (boxed) error response to send directly when no valid ID is +/// available. +fn requested_ec_id(req: &Request) -> Result>> { + let remainder = req + .uri() + .path() + .strip_prefix(ADMIN_EC_PATH) + .unwrap_or("") + .trim_matches('/'); + + let ec_id = if remainder.is_empty() { + match extract_cookie_value(req, COOKIE_TS_EC) { + Some(cookie_ec_id) => cookie_ec_id, + None => { + return Err(Box::new(json_error( + StatusCode::NOT_FOUND, + "no EC ID in path and no ts-ec cookie on the request", + ))); + } + } + } else { + remainder.to_owned() + }; + + if !is_valid_ec_id(&ec_id) { + return Err(Box::new(json_error( + StatusCode::BAD_REQUEST, + "invalid EC ID format (expected {64hex}.{6alnum})", + ))); + } + + Ok(ec_id) +} + +/// Builds the success payload from a raw KV lookup. +/// +/// Parse failures are reported in the payload rather than propagated, so +/// corrupt entries remain inspectable. +fn build_lookup_response( + registry: &PartnerRegistry, + store_name: &str, + ec_id: String, + lookup: &EcKvLookup, +) -> AdminEcLookupResponse { + let mut payload = AdminEcLookupResponse { + ec_id, + store: store_name.to_owned(), + generation: lookup.generation, + tombstone: None, + entry: None, + entry_error: None, + raw_body: None, + metadata: None, + metadata_error: None, + auction: None, + }; + + match serde_json::from_slice::(&lookup.body) { + Ok(entry) => { + payload.tombstone = Some(!entry.consent.ok); + match entry.validate() { + Ok(()) => payload.auction = Some(build_auction_view(registry, &entry)), + Err(message) => { + payload.entry_error = Some(format!( + "entry failed validation (auction reads fail closed \ + and attach no EIDs): {message}" + )); + } + } + payload.entry = Some(serde_json::to_value(&entry).expect("should serialize KvEntry")); + } + Err(error) => { + payload.entry_error = Some(format!("failed to deserialize entry: {error}")); + payload.raw_body = Some(String::from_utf8_lossy(&lookup.body).into_owned()); + } + } + + match &lookup.metadata { + None => {} + Some(bytes) => match serde_json::from_slice::(bytes) { + Ok(metadata) => { + payload.metadata = + Some(serde_json::to_value(&metadata).expect("should serialize KvMetadata")); + } + Err(error) => { + payload.metadata_error = Some(format!( + "failed to deserialize metadata: {error} (raw: {})", + String::from_utf8_lossy(bytes) + )); + } + }, + } + + payload +} + +/// Derives the auction EID view for a valid entry, mirroring the filters in +/// [`resolve_partner_ids`] and reporting why each stored ID was skipped. +fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { + let resolved = resolve_partner_ids(registry, entry); + let eids = to_eids(&resolved); + + let mut skipped = Vec::new(); + for (source_domain, partner_uid) in &entry.ids { + let reason = if partner_uid.uid.is_empty() { + "empty_uid" + } else { + match registry.get(source_domain) { + None => "not_in_registry", + Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", + Some(_) => continue, + } + }; + skipped.push(SkippedPartnerId { + source_domain: source_domain.clone(), + reason, + }); + } + + AuctionEidsView { eids, skipped } +} + +fn extract_cookie_value(req: &Request, name: &str) -> Option { + let cookie_header = req + .headers() + .get(header::COOKIE) + .and_then(|value| value.to_str().ok())?; + for pair in cookie_header.split(';') { + let pair = pair.trim(); + if let Some((key, value)) = pair.split_once('=') + && key.trim() == name + { + return Some(value.trim().to_owned()); + } + } + None +} + +fn json_error(status: StatusCode, message: &str) -> Response { + let body = serde_json::json!({ "error": message }); + json_response(status, body.to_string()) +} + +fn json_response(status: StatusCode, body: String) -> Response { + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::from(body.into_bytes())) + .expect("should build admin EC lookup response") +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; + use crate::ec::kv_types::KvPartnerId; + use crate::redacted::Redacted; + use crate::settings::EcPartner; + + fn test_ec_id() -> String { + format!("{}.abc123", "a".repeat(64)) + } + + fn make_test_partner(source_domain: &str, bidstream_enabled: bool) -> EcPartner { + EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled, + api_token: Redacted::new(format!("test-token-{source_domain:-<32}")), + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: false, + pull_sync_url: None, + pull_sync_allowed_domains: vec![], + pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: None, + } + } + + fn test_registry() -> PartnerRegistry { + PartnerRegistry::from_config(&[ + make_test_partner("bidstream.example", true), + make_test_partner("disabled.example", false), + ]) + .expect("should build test partner registry") + } + + fn get_request(path: &str) -> Request { + Request::builder() + .method("GET") + .uri(format!("https://edge.example.com{path}")) + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn get_request_with_cookie(path: &str, cookie: &str) -> Request { + Request::builder() + .method("GET") + .uri(format!("https://edge.example.com{path}")) + .header(header::COOKIE, cookie) + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn kv_with_entry(ec_id: &str, entry: &KvEntry) -> KvIdentityGraph { + let kv = KvIdentityGraph::in_memory("test-store"); + kv.create(ec_id, entry).expect("should seed KV entry"); + kv + } + + fn kv_with_raw_body(ec_id: &str, body: &str) -> KvIdentityGraph { + let metadata = serde_json::json!({ "ok": true, "country": "US", "v": 1 }).to_string(); + let store = InMemoryEcKv::new("test-store"); + store + .insert( + ec_id, + EcKvWrite { + body, + metadata: &metadata, + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed raw KV body"); + KvIdentityGraph::new(store) + } + + fn response_json(response: Response) -> JsonValue { + serde_json::from_slice(&response.into_body().into_bytes().unwrap_or_default()) + .expect("should parse response body as JSON") + } + + fn sample_entry() -> KvEntry { + let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); + entry.ids.insert( + "disabled.example".to_owned(), + KvPartnerId { + uid: "uid-disabled".to_owned(), + }, + ); + entry.ids.insert( + "unknown.example".to_owned(), + KvPartnerId { + uid: "uid-unknown".to_owned(), + }, + ); + entry + } + + #[test] + fn returns_entry_with_auction_view() { + let ec_id = test_ec_id(); + let kv = kv_with_entry(&ec_id, &sample_entry()); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "should send no-store on admin responses" + ); + + let json = response_json(response); + assert_eq!(json["ec_id"], ec_id.as_str()); + assert_eq!(json["store"], "test-store"); + assert_eq!(json["tombstone"], false); + assert_eq!( + json["entry"]["ids"]["bidstream.example"]["uid"], "uid-live", + "should echo the stored entry verbatim" + ); + + let eids = json["auction"]["eids"] + .as_array() + .expect("should have auction eids"); + assert_eq!(eids.len(), 1, "should resolve only the bidstream partner"); + assert_eq!(eids[0]["source"], "bidstream.example"); + assert_eq!(eids[0]["uids"][0]["id"], "uid-live"); + + let skipped = json["auction"]["skipped"] + .as_array() + .expect("should have skipped list"); + assert_eq!(skipped.len(), 2, "should report both filtered partners"); + assert!( + skipped + .iter() + .any(|s| s["source_domain"] == "disabled.example" + && s["reason"] == "bidstream_disabled"), + "should report the bidstream-disabled partner" + ); + assert!( + skipped.iter().any( + |s| s["source_domain"] == "unknown.example" && s["reason"] == "not_in_registry" + ), + "should report the unregistered partner" + ); + } + + #[test] + fn reports_tombstone_entries() { + let ec_id = test_ec_id(); + let kv = KvIdentityGraph::in_memory("test-store"); + kv.write_withdrawal_tombstone(&ec_id) + .expect("should write tombstone"); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["tombstone"], true, "should flag tombstone entries"); + assert!( + json["auction"]["eids"] + .as_array() + .expect("should have auction eids") + .is_empty(), + "tombstone should resolve no EIDs" + ); + } + + #[test] + fn missing_entry_returns_404() { + let kv = KvIdentityGraph::in_memory("test-store"); + let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[test] + fn invalid_id_returns_400() { + let kv = KvIdentityGraph::in_memory("test-store"); + let req = get_request("/_ts/admin/ec/not-a-valid-id"); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn corrupt_entry_returns_parse_error_and_raw_body() { + let ec_id = test_ec_id(); + let kv = kv_with_raw_body(&ec_id, "not json at all"); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!( + response.status(), + StatusCode::OK, + "corrupt entries should be inspectable, not opaque errors" + ); + let json = response_json(response); + assert!( + json["entry_error"] + .as_str() + .expect("should have entry_error") + .contains("failed to deserialize"), + "should describe the parse failure" + ); + assert_eq!(json["raw_body"], "not json at all"); + assert!(json.get("entry").is_none(), "should omit unparsed entry"); + assert!( + json.get("auction").is_none(), + "should omit auction view for unparseable entries" + ); + assert_eq!( + json["metadata"]["country"], "US", + "should still parse the stored metadata" + ); + } + + #[test] + fn invalid_schema_version_reports_validation_error() { + let ec_id = test_ec_id(); + let body = serde_json::json!({ + "v": 99, + "created": 1000, + "consent": { "ok": true, "updated": 1000 }, + "geo": { "country": "US" } + }) + .to_string(); + let kv = kv_with_raw_body(&ec_id, &body); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert!( + json["entry_error"] + .as_str() + .expect("should have entry_error") + .contains("failed validation"), + "should describe the validation failure" + ); + assert_eq!(json["entry"]["v"], 99, "should still show the parsed entry"); + assert!( + json.get("auction").is_none(), + "should omit auction view when the auction read would fail closed" + ); + } + + #[test] + fn bare_route_uses_ts_ec_cookie() { + let ec_id = test_ec_id(); + let kv = kv_with_entry(&ec_id, &sample_entry()); + let req = get_request_with_cookie("/_ts/admin/ec", &format!("other=1; ts-ec={ec_id}; x=2")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!( + json["ec_id"], + ec_id.as_str(), + "should resolve the EC ID from the ts-ec cookie" + ); + } + + #[test] + fn bare_route_without_cookie_returns_404() { + let kv = KvIdentityGraph::in_memory("test-store"); + let req = get_request("/_ts/admin/ec"); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let json = response_json(response); + assert!( + json["error"] + .as_str() + .expect("should have error message") + .contains("ts-ec cookie"), + "should explain the missing cookie" + ); + } + + #[test] + fn missing_identity_graph_returns_501() { + let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); + + let response = + handle_admin_ec_lookup(None, &test_registry(), &req).expect("should handle lookup"); + + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); + } + + #[test] + fn kv_read_failure_propagates() { + let kv = KvIdentityGraph::failing("broken-store"); + let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); + + let result = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req); + + assert!(result.is_err(), "should propagate KV read failures"); + } +} diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 7be767557..3572581ce 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,7 @@ use crate::error::TrustedServerError; use super::current_timestamp; use super::generation::ec_hash; -use super::kv_backend::{EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; +use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; use super::log_id; @@ -170,6 +170,25 @@ impl KvIdentityGraph { Ok((body, meta_str)) } + /// Reads the raw stored body, metadata, and generation for an EC ID key. + /// + /// Unlike [`Self::get`], the entry body is returned without + /// deserialization or validation, so corrupt or legacy-schema records can + /// still be inspected instead of failing closed. Used by the admin EC + /// lookup endpoint. + /// + /// Returns `Ok(None)` when the key does not exist. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store open or read failure. + pub fn lookup_raw( + &self, + ec_id: &str, + ) -> Result, Report> { + self.store.lookup(ec_id) + } + /// Reads the full entry and its generation marker for CAS writes. /// /// Returns `Ok(None)` when the key does not exist. diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 408ea9b32..50eda4d60 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -31,6 +31,7 @@ mod auth; +pub mod admin; pub mod batch_sync; pub mod consent; pub mod cookies; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 03cc535c8..eb463acf3 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2200,9 +2200,18 @@ impl Settings { /// where any of these paths lack a matching handler, ensuring admin /// endpoints are always protected by authentication. /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new - /// admin routes to `crates/trusted-server-adapter-fastly/src/main.rs`. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = - &["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"]; + /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. + /// + /// The `/_ts/admin/ec/{id}` entry is the literal router pattern; handler + /// path regexes are matched against it verbatim, so prefix-style admin + /// regexes (e.g. `^/_ts/admin`) cover it while regexes too narrow to + /// cover the parameterized route are rejected fail-closed. + pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + ]; /// Returns admin endpoint paths that no configured handler covers. /// @@ -5249,7 +5258,12 @@ origin_host_header_overide = "www.example.com""#, .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"], + vec![ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + ], "should report every admin endpoint as uncovered" ); } @@ -5283,7 +5297,11 @@ origin_host_header_overide = "www.example.com""#, .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/_ts/admin/keys/deactivate"], + vec![ + "/_ts/admin/keys/deactivate", + "/_ts/admin/ec", + "/_ts/admin/ec/{id}", + ], "should detect the admin endpoints not covered by the narrow handler" ); } From 12592bb0489371775f47bab74e186e56a6955a84 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 17 Jul 2026 18:46:43 +0530 Subject: [PATCH 145/195] Do not bot-gate the admin EC lookup KV graph The dispatch arm reused EcRequestState::kv_graph, which is deliberately None for clients that fail the browser gate. Operators hit this auth-gated endpoint with curl, so every lookup returned 501 as if no EC store were configured. Build the identity graph directly from settings instead, and document why the bot-gated copy must not be used. Also point the bare-route no-cookie 404 at the explicit-id route, since the ts-ec cookie (Domain-scoped, Secure) cannot exist on localhost. --- crates/trusted-server-adapter-fastly/src/app.rs | 6 +++++- crates/trusted-server-core/src/ec/admin.rs | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 71d906d52..b44b43703 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -569,8 +569,12 @@ async fn run_named_route( NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), NamedRouteHandler::AdminEcLookup => { + // Deliberately NOT `ec.kv_graph`: that copy is bot-gated (None for + // non-browser clients), and operators hit this auth-gated endpoint + // with curl. Build the graph directly from settings instead. + let kv = crate::maybe_identity_graph(&state.settings); let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - handle_admin_ec_lookup(ec.kv_graph.as_ref(), &partner_registry, &req) + handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 54599d59d..6c256e44e 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -164,7 +164,8 @@ fn requested_ec_id(req: &Request) -> Result { return Err(Box::new(json_error( StatusCode::NOT_FOUND, - "no EC ID in path and no ts-ec cookie on the request", + "no EC ID in path and no ts-ec cookie on the request — pass \ + an explicit id: /_ts/admin/ec/{id}", ))); } } From 9869ac7024003bf6ef5686eba11b6d0a19a59a36 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 17 Jul 2026 19:58:44 +0530 Subject: [PATCH 146/195] Add admin endpoint to echo request EID cookies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds GET /_ts/admin/eids, complementing the EC lookup endpoint with the client-side half of EID propagation: it decodes the request's ts-eids and sharedId cookies and previews what cookie ingestion would write into the EC entry's ids map — matched partner UIDs (deduplicated exactly like the ingestion path) and unmatched sources that would be dropped. The endpoint always responds 200; missing or malformed cookies are reported in the payload rather than as errors. It is pure request inspection with no KV access, so every adapter serves the real handler. The path joins Settings::ADMIN_ENDPOINTS for basic-auth coverage validation. --- crates/trusted-server-adapter-axum/src/app.rs | 17 +- .../tests/routes.rs | 26 ++ .../src/app.rs | 11 + .../tests/routes.rs | 20 ++ .../trusted-server-adapter-fastly/src/app.rs | 29 +- crates/trusted-server-adapter-spin/src/app.rs | 19 +- .../tests/routes.rs | 19 ++ crates/trusted-server-core/src/ec/admin.rs | 259 +++++++++++++++++- .../trusted-server-core/src/ec/prebid_eids.rs | 6 +- crates/trusted-server-core/src/settings.rs | 3 + 10 files changed, 400 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 12acbfc68..61f9e977d 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -12,6 +12,8 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::proxy::{ @@ -253,6 +255,7 @@ enum NamedRouteHandler { VerifySignature, AdminNotSupported, AdminEcNotSupported, + AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -280,7 +283,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 14] { +fn named_routes() -> [NamedRoute; 15] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -318,6 +321,13 @@ fn named_routes() -> [NamedRoute; 14] { primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcNotSupported, }, + // Admin EIDs echo: pure request inspection (no KV), so the dev + // server serves the real handler. + NamedRoute { + path: "/_ts/admin/eids", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEidsLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with // a 404, matching the Fastly and Cloudflare adapters: the production // basic-auth handler regex `^/_ts/admin` does not match them, and letting @@ -417,6 +427,11 @@ fn named_route_handler( ); Ok(resp) } + NamedRouteHandler::AdminEidsLookup => { + let partner_registry = + PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 7c20e2dd2..f64c9361b 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -76,6 +76,7 @@ fn all_explicit_routes_are_registered() { ("POST", "/_ts/admin/keys/deactivate"), ("GET", "/_ts/admin/ec"), ("GET", "/_ts/admin/ec/{id}"), + ("GET", "/_ts/admin/eids"), ("POST", "/admin/keys/rotate"), ("POST", "/admin/keys/deactivate"), ("POST", "/auction"), @@ -290,6 +291,31 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so the dev server + // serves the real handler. + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: the production basic-auth regex diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 85eb09f1b..593522026 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -13,6 +13,8 @@ use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; #[cfg(target_arch = "wasm32")] use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; @@ -486,6 +488,15 @@ fn build_router(state: &Arc) -> RouterService { .get("/_ts/admin/ec/{id}", |_ctx: RequestContext| async { Ok::(admin_ec_lookup_not_supported()) }) + // Admin EIDs echo: pure request inspection (no KV), so this + // adapter serves the real handler. + .get( + "/_ts/admin/eids", + make_handler(Arc::clone(&state), |s, _services, req| async move { + let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + }), + ) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 7b048833b..8ecd020b7 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -217,6 +217,7 @@ fn all_explicit_routes_are_registered() { ("POST", "/_ts/admin/keys/deactivate"), ("GET", "/_ts/admin/ec"), ("GET", "/_ts/admin/ec/{id}"), + ("GET", "/_ts/admin/eids"), ("POST", "/auction"), ("GET", "/first-party/proxy"), ("GET", "/first-party/click"), @@ -292,6 +293,25 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so this adapter + // serves the real handler. + let req = request_builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { let router = test_router(); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index b44b43703..1703b2d67 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -24,6 +24,7 @@ //! | POST | `/_ts/admin/keys/deactivate` | [`handle_deactivate_key`] | //! | GET | `/_ts/admin/ec` | [`handle_admin_ec_lookup`] | //! | GET | `/_ts/admin/ec/{id}` | [`handle_admin_ec_lookup`] | +//! | GET | `/_ts/admin/eids` | [`handle_admin_eids_lookup`] | //! | POST | `/_ts/api/v1/batch-sync` | [`handle_batch_sync`] | //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | @@ -100,7 +101,7 @@ use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::handle_admin_ec_lookup; +use trusted_server_core::ec::admin::{handle_admin_ec_lookup, handle_admin_eids_lookup}; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; @@ -576,6 +577,10 @@ async fn run_named_route( let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; handle_admin_ec_lookup(kv.as_ref(), &partner_registry, &req) } + NamedRouteHandler::AdminEidsLookup => { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + handle_admin_eids_lookup(&partner_registry, &req) + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. @@ -999,6 +1004,7 @@ enum NamedRouteHandler { RotateKey, DeactivateKey, AdminEcLookup, + AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -1063,6 +1069,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, }, + // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with + // an ingestion preview. Pure request inspection — no KV access. + NamedRoute { + path: "/_ts/admin/eids", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::AdminEidsLookup, + }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler // regex `^/_ts/admin` does not match them, and letting them fall through to @@ -1670,6 +1683,20 @@ mod tests { "{path} must have GET as its only primary method" ); } + + let eids_route = NAMED_ROUTES + .iter() + .find(|route| route.path == "/_ts/admin/eids") + .expect("should register /_ts/admin/eids as a named route"); + assert!( + matches!(eids_route.handler, NamedRouteHandler::AdminEidsLookup), + "/_ts/admin/eids must map to the admin EIDs lookup handler" + ); + assert_eq!( + eids_route.primary_methods, + &[Method::GET], + "/_ts/admin/eids must have GET as its only primary method" + ); } #[test] diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 29ca574ff..51909998d 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -11,6 +11,8 @@ use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::admin::handle_admin_eids_lookup; +use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -141,7 +143,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 15] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -149,6 +151,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { ("/_ts/admin/keys/deactivate", &[Method::POST]), ("/_ts/admin/ec", &[Method::GET]), ("/_ts/admin/ec/{id}", &[Method::GET]), + ("/_ts/admin/eids", &[Method::GET]), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), @@ -531,6 +534,19 @@ fn build_router(state: &Arc) -> RouterService { Ok::(admin_ec_lookup_not_supported()) }; + // Admin EIDs echo: pure request inspection (no KV), so this adapter + // serves the real handler. + let s = Arc::clone(&state); + let admin_eids_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let req = ctx.into_request(); + let result = PartnerRegistry::from_config(&s.settings.ec.partners) + .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); + Ok::(result.unwrap_or_else(|e| http_error(&e))) + } + }; + // /auction let s = Arc::clone(&state); let auction_handler = move |ctx: RequestContext| { @@ -757,6 +773,7 @@ fn build_router(state: &Arc) -> RouterService { // adapter has no store to read. .get("/_ts/admin/ec", admin_ec_not_supported_handler) .get("/_ts/admin/ec/{id}", admin_ec_not_supported_handler) + .get("/_ts/admin/eids", admin_eids_handler) .post("/auction", auction_handler) .get("/__ts/page-bids", page_bids_handler) .route( diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 4194baea4..d502a3944 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -139,6 +139,25 @@ async fn authenticated_admin_ec_routes_return_501() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authenticated_admin_eids_route_returns_200() { + // The EIDs echo is pure request inspection (no KV), so this adapter + // serves the real handler. + let req = request_builder() + .method("GET") + .uri("/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + + assert_eq!( + resp.status().as_u16(), + 200, + "/_ts/admin/eids should serve the real EIDs echo handler" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_route_returns_ok() { // Parity with the Fastly/Axum adapters: GET /health is a cheap liveness probe diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 6c256e44e..0724d1f4b 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -1,4 +1,4 @@ -//! Admin endpoint for inspecting EC identity graph entries. +//! Admin endpoints for inspecting EC identity state. //! //! Serves `GET /_ts/admin/ec` (EC ID taken from the request's `ts-ec` //! cookie) and `GET /_ts/admin/ec/{id}` (explicit EC ID). Returns the raw @@ -6,9 +6,13 @@ //! attach, so operators can debug KV-to-auction propagation without KV //! console access. //! +//! Also serves `GET /_ts/admin/eids`, which echoes the request's `ts-eids` +//! and `sharedId` cookies with an ingestion preview — the client-side half +//! of EID propagation that is never stored server-side. +//! //! Authentication is enforced by the `^/_ts/admin` basic-auth handler //! configuration; startup validation rejects configs that leave these paths -//! uncovered (see `Settings::ADMIN_ENDPOINTS`). Because the endpoint is +//! uncovered (see `Settings::ADMIN_ENDPOINTS`). Because the endpoints are //! auth-gated and operator-facing, responses intentionally include full //! internal detail (raw consent strings, partner UIDs, parse errors). @@ -19,7 +23,7 @@ use serde_json::Value as JsonValue; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; -use crate::constants::COOKIE_TS_EC; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS}; use crate::error::TrustedServerError; use crate::openrtb::Eid; @@ -29,6 +33,10 @@ use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; use super::kv_types::{KvEntry, KvMetadata}; use super::log_id; +use super::prebid_eids::{ + collect_prebid_eid_updates, collect_sharedid_update, dedupe_partner_updates, + parse_prebid_eids_cookie, +}; use super::registry::PartnerRegistry; /// Route prefix shared by the cookie-based and explicit-ID lookup routes. @@ -271,6 +279,125 @@ fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEid AuctionEidsView { eids, skipped } } +/// Admin EIDs echo payload. +#[derive(Debug, Serialize)] +struct AdminEidsResponse { + /// Whether a `ts-eids` cookie was present on the request. + cookie_present: bool, + /// EIDs parsed from the `ts-eids` cookie. Absent when the cookie is + /// missing or failed to parse. + #[serde(skip_serializing_if = "Option::is_none")] + eids: Option>, + /// Parse failure detail when the `ts-eids` cookie could not be decoded. + #[serde(skip_serializing_if = "Option::is_none")] + parse_error: Option, + /// Whether a `sharedId` cookie was present on the request. + sharedid_present: bool, + /// Number of partners configured in the registry. + partners_configured: usize, + /// Preview of what cookie ingestion would write into the EC entry's + /// `ids` map on a navigation carrying these cookies. + ingest: IngestPreview, +} + +/// What cookie ingestion would store, and what it would drop. +#[derive(Debug, Serialize)] +struct IngestPreview { + /// Cookie sources matched to a configured partner, with the UID that + /// would be stored (deduplicated exactly like the ingestion path). + matched: Vec, + /// `ts-eids` sources with no configured partner; dropped on ingestion. + unmatched: Vec, +} + +/// A cookie-derived partner UID that ingestion would store. +#[derive(Debug, Serialize)] +struct MatchedPartnerId { + /// Partner namespace key in the EC entry's `ids` map. + source_domain: String, + /// The UID that would be stored. + uid: String, +} + +/// Handles `GET /_ts/admin/eids`. +/// +/// Echoes the request's `ts-eids` and `sharedId` cookies: the parsed EID +/// list plus a preview of what cookie ingestion would write into the EC +/// entry's `ids` map given the configured partner registry. Pure request +/// inspection — no KV access — so it works on every adapter. +/// +/// Always responds `200 OK`; missing or malformed cookies are reported in +/// the payload instead of as errors. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] only when the response +/// payload fails JSON serialization. +pub fn handle_admin_eids_lookup( + registry: &PartnerRegistry, + req: &Request, +) -> Result, Report> { + let eids_cookie = extract_cookie_value(req, COOKIE_TS_EIDS); + let sharedid_cookie = extract_cookie_value(req, COOKIE_SHAREDID); + + let (eids, parse_error) = match &eids_cookie { + None => (None, None), + Some(value) => match parse_prebid_eids_cookie(value) { + Ok(parsed) => (Some(parsed), None), + Err(error) => ( + None, + Some(format!("failed to parse ts-eids cookie: {error}")), + ), + }, + }; + + // Mirror the ingestion path (`ingest_eid_cookies`): collect matches from + // both cookies, then dedupe the same way so the preview reports exactly + // what a navigation would store. + let mut updates = Vec::new(); + if let Some(value) = &eids_cookie { + updates.extend(collect_prebid_eid_updates(value, registry)); + } + if let Some(value) = &sharedid_cookie + && let Some(update) = collect_sharedid_update(value, registry) + { + updates.push(update); + } + let matched = dedupe_partner_updates(updates) + .into_iter() + .map(|update| MatchedPartnerId { + source_domain: update.partner_id, + uid: update.uid, + }) + .collect(); + + let unmatched = eids + .as_ref() + .map(|parsed| { + parsed + .iter() + .filter(|eid| registry.find_by_source_domain(&eid.source).is_none()) + .map(|eid| eid.source.clone()) + .collect() + }) + .unwrap_or_default(); + + let payload = AdminEidsResponse { + cookie_present: eids_cookie.is_some(), + eids, + parse_error, + sharedid_present: sharedid_cookie.is_some(), + partners_configured: registry.len(), + ingest: IngestPreview { matched, unmatched }, + }; + + let body = + serde_json::to_string(&payload).change_context(TrustedServerError::Configuration { + message: "failed to serialize admin EIDs response".to_owned(), + })?; + Ok(json_response(StatusCode::OK, body)) +} + fn extract_cookie_value(req: &Request, name: &str) -> Option { let cookie_header = req .headers() @@ -305,6 +432,8 @@ fn json_response(status: StatusCode, body: String) -> Response { mod tests { use std::time::Duration; + use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; + use super::*; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; @@ -624,4 +753,128 @@ mod tests { assert!(result.is_err(), "should propagate KV read failures"); } + + fn eids_cookie_for(entries: &serde_json::Value) -> String { + BASE64.encode(entries.to_string()) + } + + #[test] + fn eids_lookup_without_cookies_returns_empty_payload() { + let req = get_request("/_ts/admin/eids"); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["cookie_present"], false); + assert_eq!(json["sharedid_present"], false); + assert_eq!(json["partners_configured"], 2); + assert!( + json["ingest"]["matched"] + .as_array() + .expect("should have matched list") + .is_empty(), + "should preview no matches without cookies" + ); + } + + #[test] + fn eids_lookup_parses_cookie_and_previews_ingestion() { + let cookie = eids_cookie_for(&serde_json::json!([ + { + "source": "bidstream.example", + "uids": [{ "id": "uid-configured", "atype": 1 }] + }, + { + "source": "unknown.example", + "uids": [{ "id": "uid-unknown", "atype": 1 }] + } + ])); + let req = get_request_with_cookie("/_ts/admin/eids", &format!("ts-eids={cookie}")); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["cookie_present"], true); + assert_eq!( + json["eids"] + .as_array() + .expect("should have parsed eids") + .len(), + 2, + "should echo both parsed EID sources" + ); + + let matched = json["ingest"]["matched"] + .as_array() + .expect("should have matched list"); + assert_eq!(matched.len(), 1, "should match only the configured partner"); + assert_eq!(matched[0]["source_domain"], "bidstream.example"); + assert_eq!(matched[0]["uid"], "uid-configured"); + + let unmatched = json["ingest"]["unmatched"] + .as_array() + .expect("should have unmatched list"); + assert_eq!(unmatched.len(), 1, "should report the unregistered source"); + assert_eq!(unmatched[0], "unknown.example"); + } + + #[test] + fn eids_lookup_reports_parse_error() { + let req = get_request_with_cookie("/_ts/admin/eids", "ts-eids=!!!not-base64!!!"); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + assert_eq!( + response.status(), + StatusCode::OK, + "malformed cookies should be reported, not errored" + ); + let json = response_json(response); + assert_eq!(json["cookie_present"], true); + assert!( + json["parse_error"] + .as_str() + .expect("should have parse_error") + .contains("ts-eids"), + "should describe the parse failure" + ); + assert!(json.get("eids").is_none(), "should omit unparsed eids"); + assert!( + json["ingest"]["matched"] + .as_array() + .expect("should have matched list") + .is_empty(), + "unparseable cookie should preview no matches" + ); + } + + #[test] + fn eids_lookup_includes_sharedid_match() { + let registry = PartnerRegistry::from_config(&[ + make_test_partner("bidstream.example", true), + make_test_partner("sharedid.org", true), + ]) + .expect("should build sharedid test registry"); + let req = get_request_with_cookie("/_ts/admin/eids", "sharedId=shared-uid-123"); + + let response = + handle_admin_eids_lookup(®istry, &req).expect("should handle eids lookup"); + + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response); + assert_eq!(json["cookie_present"], false); + assert_eq!(json["sharedid_present"], true); + + let matched = json["ingest"]["matched"] + .as_array() + .expect("should have matched list"); + assert_eq!(matched.len(), 1, "should match the sharedid partner"); + assert_eq!(matched[0]["source_domain"], "sharedid.org"); + assert_eq!(matched[0]["uid"], "shared-uid-123"); + } } diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 9f22b78e2..5003304dd 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -179,7 +179,7 @@ fn ingest_eid_cookies_with_writer( } } -fn collect_prebid_eid_updates( +pub(crate) fn collect_prebid_eid_updates( cookie_value: &str, registry: &PartnerRegistry, ) -> Vec { @@ -213,7 +213,7 @@ fn collect_prebid_eid_updates( updates } -fn dedupe_partner_updates(updates: Vec) -> Vec { +pub(crate) fn dedupe_partner_updates(updates: Vec) -> Vec { let mut latest = std::collections::BTreeMap::new(); for update in updates { latest.insert(update.partner_id, update.uid); @@ -250,7 +250,7 @@ pub fn ingest_sharedid_cookie( ingest_eid_cookies(None, Some(cookie_value), ec_id, kv, registry); } -fn collect_sharedid_update( +pub(crate) fn collect_sharedid_update( cookie_value: &str, registry: &PartnerRegistry, ) -> Option { diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index eb463acf3..1a291479e 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2211,6 +2211,7 @@ impl Settings { "/_ts/admin/keys/deactivate", "/_ts/admin/ec", "/_ts/admin/ec/{id}", + "/_ts/admin/eids", ]; /// Returns admin endpoint paths that no configured handler covers. @@ -5263,6 +5264,7 @@ origin_host_header_overide = "www.example.com""#, "/_ts/admin/keys/deactivate", "/_ts/admin/ec", "/_ts/admin/ec/{id}", + "/_ts/admin/eids", ], "should report every admin endpoint as uncovered" ); @@ -5301,6 +5303,7 @@ origin_host_header_overide = "www.example.com""#, "/_ts/admin/keys/deactivate", "/_ts/admin/ec", "/_ts/admin/ec/{id}", + "/_ts/admin/eids", ], "should detect the admin endpoints not covered by the narrow handler" ); From 8dc31cc4e4eff0a179c29368f4744e1ca7ea9636 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 18 Jul 2026 10:01:15 +0530 Subject: [PATCH 147/195] Add ISO 8601 companions to admin EC lookup timestamps Review feedback on the admin EC lookup asked for readable dates. The echoed entry now carries derived created_iso and consent.updated_iso fields (yyyy-MM-ddTHH:mm:ss.SSSZ) next to the stored unix-seconds values, which stay untouched so the echo remains faithful to KV. --- crates/trusted-server-core/src/ec/admin.rs | 49 +++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 0724d1f4b..a4e6465ab 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -55,7 +55,9 @@ struct AdminEcLookupResponse { /// (`consent.ok = false`). Absent when the body failed to parse. #[serde(skip_serializing_if = "Option::is_none")] tombstone: Option, - /// The stored entry, re-serialized verbatim. Absent when the body + /// The stored entry, re-serialized verbatim except for derived + /// `created_iso` / `updated_iso` companions added next to the stored + /// unix-seconds timestamps for readability. Absent when the body /// failed to deserialize (see `entry_error` / `raw_body`). #[serde(skip_serializing_if = "Option::is_none")] entry: Option, @@ -226,7 +228,7 @@ fn build_lookup_response( )); } } - payload.entry = Some(serde_json::to_value(&entry).expect("should serialize KvEntry")); + payload.entry = Some(entry_json_with_iso_timestamps(&entry)); } Err(error) => { payload.entry_error = Some(format!("failed to deserialize entry: {error}")); @@ -253,6 +255,37 @@ fn build_lookup_response( payload } +/// Serializes an entry, adding derived ISO 8601 companions next to the +/// stored unix-seconds timestamps (`created_iso`, `consent.updated_iso`). +/// +/// The stored numeric values stay untouched so the echo remains faithful to +/// what is in KV; the ISO fields exist purely for operator readability. +fn entry_json_with_iso_timestamps(entry: &KvEntry) -> JsonValue { + let mut entry_json = serde_json::to_value(entry).expect("should serialize KvEntry"); + + if let Some(object) = entry_json.as_object_mut() { + if let Some(iso) = iso_timestamp(entry.created) { + object.insert("created_iso".to_owned(), JsonValue::String(iso)); + } + if let Some(consent) = object.get_mut("consent").and_then(JsonValue::as_object_mut) + && let Some(iso) = iso_timestamp(entry.consent.updated) + { + consent.insert("updated_iso".to_owned(), JsonValue::String(iso)); + } + } + + entry_json +} + +/// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). +/// +/// Returns `None` for values outside the representable date range. +fn iso_timestamp(unix_seconds: u64) -> Option { + let unix_seconds = i64::try_from(unix_seconds).ok()?; + chrono::DateTime::from_timestamp(unix_seconds, 0) + .map(|datetime| datetime.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) +} + /// Derives the auction EID view for a valid entry, mirroring the filters in /// [`resolve_partner_ids`] and reporting why each stored ID was skipped. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { @@ -559,6 +592,18 @@ mod tests { json["entry"]["ids"]["bidstream.example"]["uid"], "uid-live", "should echo the stored entry verbatim" ); + assert_eq!( + json["entry"]["created"], 1_741_824_000_u64, + "should keep the stored unix-seconds timestamp" + ); + assert_eq!( + json["entry"]["created_iso"], "2025-03-13T00:00:00.000Z", + "should add an ISO 8601 companion for created" + ); + assert_eq!( + json["entry"]["consent"]["updated_iso"], "2025-03-13T00:00:00.000Z", + "should add an ISO 8601 companion for consent.updated" + ); let eids = json["auction"]["eids"] .as_array() From bc13f2773ba97e8de07f7ba6ef6f3d0a3e37427f Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:38:03 -0700 Subject: [PATCH 148/195] Upgrade EdgeZero to the deploy-actions branch Point the edgezero-* dependencies at the feature/edgezero-deploy-actions branch (PR #316) and adapt Trusted Server to its API changes: - Wire the new ts CLI subcommands surfaced by edgezero-cli: active-version, healthcheck, and rollback, plus deploy --stage and a --version flag, with argument-parsing coverage. - Migrate TrustedServerAppConfig to the AppConfigMeta::secret_fields() method that replaces the removed SECRET_FIELDS associated constant. --- Cargo.lock | 37 +++-- Cargo.toml | 12 +- crates/trusted-server-cli/src/run.rs | 174 ++++++++++++++++++++++- crates/trusted-server-core/src/config.rs | 4 +- 4 files changed, 205 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68f14e753..bd92ec5ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1449,7 +1449,7 @@ dependencies = [ "log", "serde_json", "tempfile", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", "worker", ] @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-stream", @@ -1479,14 +1479,14 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1506,14 +1506,14 @@ dependencies = [ "subtle", "thiserror 2.0.18", "toml", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "log", "proc-macro2", @@ -5074,6 +5074,19 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -5339,7 +5352,7 @@ dependencies = [ "tokio", "tokio-rustls", "toml", - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 695099d49..54569599b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,12 +53,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } env_logger = "0.11" error-stack = "0.6" fastly = "0.12" diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 1b0bdfa29..b3be949a5 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -2,8 +2,8 @@ use std::process; use clap::{Parser, Subcommand}; use edgezero_cli::args::{ - AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, - ProvisionArgs, ServeArgs, + ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, + DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; use trusted_server_core::config::TrustedServerAppConfig; @@ -13,7 +13,7 @@ use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; #[derive(Debug, Parser)] -#[command(name = "ts", about = "Trusted Server CLI")] +#[command(name = "ts", version, about = "Trusted Server CLI")] struct Args { #[command(subcommand)] command: Command, @@ -21,6 +21,8 @@ struct Args { #[derive(Debug, Subcommand)] enum Command { + /// Print the currently active deployment version for a target adapter. + ActiveVersion(ActiveVersionArgs), /// Audit a public page and write draft Trusted Server artifacts. Audit(AuditArgs), /// Sign in / out / status against an `EdgeZero` adapter. @@ -32,10 +34,14 @@ enum Command { Config(ConfigCommand), /// Deploy the project through a target adapter. Deploy(DeployArgs), + /// Probe a deployed version until it reports healthy. + Healthcheck(HealthcheckArgs), /// Trusted Server Prebid commands. Prebid(PrebidArgs), /// Provision platform resources through a target adapter. Provision(ProvisionArgs), + /// Roll a service back to a previously active deployment version. + Rollback(RollbackArgs), /// Serve the project locally through a target adapter. Serve(ServeArgs), /// Local developer tools (e.g. the macOS-only production-hostname proxy). @@ -79,6 +85,7 @@ pub fn run_from_env() -> Result<(), String> { fn dispatch(args: Args) -> Result<(), String> { match args.command { + Command::ActiveVersion(args) => edgezero_cli::run_active_version(&args), Command::Audit(args) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); @@ -102,6 +109,7 @@ fn dispatch(args: Args) -> Result<(), String> { edgezero_cli::run_config_validate_typed::(&args) } Command::Deploy(args) => edgezero_cli::run_deploy(&args), + Command::Healthcheck(args) => edgezero_cli::run_healthcheck(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -113,6 +121,7 @@ fn dispatch(args: Args) -> Result<(), String> { } } Command::Provision(args) => edgezero_cli::run_provision(&args), + Command::Rollback(args) => edgezero_cli::run_rollback(&args), Command::Serve(args) => edgezero_cli::run_serve(&args), Command::Dev(command) => crate::commands::dev::run(command), } @@ -131,6 +140,165 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn parses_active_version() { + let args = parse(&[ + "ts", + "active-version", + "--adapter", + "fastly", + "--service-id", + "service-123", + ]); + let Command::ActiveVersion(active_version) = args.command else { + panic!("expected active-version command"); + }; + assert_eq!(active_version.adapter, "fastly"); + assert_eq!(active_version.service_id, "service-123"); + } + + #[test] + fn parses_healthcheck_with_retry_defaults() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert_eq!(healthcheck.domain, "edge.example"); + assert_eq!(healthcheck.version, "7"); + assert_eq!(healthcheck.retry, 3, "should default to 3 retries"); + assert_eq!( + healthcheck.retry_delay, 5, + "should default to a 5s retry delay" + ); + assert_eq!(healthcheck.timeout, 10, "should default to a 10s timeout"); + assert!(!healthcheck.staging, "should probe production by default"); + } + + #[test] + fn parses_healthcheck_with_staging_overrides() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + "--staging", + "--retry", + "9", + "--retry-delay", + "2", + "--timeout", + "30", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert!(healthcheck.staging); + assert_eq!(healthcheck.retry, 9); + assert_eq!(healthcheck.retry_delay, 2); + assert_eq!(healthcheck.timeout, 30); + } + + #[test] + fn healthcheck_requires_domain() { + Args::try_parse_from([ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + ]) + .expect_err("should reject healthcheck without a domain"); + } + + #[test] + fn parses_rollback_with_explicit_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--rollback-to", + "7", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert_eq!(rollback.version, "8"); + assert_eq!(rollback.rollback_to, Some("7".to_owned())); + assert!(!rollback.staging); + } + + #[test] + fn parses_staging_rollback_without_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--staging", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert!(rollback.staging); + assert_eq!( + rollback.rollback_to, None, + "staging rollback should not need an explicit target" + ); + } + + #[test] + fn rollback_requires_service_id() { + Args::try_parse_from(["ts", "rollback", "--adapter", "fastly", "--version", "8"]) + .expect_err("should reject rollback without a service id"); + } + + #[test] + fn parses_deploy_with_staging_flags() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--stage", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); + }; + assert_eq!(deploy.service_id, Some("service-123".to_owned())); + assert!(deploy.stage); + } + #[test] fn parses_audit_with_default_outputs() { let args = parse(&["ts", "audit", "https://publisher.example"]); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 7bbecd747..991ed7a2f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -110,7 +110,9 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { // app-config blob. Migrating app-level secrets to `EdgeZero` secret-store // references needs nested/array extraction support and operator migration // work tracked separately. - const SECRET_FIELDS: &'static [edgezero_core::app_config::SecretField] = &[]; + fn secret_fields() -> Vec { + Vec::new() + } } /// Runs Trusted Server deploy-time validation for pushed app config. From b743345fd7d43eaf36e0fce86ba448e27e9e5b6e Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:51:39 -0700 Subject: [PATCH 149/195] Update EdgeZero to latest deploy-actions branch tip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-resolve the six edgezero-* deps from 145f1699 to bb441162 (current tip of feature/edgezero-deploy-actions, PR #316). The deploy staging flag was renamed there from --stage to --staging, standardizing on the same verb healthcheck/rollback/config-push already use; update the deploy CLI parse test to match. No production dispatch change is needed — ts passes the edgezero-cli arg structs through, so the renamed flag is picked up automatically. --- Cargo.lock | 26 +++++++++++++------------- crates/trusted-server-cli/src/run.rs | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72fa293ce..e5caa1a7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-trait", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-stream", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-trait", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "log", "proc-macro2", @@ -3604,7 +3604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -3624,7 +3624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -3637,7 +3637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -5908,7 +5908,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index b3be949a5..7374c56a7 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -290,13 +290,13 @@ mod tests { "fastly", "--service-id", "service-123", - "--stage", + "--staging", ]); let Command::Deploy(deploy) = args.command else { panic!("expected deploy command"); }; assert_eq!(deploy.service_id, Some("service-123".to_owned())); - assert!(deploy.stage); + assert!(deploy.staging); } #[test] From 44f765878c783887a37e35cea2fb756b3949c0a9 Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 12:30:16 -0500 Subject: [PATCH 150/195] Add DataDome protection decision logs --- .../src/integrations/datadome.rs | 5 +- .../src/integrations/datadome/protection.rs | 81 ++++++++++++++++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index a46f626a6..fa5c79992 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -841,9 +841,10 @@ fn build( }; log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {})", + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {})", config.sdk_origin, - config.rewrite_sdk + config.rewrite_sdk, + config.enable_protection ); Ok(Some(DataDomeIntegration::try_new(config)?)) diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index c2759a864..a7c12bbc9 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -98,7 +98,13 @@ impl DataDomeIntegration { .change_context(Self::error("Failed to call DataDome Protection API")) .map_err(ProtectionRequestError::Runtime)?; - Ok(self.classify_protection_response(platform_response.response, input.request.method())) + let status = platform_response.response.status(); + let datadome_status = datadome_response_status(platform_response.response.headers()); + let decision = + self.classify_protection_response(platform_response.response, input.request.method()); + log_protection_result(&input, status, datadome_status, &decision); + + Ok(decision) } fn is_request_protected(&self, input: &RequestFilterInput<'_>) -> bool { @@ -126,7 +132,7 @@ impl DataDomeIntegration { match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} ProtectionScopeDecision::Skip { rule_id, reason } => { - log::debug!("[datadome] Skipping Protection API for rule {rule_id} ({reason})"); + log_protection_skip(input, &rule_id, reason); return false; } } @@ -413,6 +419,77 @@ impl DataDomeIntegration { } } +fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { + if matches!( + reason, + "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" + ) { + log::info!( + "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + rule_id, + reason, + input.request.method(), + request_host(input.request), + input.request.uri().path(), + client_ip_for_log(input), + ); + } else { + log::debug!( + "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + rule_id, + reason, + input.request.method(), + request_host(input.request), + input.request.uri().path(), + client_ip_for_log(input), + ); + } +} + +fn log_protection_result( + input: &RequestFilterInput<'_>, + status: StatusCode, + datadome_status: Option, + decision: &RequestFilterDecision, +) { + let method = input.request.method(); + let host = request_host(input.request); + let path = input.request.uri().path(); + let client_ip = client_ip_for_log(input); + + match decision { + RequestFilterDecision::Respond { .. } => log::info!( + "[datadome] protection decision=blocked status={} method={} host={} path={} client_ip={} route=short_circuit", + status.as_u16(), + method, + host, + path, + client_ip, + ), + RequestFilterDecision::Continue(_) + if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => + { + log::info!( + "[datadome] protection decision=allowed status={} method={} host={} path={} client_ip={} route=continue", + status.as_u16(), + method, + host, + path, + client_ip, + ); + } + RequestFilterDecision::Continue(_) => {} + } +} + +fn client_ip_for_log(input: &RequestFilterInput<'_>) -> String { + input + .services + .client_info() + .client_ip + .map_or_else(|| "unknown".to_string(), |ip| ip.to_string()) +} + struct ProtectionPayload { fields: Vec<(String, String)>, uses_header_client_id: bool, From 302d161e32b3d8e29665d2211f36fb8f85765ebb Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 14:15:41 -0500 Subject: [PATCH 151/195] Log incoming DataDome client IP --- .../src/integrations/datadome/protection.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index a7c12bbc9..a86791110 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -36,6 +36,16 @@ impl DataDomeIntegration { &self, input: RequestFilterInput<'_>, ) -> RequestFilterDecision { + if self.config.enable_protection { + log::info!( + "[datadome] protection incoming client_ip={} method={} host={} path={}", + client_ip_for_log(&input), + input.request.method(), + request_host(input.request), + input.request.uri().path(), + ); + } + if !self.config.enable_protection || !self.is_request_protected(&input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } From aeb3f4c7e3257c6388c9036607763f528230618f Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 17:52:25 -0500 Subject: [PATCH 152/195] feat(datadome): suppress client tag for excluded IPs --- .../benches/html_processor_bench.rs | 1 + .../trusted-server-core/src/html_processor.rs | 61 +++ .../src/integrations/datadome.rs | 31 +- .../src/integrations/datadome/protection.rs | 269 ++++++++-- .../src/integrations/registry.rs | 11 +- .../src/platform/test_support.rs | 19 + crates/trusted-server-core/src/publisher.rs | 181 ++++++- docs/guide/integrations/datadome.md | 27 + ...6-08-03-datadome-ip-excluded-client-tag.md | 475 ++++++++++++++++++ ...-datadome-ip-excluded-client-tag-design.md | 339 +++++++++++++ 10 files changed, 1378 insertions(+), 36 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md create mode 100644 docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 96eec2f1f..7c1303dd4 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -13,6 +13,7 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 889234b56..4c827ace0 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -13,6 +13,7 @@ use lol_html::{ text, }; +use crate::integrations::datadome::{DATADOME_INTEGRATION_ID, DataDomeClientTagSuppressed}; use crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision; use crate::integrations::{ AttributeRewriteOutcome, IntegrationAttributeContext, IntegrationDocumentState, @@ -175,6 +176,8 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub suppress_datadome_client_side_tag: bool, } impl HtmlProcessorConfig { @@ -196,6 +199,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -223,6 +227,13 @@ impl HtmlProcessorConfig { self.gpt_diagnostics = decision; self } + + /// Attach the request-scoped `DataDome` client-tag suppression decision. + #[must_use] + pub fn with_datadome_client_tag_suppression(mut self, suppress: bool) -> Self { + self.suppress_datadome_client_side_tag = suppress; + self + } } /// Create an HTML processor with URL replacement and integration hooks. @@ -235,6 +246,9 @@ impl HtmlProcessorConfig { pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcessor { let post_processors = config.integrations.html_post_processors(); let document_state = IntegrationDocumentState::default(); + if config.suppress_datadome_client_side_tag { + document_state.get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); + } // Simplified URL patterns structure - stores only core data and generates variants on-demand struct UrlPatterns { @@ -692,6 +706,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -950,6 +965,46 @@ mod tests { assert_eq!(config.request_scheme, "https"); } + #[test] + fn suppressed_datadome_tag_is_not_injected_into_processed_html() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "test.example.com", + "https", + ) + .with_datadome_client_tag_suppression(true); + let mut processor = create_html_processor(config); + + let output = processor + .process_chunk(b"content", true) + .expect("should process HTML"); + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + + assert!( + !html.contains("window.ddjskey"), + "should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "should omit the DataDome client tag URL" + ); + } + #[test] fn test_real_publisher_html() { // Test with publisher HTML from test_publisher.html @@ -1539,6 +1594,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1613,6 +1669,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1649,6 +1706,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); // Malformed HTML with two elements (common in CMS template pages) @@ -1684,6 +1742,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1737,6 +1796,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1764,6 +1824,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index fa5c79992..db5932fb2 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -88,7 +88,12 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; -pub(super) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; + +/// Request marker indicating that Trusted Server should omit its automatic +/// `DataDome` client-side tag for the current response. +#[derive(Debug, Clone, Copy)] +pub(crate) struct DataDomeClientTagSuppressed; /// Regex pattern for matching and rewriting `DataDome` URLs in script content. /// @@ -765,7 +770,15 @@ impl IntegrationHeadInjector for DataDomeIntegration { DATADOME_INTEGRATION_ID } - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + if ctx + .document_state + .get::(DATADOME_INTEGRATION_ID) + .is_some() + { + return Vec::new(); + } + if !self.config.inject_client_side_tag || self.config.client_side_key.trim().is_empty() { return Vec::new(); } @@ -1249,6 +1262,20 @@ mod tests { #[test] fn head_injector_omits_client_side_tag_when_disabled_or_blank() { + let mut suppressed = test_config(); + suppressed.client_side_key = "test-client-key".to_string(); + let suppressed_integration = DataDomeIntegration::new(suppressed); + let suppressed_state = crate::integrations::IntegrationDocumentState::default(); + suppressed_state + .get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); + let suppressed_ctx = html_context_for_tests(&suppressed_state); + assert!( + suppressed_integration + .head_inserts(&suppressed_ctx) + .is_empty(), + "should omit the tag when the request is IP-excluded" + ); + let mut blank_key = test_config(); blank_key.client_side_key = " ".to_string(); let integration = DataDomeIntegration::new(blank_key); diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index a86791110..4c74c71e8 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -34,19 +34,18 @@ enum ProtectionRequestError { impl DataDomeIntegration { pub(super) async fn filter_protection_request( &self, - input: RequestFilterInput<'_>, + mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { if self.config.enable_protection { log::info!( - "[datadome] protection incoming client_ip={} method={} host={} path={}", - client_ip_for_log(&input), + "[datadome] protection incoming method={} host={} path={}", input.request.method(), request_host(input.request), input.request.uri().path(), ); } - if !self.config.enable_protection || !self.is_request_protected(&input) { + if !self.config.enable_protection || !self.is_request_protected(&mut input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -117,8 +116,8 @@ impl DataDomeIntegration { Ok(decision) } - fn is_request_protected(&self, input: &RequestFilterInput<'_>) -> bool { - let req = input.request; + fn is_request_protected(&self, input: &mut RequestFilterInput<'_>) -> bool { + let req = &*input.request; if req.method() == Method::OPTIONS { return false; } @@ -142,6 +141,13 @@ impl DataDomeIntegration { match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} ProtectionScopeDecision::Skip { rule_id, reason } => { + let client_tag_omitted = is_ip_exclusion_reason(reason); + if client_tag_omitted { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + } log_protection_skip(input, &rule_id, reason); return false; } @@ -210,7 +216,7 @@ impl DataDomeIntegration { input: &RequestFilterInput<'_>, server_side_key: &Redacted, ) -> ProtectionPayload { - let req = input.request; + let req = &*input.request; let client_info = input.services.client_info(); let mut fields = Vec::new(); let header_client_id = header_value(req, HEADER_DATADOME_CLIENT_ID); @@ -429,29 +435,31 @@ impl DataDomeIntegration { } } -fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { - if matches!( +fn is_ip_exclusion_reason(reason: &str) -> bool { + matches!( reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - ) { + ) +} + +fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { + if is_ip_exclusion_reason(reason) { log::info!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={} host={} path={}", rule_id, reason, input.request.method(), request_host(input.request), input.request.uri().path(), - client_ip_for_log(input), ); } else { log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={}", rule_id, reason, input.request.method(), request_host(input.request), input.request.uri().path(), - client_ip_for_log(input), ); } } @@ -465,41 +473,30 @@ fn log_protection_result( let method = input.request.method(); let host = request_host(input.request); let path = input.request.uri().path(); - let client_ip = client_ip_for_log(input); match decision { RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} host={} path={} client_ip={} route=short_circuit", + "[datadome] protection decision=blocked status={} method={} host={} path={} route=short_circuit", status.as_u16(), method, host, path, - client_ip, ), RequestFilterDecision::Continue(_) if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => { log::info!( - "[datadome] protection decision=allowed status={} method={} host={} path={} client_ip={} route=continue", + "[datadome] protection decision=allowed status={} method={} host={} path={} route=continue", status.as_u16(), method, host, path, - client_ip, ); } RequestFilterDecision::Continue(_) => {} } } -fn client_ip_for_log(input: &RequestFilterInput<'_>) -> String { - input - .services - .client_info() - .client_ip - .map_or_else(|| "unknown".to_string(), |ip| ip.to_string()) -} - struct ProtectionPayload { fields: Vec<(String, String)>, uses_header_client_id: bool, @@ -732,11 +729,17 @@ fn truncate_utf8(value: &str, limit: i32) -> String { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; - use crate::integrations::datadome::DataDomeConfig; + use crate::integrations::datadome::{ + DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, + }; + use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapSecretStore, NoopConfigStore, NoopSecretStore, build_services_with_config_and_secret, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, + build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, + noop_services_with_client_ip, }; use crate::settings::Settings; @@ -751,6 +754,210 @@ mod tests { DataDomeIntegration::try_new(config).expect("should create integration") } + fn request_for_filter() -> Request { + request_builder() + .method(Method::GET.as_str()) + .uri("https://publisher.example/page") + .body(EdgeBody::empty()) + .expect("should build filter request") + } + + fn filter_marks_request( + config: DataDomeConfig, + services: &RuntimeServices, + ) -> Request { + filter_marks_request_with_geo(config, services, None) + } + + fn filter_marks_request_with_geo( + config: DataDomeConfig, + services: &RuntimeServices, + geo_info: Option<&GeoInfo>, + ) -> Request { + let integration = + DataDomeIntegration::try_new(config).expect("should create DataDome integration"); + let settings = Settings::default(); + let mut request = request_for_filter(); + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services, + request: &mut request, + geo_info, + is_integration_route: false, + }, + )); + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an excluded request should continue without a Protection API response" + ); + request + } + + fn has_client_tag_suppression_marker(request: &Request) -> bool { + request + .extensions() + .get::() + .is_some() + } + + #[test] + fn ip_exclusions_mark_requests_for_client_tag_suppression() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let mut inline = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], + ..DataDomeConfig::default() + }; + let inline_request = + filter_marks_request(inline.clone(), &noop_services_with_client_ip(ip)); + assert!( + has_client_tag_suppression_marker(&inline_request), + "inline IP exclusions should mark the request" + ); + + inline.protection_excluded_ip_cidrs.clear(); + inline.protection_excluded_ip_cidr_sources = + vec![super::super::ProtectionIpCidrSourceConfig { + config_store: "datadome-test-source".to_string(), + key: "inline-source".to_string(), + }]; + let mut source_values = HashMap::new(); + source_values.insert("inline-source".to_string(), "192.0.2.0/24".to_string()); + let source_services = build_services_with_config_and_secret_and_client_ip( + HashMapConfigStore::new(source_values), + NoopSecretStore, + ip, + ); + let source_request = filter_marks_request(inline, &source_services); + assert!( + has_client_tag_suppression_marker(&source_request), + "Config Store IP exclusions should mark the request" + ); + + let structured_ip = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "structured-ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }], + ..DataDomeConfig::default() + }; + let structured_request = + filter_marks_request(structured_ip, &noop_services_with_client_ip(ip)); + assert!( + has_client_tag_suppression_marker(&structured_request), + "structured IP exclusions should mark the request" + ); + + let structured_source = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "structured-ip-source".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidrSource { + config_store: "datadome-test-source".to_string(), + key: "structured-source".to_string(), + }, + }], + ..DataDomeConfig::default() + }; + let mut structured_values = HashMap::new(); + structured_values.insert("structured-source".to_string(), "192.0.2.0/24".to_string()); + let structured_services = build_services_with_config_and_secret_and_client_ip( + HashMapConfigStore::new(structured_values), + NoopSecretStore, + ip, + ); + let structured_source_request = + filter_marks_request(structured_source, &structured_services); + assert!( + has_client_tag_suppression_marker(&structured_source_request), + "structured Config Store IP exclusions should mark the request" + ); + } + + #[test] + fn non_ip_exclusions_do_not_mark_requests_for_client_tag_suppression() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let cases = [DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "path".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + ..DataDomeConfig::default() + }]; + + for config in cases { + let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + assert!( + !has_client_tag_suppression_marker(&request), + "non-IP exclusions should not mark the request" + ); + } + } + + #[test] + fn asn_exclusions_do_not_mark_requests_for_client_tag_suppression() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_asns: vec![64500], + ..DataDomeConfig::default() + }; + let geo_info = GeoInfo { + city: String::new(), + country: String::new(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: Some(64500), + }; + let request = filter_marks_request_with_geo( + config, + &noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + Some(&geo_info), + ); + assert!( + !has_client_tag_suppression_marker(&request), + "ASN exclusions should not mark the request" + ); + } + + #[test] + fn non_matching_ip_does_not_mark_request_for_client_tag_suppression() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], + ..DataDomeConfig::default() + }; + let request = filter_marks_request( + config, + &noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10))), + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching IP should not mark the request" + ); + } + #[test] fn load_server_side_key_reads_secret_store() { let mut secrets = HashMap::new(); @@ -835,7 +1042,7 @@ mod tests { // the Protection API. let services = build_services_with_config_and_secret(NoopConfigStore, NoopSecretStore); let settings = Settings::default(); - let request = request_builder() + let mut request = request_builder() .method(Method::OPTIONS.as_str()) .uri("https://publisher.example/_ts/api/v1/identify") .body(EdgeBody::empty()) @@ -846,7 +1053,7 @@ mod tests { RequestFilterInput { settings: &settings, services: &services, - request: &request, + request: &mut request, geo_info: None, is_integration_route: false, }, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 0644fb522..291a54242 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -327,7 +327,7 @@ pub trait IntegrationProxy: Send + Sync { pub struct RequestFilterInput<'a> { pub settings: &'a Settings, pub services: &'a RuntimeServices, - pub request: &'a Request, + pub request: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, /// Whether the request matches a registered integration proxy route. pub is_integration_route: bool, @@ -1345,6 +1345,8 @@ mod tests { } struct EnrichingRequestFilter; + #[derive(Clone, Copy)] + struct RequestAnnotation; #[async_trait(?Send)] impl IntegrationRequestFilter for EnrichingRequestFilter { @@ -1354,8 +1356,9 @@ mod tests { async fn filter_request( &self, - _input: RequestFilterInput<'_>, + input: RequestFilterInput<'_>, ) -> Result> { + input.request.extensions_mut().insert(RequestAnnotation); Ok(RequestFilterDecision::Continue(RequestFilterEffects { request_headers: vec![HeaderMutation::set("x-datadome-isbot", "1")], response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], @@ -1487,6 +1490,10 @@ mod tests { Some("1"), "should apply DataDome-style request enrichment before routing" ); + assert!( + req.extensions().get::().is_some(), + "should preserve private request annotations for downstream routing" + ); match outcome { RequestFilterRegistryOutcome::Continue(effects) => { assert_eq!( diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 94b42161c..13b6d70cd 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -631,6 +631,25 @@ pub(crate) fn build_services_with_config_and_secret( .build() } +pub(crate) fn build_services_with_config_and_secret_and_client_ip( + config_store: impl PlatformConfigStore + 'static, + secret_store: impl PlatformSecretStore + 'static, + client_ip: IpAddr, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(config_store)) + .secret_store(Arc::new(secret_store)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: Some(client_ip), + ..ClientInfo::default() + }) + .build() +} + pub(crate) fn build_request_signing_services() -> RuntimeServices { let signing_key = SigningKey::generate(&mut OsRng); let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d3410b4ed..48e0eaacf 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -357,6 +357,7 @@ struct ProcessResponseParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, + suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, } @@ -383,6 +384,7 @@ impl PublisherBodyProcessor { integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(¶ms.ad_bids_state), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), })?) } else if is_rsc_flight { @@ -460,6 +462,7 @@ fn process_response_streaming( integration_registry: params.integration_registry, ad_slots_script: params.ad_slots_script.map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), })?; StreamingPipeline::new(config, processor) @@ -944,6 +947,7 @@ struct HtmlStreamProcessorParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option, ad_bids_state: Arc>>, + suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, } @@ -960,7 +964,8 @@ fn create_html_stream_processor( params.request_scheme, ) .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics); + .with_gpt_diagnostics(params.gpt_diagnostics) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1081,6 +1086,8 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub(crate) suppress_datadome_client_side_tag: bool, /// Request-scoped conditional diagnostics delivery decision. pub(crate) gpt_diagnostics: Option, @@ -1431,6 +1438,28 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } +/// Prevent shared caches from replaying tag-suppressed HTML to other clients. +fn apply_datadome_client_tag_cache_privacy( + response: &mut Response, + method: &Method, + suppress_datadome_client_side_tag: bool, + content_type: &str, +) { + if !suppress_datadome_client_side_tag + || !response_carries_body(method, response.status()) + || !is_html_content_type(content_type) + { + return; + } + + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + response.headers_mut().remove("surrogate-control"); + response.headers_mut().remove("fastly-surrogate-control"); +} + /// Drop a bodiless response's body and correct its framing headers. /// /// The response keeps no body, and its `Content-Length` is corrected where the @@ -1547,6 +1576,7 @@ pub fn stream_publisher_body( integration_registry, ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: ¶ms.ad_bids_state, + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.as_ref(), }; process_response_streaming(body, output, &borrowed) @@ -1640,6 +1670,7 @@ pub async fn stream_publisher_body_async( integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), }) { Ok(processor) => processor, @@ -2859,6 +2890,11 @@ pub async fn handle_publisher_request( // sets the flag unconditionally and tolerates buffered fallback): adapters // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. + let request_method = req.method().clone(); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -3051,6 +3087,12 @@ pub async fn handle_publisher_request( content_encoding ); + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &content_type, + ); let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); response.headers_mut().remove(header::CONTENT_LENGTH); @@ -3066,6 +3108,7 @@ pub async fn handle_publisher_request( content_type, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), + suppress_datadome_client_side_tag, auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, @@ -4282,6 +4325,7 @@ mod tests { dispatched_auction: None, price_granularity: Default::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -5274,6 +5318,121 @@ mod tests { ); } + #[test] + fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let mut params = make_stream_params(&settings, "identity"); + params.content_type = "text/html; charset=utf-8".to_string(); + params.suppress_datadome_client_side_tag = true; + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should process suppressed HTML"); + + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + assert!( + !html.contains("window.ddjskey"), + "publisher processing should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "publisher processing should omit the DataDome client tag URL" + ); + } + + #[test] + fn suppressed_datadome_html_is_private_and_not_shared_cached() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "suppressed HTML should be private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "suppressed HTML should not retain Surrogate-Control" + ); + assert!( + response.headers().get("fastly-surrogate-control").is_none(), + "suppressed HTML should not retain Fastly-Surrogate-Control" + ); + } + + #[test] + fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + false, + "text/html; charset=utf-8", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "unsuppressed HTML should retain its existing cache policy" + ); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/css", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "non-HTML should retain its existing cache policy" + ); + } + #[test] fn response_carries_body_preserves_bodiless_metadata() { // A processable GET 200 buffers a body and recomputes Content-Length. @@ -6229,6 +6388,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6277,6 +6437,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6314,6 +6475,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -6429,6 +6591,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -6482,6 +6645,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6538,6 +6702,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6594,6 +6759,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6650,6 +6816,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6694,6 +6861,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -6888,6 +7056,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -6952,6 +7121,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -7015,6 +7185,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -7071,6 +7242,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7207,6 +7379,7 @@ mod tests { dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -7558,6 +7731,7 @@ mod tests { )), price_granularity: PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } }; let make_stream_response = || PublisherResponse::Stream { @@ -7737,6 +7911,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7804,6 +7979,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -7854,6 +8030,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -7962,6 +8139,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -8019,6 +8197,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 9c342679b..1743a1b75 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -173,6 +173,33 @@ Static assets are excluded by default using a case-insensitive file-extension re Auction traffic at `/auction` is protected by default. +### IP-excluded client-side tag behavior + +On the Fastly adapter, a request that matches an IP-based DataDome exclusion +also omits Trusted Server's automatically injected client-side DataDome tag +from processed HTML. This keeps the client-side layer consistent with the +server-side Protection API skip. + +This behavior applies to: + +- `protection_excluded_ip_cidrs`; +- `protection_excluded_ip_cidr_sources`; +- structured `ip_cidr` rules; and +- structured `ip_cidr_source` rules. + +ASN, method, path, query-parameter, static-asset, and internal-route +exclusions do not automatically suppress the client-side tag. DataDome tags +already present in publisher HTML are not removed or changed by this behavior, +and `/integrations/datadome/tags.js` remains available when requested directly. + +Because the processed HTML differs by client IP, tag-suppressed HTML is marked +`private, max-age=0` and removed from shared surrogate caches. The decision is +reported in the existing protection log, for example: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +``` + ### Structured exclusion rules Use structured rules for all DataDome protection exclusions. Each rule has an `id`, optional `methods`, and a typed matcher. The default configuration includes a `path_regex` rule for common static assets. diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md new file mode 100644 index 000000000..5ea5f6ff3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -0,0 +1,475 @@ +# DataDome IP-excluded client tag suppression — Implementation Plan + +> **Status:** Approved for implementation +> +> **For implementers:** Work task by task and keep the workspace buildable. +> Follow `CLAUDE.md`: use target-matched Cargo aliases, do not use bare +> workspace tests, and do not add an internal HTTP header for request state. + +**Goal:** When Fastly's authoritative client IP matches a DataDome IP exclusion, +skip the Protection API call and omit only Trusted Server's automatically +injected DataDome client tag from every processed HTML response. + +**Issue:** #994 +**Design:** +`docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md` + +## Approved behavior + +| Request condition | Protection API | Trusted Server auto-injected tag | Publisher-originated tag | +| ------------------------------------------------------------- | --------------- | -------------------------------- | ------------------------ | +| Inline IP CIDR match | Skipped | Omitted | Unchanged | +| Config Store IP CIDR-source match | Skipped | Omitted | Unchanged | +| Structured `ip_cidr` match | Skipped | Omitted | Unchanged | +| Structured `ip_cidr_source` match | Skipped | Omitted | Unchanged | +| ASN, method, path, query, static, or internal-route exclusion | Skipped | Preserved | Unchanged | +| No exclusion match | Called normally | Preserved | Unchanged | +| Protection API fail-open | Continued | Preserved | Unchanged | + +The Fastly-only scope means that other adapters receive the default +non-suppressed value. Do not add a configuration option and do not modify their +request-filter wiring. + +## Runtime contracts + +1. **Trusted identity source:** determine exclusion from + `RuntimeServices::client_info().client_ip`, never a caller-provided header. +2. **Single evaluation:** use the existing `ProtectionScope` decision. Do not + evaluate CIDRs a second time while injecting HTML; this avoids diverging + Config Store/cache behavior. +3. **Private marker:** communicate the decision with a typed request extension, + never a request/response header. The marker cannot leak to the origin or + client. +4. **Precise scope:** tag suppression is keyed only on decision reasons + `client_ip`, `client_ip_source`, `ip_cidr`, and `ip_cidr_source`. +5. **Cache safety:** an HTML response with the tag omitted differs by client IP. + A suppressed processed HTML response must be `private, max-age=0` and have + `Surrogate-Control` and `Fastly-Surrogate-Control` removed. Do not alter + cache headers when the response is not processed HTML, because this feature + does not alter that body. +6. **No behavior drift:** DataDome proxy endpoints, response-header effects, + `rewrite_sdk`, and DataDome tags that were already in origin HTML retain + their current behavior. + +## File map + +| File | Change | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/registry.rs` | Permit filters to attach private typed request extensions while retaining header-effect semantics. Extend the HTML context with the propagated boolean. | +| `crates/trusted-server-core/src/integrations/datadome.rs` | Define the crate-private marker and have the head injector honor the HTML-context flag. | +| `crates/trusted-server-core/src/integrations/datadome/protection.rs` | Recognize IP scope skips, attach the marker, and add `client_tag=omitted` to the existing info log. | +| `crates/trusted-server-core/src/html_processor.rs` | Carry the per-response suppression boolean from config to all integration HTML contexts. | +| `crates/trusted-server-core/src/publisher.rs` | Snapshot the marker before origin dispatch, propagate it through every HTML streaming path, and apply cache privacy to suppressed processed HTML. | +| `docs/guide/integrations/datadome.md` | Document the Fastly IP-exclusion behavior and its limits. | +| `docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md` | Already updated with the cache-variance safeguard. | + +No changes are expected in `trusted-server.example.toml`, JavaScript bundles, +or non-Fastly adapters. + +--- + +## Task 1: Make the request-filter input capable of private annotations + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: its existing `#[cfg(test)]` module + +The current `RequestFilterInput` holds `&Request`. Change it to hold +`&mut Request` so a request filter can add a typed extension. This is +the narrowest safe transport because the registry already has exclusive mutable +access to the request while it invokes each filter. + +- [ ] **Step 1: Add a regression test for an extension-producing filter.** Create + a test-only zero-sized marker and filter that writes it to + `input.request.extensions_mut()`. Run `IntegrationRegistry::filter_request` + and assert the original mutable request has the marker afterward. In the + same test, verify normal `RequestFilterEffects` still apply their request + header mutation and return their response header mutation. +- [ ] **Step 2: Change `RequestFilterInput::request` to a mutable borrow.** Keep + the `IntegrationRequestFilter` method signature and `RequestFilterEffects` + unchanged. +- [ ] **Step 3: Update `IntegrationRegistry::filter_request`.** Pass its existing + `&mut Request` directly to each `RequestFilterInput`. Keep the ordering: + filter mutation first, then registry-applied request-header effects, then + the next filter. +- [ ] **Step 4: Update all direct filter tests and test filters.** Calls that build + `RequestFilterInput` must construct a mutable request and pass + `request: &mut request`. Read-only filters should continue to compile by + simply not mutating the request. +- [ ] **Step 5: Run focused tests.** + +```bash +cargo test-fastly integrations::registry +``` + +**Acceptance:** a filter can retain a typed marker for downstream route handling +without emitting a synthetic `x-*` header, and existing header effects retain +their behavior. + +--- + +## Task 2: Mark IP-based DataDome exclusions and log the outcome + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/datadome.rs` +- Modify: `crates/trusted-server-core/src/integrations/datadome/protection.rs` +- Test: `crates/trusted-server-core/src/integrations/datadome/protection.rs` +- Reuse: `crates/trusted-server-core/src/integrations/datadome/protection_scope.rs` + +- [ ] **Step 1: Add a crate-private marker in `datadome.rs`.** Define a + zero-sized type with a behavior-oriented name, such as + `DataDomeClientTagSuppressed`. It must be visible to `publisher.rs` and + `protection.rs` through `pub(crate)`, but must not be exported as public + integration configuration or API. +- [ ] **Step 2: Add an IP-reason predicate beside protection logging.** Centralize + the exact four eligible scope reasons in one helper: + +```rust +matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source") +``` + + Do not infer eligibility from rule ID: Config Store source rule IDs are + operator-configured strings. + +- [ ] **Step 3: Make `filter_protection_request` own a mutable input and pass it + mutably to `is_request_protected`.** In the existing + `ProtectionScopeDecision::Skip` arm: + + 1. determine whether the reason is IP-based; + 2. if so, insert the typed marker into `input.request.extensions_mut()`; + 3. call the updated skip logger with `client_tag_omitted = true`; and + 4. return `false` exactly as today so the Protection API is not called. + + Do not set the marker for the early method/integration/internal-route + returns. Do not set it when the API call returns a fail-open error. + +- [ ] **Step 4: Update `log_protection_skip`.** Keep IP exclusions at `info` and + non-IP exclusions at `debug`. For the IP branch, extend the existing + structured text after the reason with `client_tag=omitted`; retain rule, + reason, method, host, and path, but do not include the client IP. The + desired shape is: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +``` + +- [ ] **Step 5: Add filter-level marker tests.** Add small helpers in the + protection test module to build `RuntimeServices` with a fixed client IP, + optional Config Store data, and a mutable request. For each case, call + `filter_protection_request`, assert it returns `Continue`, and inspect the + request extension: + + - inline `protection_excluded_ip_cidrs` match → marker present; + - `protection_excluded_ip_cidr_sources` match → marker present; + - structured `ProtectionMatcherConfig::IpCidr` match → marker present; + - structured `ProtectionMatcherConfig::IpCidrSource` match → marker present. + + Clear the process-global CIDR-source test cache before and after source + tests so cached values cannot affect another case. + +- [ ] **Step 6: Add negative filter-level tests.** Assert the marker is absent + for a non-matching IP, a configured ASN match, a structured path match, + a structured query match, an excluded method, and an internal/integration + route. Reuse the existing `ProtectionScope` unit tests for matching + semantics; these new tests verify only the new side effect. +- [ ] **Step 7: Preserve API-call behavior.** For an IP marker test, use an HTTP + client double that records calls or errors if called. Assert no Protection + API request is sent. This protects against accidentally marking a request + while still invoking DataDome. +- [ ] **Step 8: Run focused tests.** + +```bash +cargo test-fastly datadome::protection +cargo test-fastly datadome::protection_scope +``` + +**Acceptance:** only the four IP decision reasons add the private marker and +produce the augmented informational skip log; all other exclusion and fail-open +paths keep their current tag behavior. + +--- + +## Task 3: Thread suppression through publisher response processing + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/html_processor.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: `publisher.rs` and `html_processor.rs` test modules + +### Data flow to implement + +```text +DataDomeClientTagSuppressed request extension + -> bool captured by handle_publisher_request before origin dispatch + -> OwnedProcessResponseParams + -> ProcessResponseParams / HtmlStreamProcessorParams + -> HtmlProcessorConfig + -> IntegrationHtmlContext + -> DataDomeIntegration::head_inserts +``` + +- [ ] **Step 1: Capture the marker once in `handle_publisher_request`.** Read + `req.extensions().get::().is_some()` before + `req` is rewritten and moved into `PlatformHttpRequest`. Store the boolean + only in the `PublisherResponse::Stream` parameters, because that is the + only response route that passes through HTML injection. +- [ ] **Step 2: Add a boolean to the owned and borrowed publisher-processing + parameter structs.** Add a clearly named field such as + `suppress_datadome_client_side_tag` to: + + - `OwnedProcessResponseParams`; + - `ProcessResponseParams`; and + - `HtmlStreamProcessorParams`. + + Pass it through all three existing HTML construction sites: + + - `PublisherBodyProcessor::new` for async buffered processing; + - `process_response_streaming` for synchronous processing; and + - `stream_publisher_body_async` for the Fastly streaming auction-hold path. + + Every test fixture that constructs `OwnedProcessResponseParams` directly + must set `false` unless it explicitly exercises suppression. + +- [ ] **Step 3: Extend `HtmlProcessorConfig`.** Add the same boolean, default it + to `false` in `from_settings`, and add a narrow builder method used by + `create_html_stream_processor`. Update direct `HtmlProcessorConfig` + fixtures and the benchmark fixture to set `false` explicitly. +- [ ] **Step 4: Extend `IntegrationHtmlContext`.** Add the boolean as immutable + request-scoped context. Populate it at both construction sites in + `html_processor.rs`: + + - the streaming `` element handler; and + - `HtmlWithPostProcessing::process_chunk` for full-document post-processors. + + Update every test helper that constructs `IntegrationHtmlContext` to set + `false` by default. + +- [ ] **Step 5: Add plumbing tests.** + + - `HtmlProcessorConfig::from_settings` defaults to non-suppressed. + - A test head injector records the context flag and sees `true` when a config + is built with suppression. + - A `publisher.rs` route test inserts the DataDome marker into a request, + receives a processable HTML `PublisherResponse::Stream`, and verifies the + owned parameters carry `true`. + - A buffered and a streaming-body path both preserve `true` to head injection. + +- [ ] **Step 6: Run focused tests.** + +```bash +cargo test-fastly html_processor +cargo test-fastly publisher +``` + +**Acceptance:** the decision is read once from a private request extension and +is available to every head injector for every processed HTML response, including +Fastly's streaming path. + +--- + +## Task 4: Omit only Trusted Server's injected DataDome tag + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/datadome.rs` +- Test: `crates/trusted-server-core/src/integrations/datadome.rs` +- Test: `crates/trusted-server-core/src/html_processor.rs` or `publisher.rs` + +- [ ] **Step 1: Add a direct head-injector regression test.** With a client-side + key configured and `ctx.suppress_datadome_client_side_tag = true`, assert + `head_inserts()` returns an empty vector. The same config with `false` + must still return exactly one snippet containing both `window.ddjskey` and + the configured tag URL. +- [ ] **Step 2: Implement the guard as the first condition in + `DataDomeIntegration::head_inserts`.** Return an empty vector when the + context flag is true; otherwise retain all current serialization, + escaping, blank-key, and `inject_client_side_tag` behavior unchanged. +- [ ] **Step 3: Add an end-to-end HTML pipeline test.** Configure the DataDome + integration with a client-side key, process representative HTML with + suppression enabled, and assert the result contains neither: + +```text +window.ddjskey= +/integrations/datadome/tags.js +``` + + Repeat with suppression disabled and assert both appear. + +- [ ] **Step 4: Pin publisher-originated-tag behavior.** Feed origin HTML that + contains a DataDome `tags.js` element. With suppression enabled, assert + that element remains in output and is rewritten by `rewrite_sdk` exactly + as before. This distinguishes automatic injection from origin markup. +- [ ] **Step 5: Pin direct route behavior.** Retain or add a DataDome proxy test + showing that `GET /integrations/datadome/tags.js` remains registered and + fetches/proxies the SDK normally; suppression affects only HTML injection. +- [ ] **Step 6: Run focused tests.** + +```bash +cargo test-fastly datadome +``` + +**Acceptance:** suppression removes only the generated configuration/script +pair; nothing removes publisher markup or disables DataDome endpoints. + +--- + +## Task 5: Make tag-suppressed processed HTML private + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +The automatic tag makes the processed HTML vary by client IP. Cache privacy is +therefore a correctness and protection requirement, not an optional +optimization. + +- [ ] **Step 1: Add a failing cache-privacy test.** Build a `PublisherResponse` + with a processable HTML content type, suppression `true`, and cacheable + origin headers (`Cache-Control`, `Surrogate-Control`, and + `Fastly-Surrogate-Control`). Assert the stream response is: + + - `Cache-Control: private, max-age=0`; and + - missing both surrogate cache headers. + +- [ ] **Step 2: Apply privacy only in the `ResponseRoute::Stream` HTML arm.** + After response classification confirms a processable HTML stream, use the + existing per-user ad-stack policy as the model. Do not alter cache headers + for CSS, RSC, non-processable pass-through, unsupported encodings, HEAD, + 204/205/304, or responses without suppression: none has a body variation + created by this feature. +- [ ] **Step 3: Add non-regression cache tests.** Verify that: + + - non-suppressed processed HTML keeps its existing cache headers unless + another existing policy changes them; + - a suppressed CSS/non-HTML stream is not made private by this feature; and + - existing ad-stack privacy behavior remains unchanged when both features are + active. + +- [ ] **Step 4: Run focused tests.** + +```bash +cargo test-fastly publisher +``` + +**Acceptance:** a shared cache cannot replay an IP-excluded client's tagless +HTML to a non-excluded visitor, while unchanged responses retain their existing +cacheability. + +--- + +## Task 6: Add Fastly-path regression coverage + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` tests only if the + existing dispatch helpers can exercise the DataDome registry with a stubbed + publisher response. +- Otherwise, document the existing core filter + publisher pipeline tests as + the executable behavioral coverage; do not refactor Fastly production code + merely to enable a duplicate test. + +- [ ] **Step 1: Extend the existing Fastly request-filter dispatch regression + test or add a focused equivalent.** Configure a DataDome request filter, + insert trusted `ClientInfo` into the request extensions with a matching + IP, and confirm the filter runs before publisher routing. +- [ ] **Step 2: Assert that the routed request retains the private DataDome + marker.** The assertion must inspect request extensions or the processed + HTML result, not an HTTP header. +- [ ] **Step 3: Ensure no actual DataDome API call occurs for the matching IP.** + Use a recording/failing HTTP client or the existing Fastly test seam. +- [ ] **Step 4: Add the non-matching counterpart.** It must not receive the + marker and must continue to inject the configured tag when HTML is + processed. +- [ ] **Step 5: Run Fastly adapter tests.** + +```bash +cargo test-fastly +``` + +**Acceptance:** the production adapter's actual filter ordering preserves the +marker from authoritative Fastly client metadata through publisher HTML +processing. If the current test seam cannot stub a full origin response, retain +this as focused request-filter-order coverage and rely on Task 3's core +pipeline tests for body output rather than expanding adapter production code. + +--- + +## Task 7: Document operator-visible behavior + +**Files:** + +- Modify: `docs/guide/integrations/datadome.md` +- Do not modify: `trusted-server.example.toml` + +- [ ] **Step 1: Add a subsection adjacent to “Protected traffic” or “Client-side + setup.”** State that, on Fastly, an IP exclusion skips the Protection API + and suppresses only Trusted Server's automatic DataDome tag injection on + processed HTML. +- [ ] **Step 2: List the four covered IP sources.** Use the exact configuration + names and structured rule types. +- [ ] **Step 3: State the exclusions that do not suppress the client tag.** ASN, + method, path, query, static-asset, and internal-route exclusions retain + normal auto-injection. +- [ ] **Step 4: State the limits.** Publisher-originated/manual tags are not + removed; `/integrations/datadome/tags.js` remains available; no new + configuration is required; and tag-suppressed processed HTML is private + to prevent shared-cache replay. +- [ ] **Step 5: Add the diagnostic example.** Use an example-only host/IP and + include `client_tag=omitted` with rule and reason. +- [ ] **Step 6: Format-check the changed documentation.** + +```bash +cd docs +npx prettier --check guide/integrations/datadome.md \ + superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md \ + superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +``` + +**Acceptance:** operators can predict exactly when the tag will be omitted and +understand that this is an IP-based Fastly behavior, not a general exclusion +side effect. + +--- + +## Final verification + +- [ ] Confirm the working tree contains only the intended core, Fastly-test, + guide, spec, and plan changes. +- [ ] Run formatting. + +```bash +cargo fmt --all -- --check +``` + +- [ ] Run the relevant target-matched test suites. + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +``` + +- [ ] Run required lint suites. + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +- [ ] Run the docs check from Task 7. +- [ ] Review the diff for accidental exposure of the marker as a request or + response header, duplicate CIDR evaluation, unintended publisher-tag + removal, or shared-cacheable tag-suppressed HTML. + +## Deferred acceptance + +Do **not** perform live production/browser verification in this change. After +deployment, the separate testing workflow should verify that a matching +whitelisted IP receives processed HTML without Trusted Server's +`/integrations/datadome/tags.js` injection, while an unlisted IP retains it. diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md new file mode 100644 index 000000000..c48ad2fda --- /dev/null +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -0,0 +1,339 @@ +# DataDome IP-excluded client tag suppression + +**Issue:** #994 +**Date:** 2026-08-03 +**Status:** Proposed + +## Problem + +Trusted Server has two DataDome protection layers: + +1. Server-side Protection API validation, which can be skipped for configured + client IP CIDRs. +2. Client-side tag auto-injection, which adds `window.ddjskey`, + `window.ddoptions`, and the configured `tags.js` script to processed HTML. + +When a request matches an IP-based server-side exclusion, the Protection API is +skipped, but the client-side tag is currently still injected. The browser can +therefore continue running client-side DataDome protection for a request that +was explicitly whitelisted at Trusted Server. + +The desired behavior is that Fastly requests skipped by an IP-based DataDome +exclusion also omit Trusted Server's automatically injected client-side tag. + +## Goals + +- Suppress Trusted Server's automatically injected DataDome client-side tag for + Fastly requests skipped by an IP-based protection exclusion. +- Reuse the existing authoritative protection-scope decision. +- Cover all supported IP-based exclusion mechanisms: + - `protection_excluded_ip_cidrs` + - `protection_excluded_ip_cidr_sources` + - structured `ip_cidr` rules + - structured `ip_cidr_source` rules +- Preserve current behavior for non-IP exclusions. +- Leave publisher-originated or manually configured DataDome tags untouched. +- Add an informational diagnostic indicating that the client tag is omitted. +- Keep the implementation independent of caller-supplied IP headers. +- Prevent a shared cache from replaying IP-specific, tag-suppressed HTML to + non-excluded visitors. + +## Non-goals + +- Do not add a configuration flag or make this behavior opt-in. +- Do not change Axum, Cloudflare, or Spin request-filter wiring. This behavior + is intentionally scoped to the Fastly adapter, where the DataDome server-side + request filter is currently run. +- Do not suppress DataDome tags that originate in publisher HTML. +- Do not remove or disable the `/integrations/datadome/tags.js` route. +- Do not change DataDome signal-collection proxy behavior. +- Do not change ASN, path, query-parameter, method, static-asset, or internal + route exclusions. +- Do not perform live production verification as part of implementation. + +## Confirmed decisions + +1. **Adapter scope:** Fastly only. +2. **IP scope:** all four IP-based exclusion mechanisms listed above. +3. **Tag scope:** Trusted Server's auto-injected tag only. +4. **HTML scope:** every HTML response that enters the existing HTML processing + pipeline. +5. **Logging:** enrich the existing IP-exclusion skip log with + `client_tag=omitted`, including the matched rule and reason. +6. **Live testing:** deferred until after implementation and deployment/testing + workflow review. + +## Current architecture + +### Server-side protection + +`DataDomeIntegration::is_request_protected()` in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` evaluates +method, internal-route, ASN, IP, and structured exclusion conditions. It uses +the client IP from `RuntimeServices::client_info()`, which is populated from +trusted Fastly request metadata. It does not use a caller-supplied IP header. + +The current function reduces the protection-scope result to a boolean. For an +IP exclusion it logs the skip and returns `false`, causing the request filter to +continue without calling the Protection API. + +The Fastly EdgeZero fallback path runs this request filter before route +selection and publisher proxying. The request continues into +`handle_publisher_request()` after the filter returns a continue decision. + +### Client-side injection + +`DataDomeIntegration::head_inserts()` in +`crates/trusted-server-core/src/integrations/datadome.rs` emits the client-side +snippet when: + +- `inject_client_side_tag` is true; and +- `client_side_key` is non-empty. + +The injector currently receives `IntegrationHtmlContext`, which contains HTML +host/scheme and document state but no request IP or protection decision. + +The publisher response path carries request-specific values through: + +```text +Request + -> OwnedProcessResponseParams + -> HtmlStreamProcessorParams + -> HtmlProcessorConfig + -> IntegrationHtmlContext + -> IntegrationHeadInjector +``` + +The existing DataDome attribute rewriter separately rewrites DataDome URLs +found in publisher HTML. That behavior must remain unchanged. + +## Design + +### 1. Capture an IP-exclusion marker at the request filter + +The request filter must attach a typed, internal request-scoped marker when the +existing protection-scope evaluation returns a skip for one of these reasons: + +- `client_ip` +- `client_ip_source` +- `ip_cidr` +- `ip_cidr_source` + +The marker must be attached only after the existing scope decision confirms the +IP exclusion. It must not be inferred from request headers or recomputed later +in the HTML pipeline. + +The request-filter API currently exposes an immutable request view. Add the +smallest internal mechanism needed for a filter to attach a typed request +extension without introducing a caller-visible header. Header mutations should +continue to use `RequestFilterEffects` as they do today. + +The marker should be a zero-sized or otherwise minimal internal type. It only +needs to answer whether Trusted Server's DataDome client tag should be +suppressed; the existing skip log supplies the rule ID and reason. + +The marker must not be attached for: + +- `OPTIONS` or other excluded methods before scope evaluation; +- internal or integration routes; +- ASN exclusions; +- path, query, or other non-IP structured exclusions; +- unmatched IP rules; +- Protection API fail-open behavior; or +- requests where `enable_protection` is false and the request filter does not + run. + +### 2. Enrich the existing skip log + +For IP-based skips, extend the existing informational log with +`client_tag=omitted`: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +``` + +The existing rule ID, reason, and request metadata remain part of the log. +Client IP values are not included. Non-IP skip logs retain their current +behavior and level. + +This log represents the request policy decision. It may also apply to a +non-HTML response, for which no HTML tag would have been injected anyway. + +### 3. Propagate the marker into HTML processing + +Before the publisher request is moved into the platform HTTP client, snapshot +whether the request carries the marker. Carry that request-scoped boolean +through `OwnedProcessResponseParams`, `HtmlStreamProcessorParams`, and +`HtmlProcessorConfig`. + +The value should default to `false` in all existing constructors and direct +unit-test fixtures. Non-Fastly adapters will naturally retain the default +because they do not currently produce the Fastly request-filter marker. + +Expose the value to head injectors through the existing HTML processing context +or equivalent request-scoped integration context. The propagation must work for +both: + +- the normal buffered HTML path; and +- the streaming HTML path, including the auction-hold path. + +The value is irrelevant for non-HTML, RSC, pass-through, and unmodified +responses, which should retain their current processing. + +### 4. Keep IP-specific HTML out of shared cache + +A processed HTML response differs by client IP when the generated tag is +suppressed. In the `PublisherResponse::Stream` path, when suppression is active +and the response is HTML, set `Cache-Control: private, max-age=0` and remove +`Surrogate-Control` and `Fastly-Surrogate-Control` before the body is streamed. + +This matches the existing per-user ad-stack cache policy. It prevents Fastly or +another shared cache from replaying a tag-suppressed response to a visitor whose +IP does not match an exclusion. Do not change cache headers for non-HTML, +pass-through, or unmodified responses because their output does not vary by this +feature. + +### 5. Suppress only the generated DataDome snippet + +At the start of `DataDomeIntegration::head_inserts()`: + +1. Check the request-scoped suppression marker. +2. If present, return no DataDome head inserts. +3. Otherwise preserve the current `inject_client_side_tag` and + `client_side_key` checks and emit the existing snippet unchanged. + +When suppression is active, omit both: + +```html + + +``` + +Do not alter: + +- publisher-originated DataDome script tags; +- `rewrite_sdk` behavior; +- the DataDome SDK proxy route; +- the signal collection API proxy; +- DataDome configuration serialization for non-suppressed requests; or +- injection behavior for requests without the marker. + +## Testing plan + +### Protection-filter tests + +Add or extend tests in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` to verify +that the marker is attached for: + +- a matching inline IPv4 CIDR; +- a matching Config Store-backed CIDR source; +- a matching structured `ip_cidr` rule; and +- a matching structured `ip_cidr_source` rule. + +Verify that the marker is absent for: + +- a non-matching IP; +- an ASN exclusion; +- a path exclusion; +- a query-parameter exclusion; +- an excluded method; and +- an internal or integration route. + +Verify the existing protection behavior remains unchanged: IP-matched requests +continue without a Protection API call. + +### Head-injector tests + +Add tests in +`crates/trusted-server-core/src/integrations/datadome.rs` verifying that: + +- a configured client tag is omitted when suppression is active; +- a configured client tag is emitted when suppression is inactive; +- a blank client-side key remains a no-op; and +- `inject_client_side_tag = false` remains a no-op. + +### HTML pipeline tests + +Add coverage for the request-scoped value flowing through the HTML processor, +including the streaming path. Confirm that a suppressed processed HTML response +contains neither the injected `window.ddjskey` configuration nor the configured +DataDome `tags.js` script. For a suppressed HTML stream, assert the response is +private and has no surrogate cache headers. Confirm a non-suppressed HTML stream +retains its origin cache behavior. + +Confirm that publisher-originated DataDome tags remain in the output and are +still rewritten according to the existing `rewrite_sdk` behavior. + +### Fastly dispatch tests + +Add a Fastly adapter dispatch test with: + +- DataDome protection enabled; +- a client IP matching an inline exclusion; +- a configured client-side key; and +- an HTML publisher response. + +The test should verify that the request continues without a Protection API +call, the response includes the `client_tag=omitted` decision log through the +existing test logging seam where available, and the generated tag is absent. + +Also cover a non-excluded request to confirm the generated tag remains present. + +## Documentation changes + +Update `docs/guide/integrations/datadome.md` to state that IP-excluded Fastly +requests skip both: + +- server-side Protection API validation; and +- Trusted Server's automatic client-side tag injection. + +Document that this does not remove or disable publisher-originated DataDome +tags, and that non-IP exclusions do not automatically suppress the client-side +tag. + +No configuration template changes are required because this behavior has no +new setting. + +## Files expected to change + +- `crates/trusted-server-core/src/integrations/registry.rs` + - Support the internal request-scoped annotation mechanism. +- `crates/trusted-server-core/src/integrations/datadome.rs` + - Define the marker and conditionally suppress head injection. +- `crates/trusted-server-core/src/integrations/datadome/protection.rs` + - Attach the marker for IP-based scope skips and enrich the skip log. +- `crates/trusted-server-core/src/integrations/registry.rs` or the relevant + HTML context definition + - Carry the suppression decision into head injection. +- `crates/trusted-server-core/src/html_processor.rs` + - Carry the request-scoped value into HTML integration context. +- `crates/trusted-server-core/src/publisher.rs` + - Snapshot and propagate the request marker through response processing. +- `docs/guide/integrations/datadome.md` + - Document the behavior. +- Relevant unit and Fastly adapter test modules. + +The exact split between registry request annotations and HTML context plumbing +should remain minimal and should not introduce a new public configuration API. + +## Verification + +Implementation verification should use the repository's target-matched +commands: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +No live production validation is required for this implementation task. Live +browser verification will be performed later through the deployment/testing +workflow. From fc963341a085da137212e5e2ee83ec07c462a082 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 5 Aug 2026 19:49:24 -0500 Subject: [PATCH 153/195] Add DataDome staging test bypass --- .../src/integrations/datadome.rs | 128 +++++++++++++- .../src/integrations/datadome/protection.rs | 161 +++++++++++++++++- docs/guide/integrations/datadome.md | 52 ++++-- 3 files changed, 328 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index db5932fb2..4dd55b28f 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -61,7 +61,7 @@ use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; -use http::{Method, StatusCode}; +use http::{HeaderName, Method, StatusCode}; use regex::Regex; use serde::Deserialize; use serde_json::Value as JsonValue; @@ -77,6 +77,7 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -117,6 +118,27 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { .expect("DataDome URL rewrite regex should compile") }); +/// Temporary static-header bypass for server-side `DataDome` protection. +/// +/// This is intended only for an access-controlled staging environment. A +/// matching header bypasses the server-side Protection API and is removed +/// before the publisher origin receives the request. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProtectionTestBypassConfig { + /// Enables the bypass. Defaults to disabled when the section is present. + #[serde(default)] + pub enabled: bool, + + /// Header name carrying the temporary bypass credential. + #[serde(default)] + pub header_name: String, + + /// Static credential expected in [`Self::header_name`]. + #[serde(default)] + pub credential: Redacted, +} + /// Configuration for `DataDome` integration. #[derive(Debug, Clone, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -199,6 +221,10 @@ pub struct DataDomeConfig { )] pub protection_exclusion_rules: Vec, + /// Temporary static-header bypass for access-controlled staging tests. + #[serde(default)] + pub protection_test_bypass: Option, + /// Reserved flag for future GraphQL payload extraction. #[serde(default)] pub enable_graphql_support: bool, @@ -329,6 +355,7 @@ impl Default for DataDomeConfig { protection_excluded_ip_cidr_sources: Vec::new(), protection_ip_list_cache_ttl_seconds: default_protection_ip_list_cache_ttl_seconds(), protection_exclusion_rules: default_protection_exclusion_rules(), + protection_test_bypass: None, enable_graphql_support: false, client_side_key: String::new(), inject_client_side_tag: default_inject_client_side_tag(), @@ -362,6 +389,9 @@ impl DataDomeIntegration { config.server_side_key_secret_name = config.server_side_key_secret_name.trim().to_string(); config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); + if let Some(bypass) = &mut config.protection_test_bypass { + bypass.header_name = bypass.header_name.trim().to_string(); + } if config.enable_protection { if config.server_side_key_secret_store.is_empty() @@ -373,6 +403,7 @@ impl DataDomeIntegration { } Self::validate_protection_api_origin(&config.protection_api_origin)?; } + Self::validate_protection_test_bypass(&config)?; if config.inject_client_side_tag { Self::validate_client_side_tag_url(&config.client_side_tag_url)?; @@ -422,6 +453,36 @@ impl DataDomeIntegration { Ok(()) } + fn validate_protection_test_bypass( + config: &DataDomeConfig, + ) -> Result<(), Report> { + let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return Ok(()); + }; + + if !config.enable_protection { + return Err(Report::new(Self::error( + "protection_test_bypass requires enable_protection to be true", + ))); + } + if HeaderName::from_bytes(bypass.header_name.as_bytes()).is_err() { + return Err(Report::new(Self::error( + "protection_test_bypass.header_name must be a valid HTTP header name", + ))); + } + if bypass.credential.expose().is_empty() { + return Err(Report::new(Self::error( + "protection_test_bypass.credential must not be empty when enabled", + ))); + } + + Ok(()) + } + fn validate_client_side_tag_url(tag_url: &str) -> Result<(), Report> { if tag_url.starts_with('/') && !tag_url.starts_with("//") { if tag_url.chars().any(is_unsafe_client_side_tag_path_char) { @@ -1098,6 +1159,71 @@ mod tests { config.server_side_key_secret_name, "datadome_server_side_key" ); + assert!( + config.protection_test_bypass.is_none(), + "the temporary test bypass should be disabled by default" + ); + } + + #[test] + fn protection_test_bypass_deserializes_nested_configuration() { + let config: DataDomeConfig = toml::from_str( + r#" + enabled = true + enable_protection = true + + [protection_test_bypass] + enabled = true + header_name = "x-ts-datadome-test-bypass" + credential = "temporary-test-credential" + "#, + ) + .expect("should deserialize DataDome test bypass configuration"); + let bypass = config + .protection_test_bypass + .expect("should deserialize the nested test bypass configuration"); + + assert!(bypass.enabled, "should retain the enabled flag"); + assert_eq!( + bypass.header_name, "x-ts-datadome-test-bypass", + "should retain the configured header name" + ); + assert_eq!( + bypass.credential.expose(), + "temporary-test-credential", + "should retain the configured credential" + ); + } + + #[test] + fn protection_test_bypass_requires_protection_header_and_credential() { + for (enable_protection, header_name, credential, expected_message) in [ + ( + false, + "x-ts-datadome-test-bypass", + "temporary-test-credential", + "requires enable_protection", + ), + (true, "", "temporary-test-credential", "header_name"), + (true, "x-ts-datadome-test-bypass", "", "credential"), + ] { + let mut config = test_config(); + config.enable_protection = enable_protection; + config.protection_test_bypass = Some(ProtectionTestBypassConfig { + enabled: true, + header_name: header_name.to_string(), + credential: Redacted::new(credential.to_string()), + }); + + let err = match DataDomeIntegration::try_new(config) { + Ok(_) => panic!("should reject invalid protection test bypass configuration"), + Err(err) => err, + }; + assert!( + format!("{err:?}").contains(expected_message), + "should explain the invalid protection test bypass configuration" + ); + } } #[test] diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4c74c71e8..e67a83e36 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -4,6 +4,8 @@ use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::{HeaderMap, HeaderName, request_builder}; use error_stack::{Report, ResultExt}; use http::{Method, Request, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use subtle::ConstantTimeEq as _; use url::Url; use crate::error::TrustedServerError; @@ -45,10 +47,20 @@ impl DataDomeIntegration { ); } + let test_bypass_matched = self.take_protection_test_bypass_header(input.request); if !self.config.enable_protection || !self.is_request_protected(&mut input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + if test_bypass_matched { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + log_protection_test_bypass(&input); + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { @@ -156,6 +168,26 @@ impl DataDomeIntegration { true } + fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { + let Some(bypass) = self + .config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return false; + }; + let header_name = HeaderName::from_bytes(bypass.header_name.as_bytes()) + .expect("should validate protection test bypass header name during setup"); + let Some(value) = req.headers_mut().remove(&header_name) else { + return false; + }; + + let actual = Sha256::digest(value.as_bytes()); + let expected = Sha256::digest(bypass.credential.expose().as_bytes()); + bool::from(actual.ct_eq(&expected)) + } + fn protection_validate_url(&self) -> String { format!( "{}{}", @@ -442,6 +474,15 @@ fn is_ip_exclusion_reason(reason: &str) -> bool { ) } +fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { + log::info!( + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={} host={} path={}", + input.request.method(), + request_host(input.request), + input.request.uri().path(), + ); +} + fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { if is_ip_exclusion_reason(reason) { log::info!( @@ -734,12 +775,13 @@ mod tests { use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, + ProtectionTestBypassConfig, }; use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, StubHttpClient, build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, - noop_services_with_client_ip, + build_services_with_secret_and_http_client, noop_services_with_client_ip, }; use crate::settings::Settings; @@ -801,6 +843,121 @@ mod tests { .is_some() } + #[test] + fn protection_test_bypass_skips_api_suppresses_tag_and_strips_header() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + header_name: "x-ts-datadome-test-bypass".to_string(), + credential: Redacted::new("temporary-test-credential".to_string()), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let http_client = Arc::new(StubHttpClient::new()); + let services = + build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + "x-ts-datadome-test-bypass", + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue without a challenge" + ); + assert!( + has_client_tag_suppression_marker(&request), + "the bypass should suppress the automatic DataDome client tag" + ); + assert!( + request.headers().get("x-ts-datadome-test-bypass").is_none(), + "the bypass credential must not reach the publisher origin" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + + #[test] + fn protection_test_bypass_strips_invalid_credential_without_bypassing() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + header_name: "x-ts-datadome-test-bypass".to_string(), + credential: Redacted::new("temporary-test-credential".to_string()), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + "x-ts-datadome-test-bypass", + edgezero_core::http::HeaderValue::from_static("wrong-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching credential must not suppress the DataDome client tag" + ); + assert!( + request.headers().get("x-ts-datadome-test-bypass").is_none(), + "an invalid bypass credential must not reach the publisher origin" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "a non-matching credential must still call the Protection API" + ); + } + #[test] fn ip_exclusions_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 1743a1b75..ba3081f2a 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -86,6 +86,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | | `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | | `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | +| `protection_test_bypass` | object | omitted | Temporary static-header bypass for access-controlled staging tests | | `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | | `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | | `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | @@ -168,36 +169,67 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. +8. The request does not contain a matching enabled `protection_test_bypass` credential. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. -### IP-excluded client-side tag behavior +### Staging test bypass + +For short-lived browser automation on an access-controlled staging site, you +can configure a static header credential that skips only the server-side +Protection API: + +```toml +[integrations.datadome.protection_test_bypass] +enabled = true +header_name = "x-ts-datadome-test-bypass" +credential = "temporary-test-credential" +``` + +`protection_test_bypass` requires `enable_protection = true`; it is disabled +when omitted. Treat the credential as a temporary secret: configure it only +while needed, protect the site with an outer access control such as Basic Auth, +and remove the section when testing finishes. Do not enable it in production. + +A matching header is compared in constant time, removed before the request can +reach DataDome or the publisher origin, and never logged. With Playwright, +apply it to the browser context: + +```ts +await context.setExtraHTTPHeaders({ + "X-TS-DataDome-Test-Bypass": process.env.DATADOME_TEST_BYPASS!, +}); +``` + +### Client-side tag suppression behavior On the Fastly adapter, a request that matches an IP-based DataDome exclusion -also omits Trusted Server's automatically injected client-side DataDome tag -from processed HTML. This keeps the client-side layer consistent with the -server-side Protection API skip. +or the configured test-bypass credential also omits Trusted Server's +automatically injected client-side DataDome tag from processed HTML. This keeps +the client-side layer consistent with the server-side Protection API skip. This behavior applies to: - `protection_excluded_ip_cidrs`; - `protection_excluded_ip_cidr_sources`; -- structured `ip_cidr` rules; and -- structured `ip_cidr_source` rules. +- structured `ip_cidr` rules; +- structured `ip_cidr_source` rules; and +- a matching enabled `protection_test_bypass` credential. ASN, method, path, query-parameter, static-asset, and internal-route exclusions do not automatically suppress the client-side tag. DataDome tags already present in publisher HTML are not removed or changed by this behavior, and `/integrations/datadome/tags.js` remains available when requested directly. -Because the processed HTML differs by client IP, tag-suppressed HTML is marked -`private, max-age=0` and removed from shared surrogate caches. The decision is -reported in the existing protection log, for example: +Because the processed HTML differs by client IP or test credential, +tag-suppressed HTML is marked `private, max-age=0` and removed from shared +surrogate caches. The decision is reported in the existing protection log, for +example: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET host=example.com path=/page ``` ### Structured exclusion rules From 1ce7892b145d3643a0e916311291035759d3e7ea Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 10:49:17 -0500 Subject: [PATCH 154/195] Log DataDome test-bypass registration state Log whether protection_test_bypass is enabled when registering the DataDome integration and include configured header name when enabled. Keep credential secret out of logs., --- .../src/integrations/datadome.rs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 4dd55b28f..112d41da1 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -914,12 +914,26 @@ fn build( return Ok(None); }; - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {})", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection - ); + if let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + { + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: enabled, protection_test_bypass_header: {})", + config.sdk_origin, + config.rewrite_sdk, + config.enable_protection, + bypass.header_name, + ); + } else { + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: disabled)", + config.sdk_origin, + config.rewrite_sdk, + config.enable_protection, + ); + } Ok(Some(DataDomeIntegration::try_new(config)?)) } From 7e6f365ad04b329dfa138cac0736c12220ebb8be Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 12:49:49 -0500 Subject: [PATCH 155/195] fix datadome staging bypass privacy --- crates/trusted-server-core/src/config.rs | 42 ++++- .../src/integrations/datadome.rs | 111 +++++------ .../src/integrations/datadome/protection.rs | 173 +++++++++++++----- crates/trusted-server-core/src/publisher.rs | 102 ++++++++++- .../src/response_privacy.rs | 3 +- docs/guide/integrations/datadome.md | 31 ++-- ...6-08-03-datadome-ip-excluded-client-tag.md | 2 +- ...-datadome-ip-excluded-client-tag-design.md | 4 +- 8 files changed, 347 insertions(+), 121 deletions(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..e74ef4150 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -155,7 +155,9 @@ fn validate_enabled_integrations( validate_integration::(settings, "sourcepoint")?; validate_integration::(settings, "osano")?; validate_integration::(settings, "google_tag_manager")?; - validate_integration::(settings, "datadome")?; + if let Some(config) = settings.integration_config::("datadome")? { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + } validate_integration::(settings, "gpt")?; validate_integration::(settings, "gpt_diagnostics")?; @@ -404,6 +406,44 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn deploy_validation_rejects_invalid_datadome_test_bypass() { + for (enable_protection, store, name, expected_message) in [ + ( + false, + "ts_secrets", + "datadome_test_bypass", + "requires enable_protection", + ), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), + ] { + let mut settings = valid_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": enable_protection, + "protection_test_bypass": { + "enabled": true, + "credential_secret_store": store, + "credential_secret_name": name, + }, + }), + ) + .expect("should insert DataDome config"); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject invalid DataDome test bypass"); + assert!( + format!("{err:?}").contains(expected_message), + "error should mention the invalid bypass setting: {err:?}" + ); + } + } + #[test] fn validate_trait_reports_deploy_errors() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 112d41da1..2486c16be 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -61,7 +61,7 @@ use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; -use http::{HeaderName, Method, StatusCode}; +use http::{Method, StatusCode}; use regex::Regex; use serde::Deserialize; use serde_json::Value as JsonValue; @@ -77,7 +77,6 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; -use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -90,6 +89,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic /// `DataDome` client-side tag for the current response. @@ -121,8 +121,9 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { /// Temporary static-header bypass for server-side `DataDome` protection. /// /// This is intended only for an access-controlled staging environment. A -/// matching header bypasses the server-side Protection API and is removed -/// before the publisher origin receives the request. +/// matching `x-ts-datadome-bypass` header bypasses the server-side Protection +/// API and is removed before the publisher origin receives the request. The +/// credential itself is loaded from the Secret Store at runtime. #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct ProtectionTestBypassConfig { @@ -130,13 +131,13 @@ pub struct ProtectionTestBypassConfig { #[serde(default)] pub enabled: bool, - /// Header name carrying the temporary bypass credential. - #[serde(default)] - pub header_name: String, + /// Secret Store containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_store")] + pub credential_secret_store: String, - /// Static credential expected in [`Self::header_name`]. - #[serde(default)] - pub credential: Redacted, + /// Secret name containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_name")] + pub credential_secret_name: String, } /// Configuration for `DataDome` integration. @@ -278,6 +279,14 @@ fn default_server_side_key_secret_name() -> String { "datadome_server_side_key".to_string() } +fn default_protection_test_bypass_secret_store() -> String { + "ts_secrets".to_string() +} + +fn default_protection_test_bypass_secret_name() -> String { + "datadome_test_bypass".to_string() +} + fn default_timeout_ms() -> u32 { 1500 } @@ -390,7 +399,8 @@ impl DataDomeIntegration { config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); if let Some(bypass) = &mut config.protection_test_bypass { - bypass.header_name = bypass.header_name.trim().to_string(); + bypass.credential_secret_store = bypass.credential_secret_store.trim().to_string(); + bypass.credential_secret_name = bypass.credential_secret_name.trim().to_string(); } if config.enable_protection { @@ -453,6 +463,12 @@ impl DataDomeIntegration { Ok(()) } + pub(crate) fn validate_config_for_startup( + config: DataDomeConfig, + ) -> Result<(), Report> { + Self::try_new(config).map(|_| ()) + } + fn validate_protection_test_bypass( config: &DataDomeConfig, ) -> Result<(), Report> { @@ -469,14 +485,9 @@ impl DataDomeIntegration { "protection_test_bypass requires enable_protection to be true", ))); } - if HeaderName::from_bytes(bypass.header_name.as_bytes()).is_err() { + if bypass.credential_secret_store.is_empty() || bypass.credential_secret_name.is_empty() { return Err(Report::new(Self::error( - "protection_test_bypass.header_name must be a valid HTTP header name", - ))); - } - if bypass.credential.expose().is_empty() { - return Err(Report::new(Self::error( - "protection_test_bypass.credential must not be empty when enabled", + "protection_test_bypass credential_secret_store and credential_secret_name must not be empty when enabled", ))); } @@ -914,28 +925,25 @@ fn build( return Ok(None); }; - if let Some(bypass) = config + let integration = DataDomeIntegration::try_new(config)?; + let protection_test_bypass = integration + .config .protection_test_bypass .as_ref() - .filter(|bypass| bypass.enabled) - { - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: enabled, protection_test_bypass_header: {})", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection, - bypass.header_name, - ); - } else { - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: disabled)", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection, - ); - } + .is_some_and(|bypass| bypass.enabled); + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", + integration.config.sdk_origin, + integration.config.rewrite_sdk, + integration.config.enable_protection, + if protection_test_bypass { + "enabled" + } else { + "disabled" + }, + ); - Ok(Some(DataDomeIntegration::try_new(config)?)) + Ok(Some(integration)) } /// Register the `DataDome` integration with Trusted Server. @@ -1188,8 +1196,8 @@ mod tests { [protection_test_bypass] enabled = true - header_name = "x-ts-datadome-test-bypass" - credential = "temporary-test-credential" + credential_secret_store = "ts_secrets" + credential_secret_name = "datadome_test_bypass" "#, ) .expect("should deserialize DataDome test bypass configuration"); @@ -1199,34 +1207,33 @@ mod tests { assert!(bypass.enabled, "should retain the enabled flag"); assert_eq!( - bypass.header_name, "x-ts-datadome-test-bypass", - "should retain the configured header name" + bypass.credential_secret_store, "ts_secrets", + "should retain the configured credential Secret Store" ); assert_eq!( - bypass.credential.expose(), - "temporary-test-credential", - "should retain the configured credential" + bypass.credential_secret_name, "datadome_test_bypass", + "should retain the configured credential secret name" ); } #[test] - fn protection_test_bypass_requires_protection_header_and_credential() { - for (enable_protection, header_name, credential, expected_message) in [ + fn protection_test_bypass_requires_protection_and_secret_references() { + for (enable_protection, store, name, expected_message) in [ ( false, - "x-ts-datadome-test-bypass", - "temporary-test-credential", + "ts_secrets", + "datadome_test_bypass", "requires enable_protection", ), - (true, "", "temporary-test-credential", "header_name"), - (true, "x-ts-datadome-test-bypass", "", "credential"), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), ] { let mut config = test_config(); config.enable_protection = enable_protection; config.protection_test_bypass = Some(ProtectionTestBypassConfig { enabled: true, - header_name: header_name.to_string(), - credential: Redacted::new(credential.to_string()), + credential_secret_store: store.to_string(), + credential_secret_name: name.to_string(), }); let err = match DataDomeIntegration::try_new(config) { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index e67a83e36..bc06214ff 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -38,20 +38,8 @@ impl DataDomeIntegration { &self, mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { - if self.config.enable_protection { - log::info!( - "[datadome] protection incoming method={} host={} path={}", - input.request.method(), - request_host(input.request), - input.request.uri().path(), - ); - } - - let test_bypass_matched = self.take_protection_test_bypass_header(input.request); - if !self.config.enable_protection || !self.is_request_protected(&mut input) { - return RequestFilterDecision::Continue(RequestFilterEffects::default()); - } - + let test_bypass_matched = + self.take_protection_test_bypass_header(input.request, input.services); if test_bypass_matched { input .request @@ -61,6 +49,10 @@ impl DataDomeIntegration { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + if !self.config.enable_protection || !self.is_request_protected(&mut input) { + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { @@ -168,7 +160,11 @@ impl DataDomeIntegration { true } - fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { + fn take_protection_test_bypass_header( + &self, + req: &mut Request, + services: &RuntimeServices, + ) -> bool { let Some(bypass) = self .config .protection_test_bypass @@ -177,14 +173,32 @@ impl DataDomeIntegration { else { return false; }; - let header_name = HeaderName::from_bytes(bypass.header_name.as_bytes()) - .expect("should validate protection test bypass header name during setup"); - let Some(value) = req.headers_mut().remove(&header_name) else { + let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { return false; }; + let store_name = StoreName::from(bypass.credential_secret_store.as_str()); + let credential = match services + .secret_store() + .get_string(&store_name, &bypass.credential_secret_name) + { + Ok(credential) if !credential.is_empty() => credential, + Ok(_) => { + log::warn!( + "[datadome] DataDome test bypass credential is empty; ignoring bypass header" + ); + return false; + } + Err(err) => { + log::warn!( + "[datadome] Failed to load DataDome test bypass credential; ignoring bypass header: {err:?}" + ); + return false; + } + }; + let actual = Sha256::digest(value.as_bytes()); - let expected = Sha256::digest(bypass.credential.expose().as_bytes()); + let expected = Sha256::digest(credential.as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -476,31 +490,25 @@ fn is_ip_exclusion_reason(reason: &str) -> bool { fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { log::info!( - "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={} host={} path={}", + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={}", input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { if is_ip_exclusion_reason(reason) { log::info!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={} host={} path={}", + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", rule_id, reason, input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } else { log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={}", + "[datadome] protection decision=skipped rule={} reason={} method={}", rule_id, reason, input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } } @@ -512,26 +520,20 @@ fn log_protection_result( decision: &RequestFilterDecision, ) { let method = input.request.method(); - let host = request_host(input.request); - let path = input.request.uri().path(); match decision { RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} host={} path={} route=short_circuit", + "[datadome] protection decision=blocked status={} method={} route=short_circuit", status.as_u16(), method, - host, - path, ), RequestFilterDecision::Continue(_) if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => { log::info!( - "[datadome] protection decision=allowed status={} method={} host={} path={} route=continue", + "[datadome] protection decision=allowed status={} method={} route=continue", status.as_u16(), method, - host, - path, ); } RequestFilterDecision::Continue(_) => {} @@ -850,19 +852,26 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - header_name: "x-ts-datadome-test-bypass".to_string(), - credential: Redacted::new("temporary-test-credential".to_string()), + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), }), ..DataDomeConfig::default() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); let http_client = Arc::new(StubHttpClient::new()); - let services = - build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( - "x-ts-datadome-test-bypass", + super::super::HEADER_DATADOME_TEST_BYPASS, edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); @@ -885,7 +894,10 @@ mod tests { "the bypass should suppress the automatic DataDome client tag" ); assert!( - request.headers().get("x-ts-datadome-test-bypass").is_none(), + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), "the bypass credential must not reach the publisher origin" ); assert!( @@ -894,6 +906,68 @@ mod tests { ); } + #[test] + fn protection_test_bypass_wins_over_other_exclusions() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "staging-page-exclusion".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue" + ); + assert!( + has_client_tag_suppression_marker(&request), + "a matching test credential should suppress the tag even on an excluded path" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + #[test] fn protection_test_bypass_strips_invalid_credential_without_bypassing() { let config = DataDomeConfig { @@ -901,8 +975,8 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - header_name: "x-ts-datadome-test-bypass".to_string(), - credential: Redacted::new("temporary-test-credential".to_string()), + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), }), ..DataDomeConfig::default() }; @@ -912,6 +986,10 @@ mod tests { "datadome_server_side_key".to_string(), b"server-side-key".to_vec(), ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( 200, @@ -925,7 +1003,7 @@ mod tests { let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( - "x-ts-datadome-test-bypass", + super::super::HEADER_DATADOME_TEST_BYPASS, edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); @@ -948,7 +1026,10 @@ mod tests { "a non-matching credential must not suppress the DataDome client tag" ); assert!( - request.headers().get("x-ts-datadome-test-bypass").is_none(), + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), "an invalid bypass credential must not reach the publisher origin" ); assert_eq!( diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 48e0eaacf..5fc8dac0e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1452,12 +1452,21 @@ fn apply_datadome_client_tag_cache_privacy( return; } - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|value| value.contains("private") || value.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } + for header_name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*header_name); + } } /// Drop a bodiless response's body and correct its framing headers. @@ -2895,6 +2904,10 @@ pub async fn handle_publisher_request( .extensions() .get::() .is_some(); + if suppress_datadome_client_side_tag { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -5203,6 +5216,50 @@ mod tests { ); } + #[tokio::test] + async fn suppressed_publisher_request_removes_conditional_validators() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), + "suppressed requests must not forward If-None-Match" + ); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), + "suppressed requests must not forward If-Modified-Since" + ); + } + #[tokio::test] async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { let settings = create_test_settings(); @@ -5365,6 +5422,8 @@ mod tests { .header(header::CACHE_CONTROL, "public, max-age=600") .header("surrogate-control", "max-age=600") .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); @@ -5391,6 +5450,37 @@ mod tests { response.headers().get("fastly-surrogate-control").is_none(), "suppressed HTML should not retain Fastly-Surrogate-Control" ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "suppressed HTML should not retain CDN-Cache-Control" + ); + + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_datadome_client_tag_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store"), + "suppressed HTML should preserve an existing no-store policy" + ); } #[test] diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index e23348211..2205ae892 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,6 +22,7 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", + "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. @@ -37,7 +38,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Surrogate cache headers must come off every cookie-bearing response, even + // Shared-cache control headers must come off every cookie-bearing response, even // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index ba3081f2a..35eda5d3e 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -184,23 +184,30 @@ Protection API: ```toml [integrations.datadome.protection_test_bypass] enabled = true -header_name = "x-ts-datadome-test-bypass" -credential = "temporary-test-credential" +credential_secret_store = "ts_secrets" +credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted. Treat the credential as a temporary secret: configure it only -while needed, protect the site with an outer access control such as Basic Auth, -and remove the section when testing finishes. Do not enable it in production. +when omitted. Store the temporary credential in the configured Secret Store, +configure this section only while needed, protect the site with an outer access +control such as Basic Auth, and remove the section when testing finishes. Do not +enable it in production. -A matching header is compared in constant time, removed before the request can -reach DataDome or the publisher origin, and never logged. With Playwright, -apply it to the browser context: +The fixed `x-ts-datadome-bypass` header is compared in constant time, removed +before the request can reach DataDome or the publisher origin, and never +logged. Scope the header to the staging origin; do not attach it to every +request in a browser context because that can disclose the credential to +third-party origins. With Playwright: ```ts -await context.setExtraHTTPHeaders({ - "X-TS-DataDome-Test-Bypass": process.env.DATADOME_TEST_BYPASS!, -}); +await context.route('https://staging.example.com/**', async (route) => { + const headers = { + ...route.request().headers(), + 'x-ts-datadome-bypass': process.env.DATADOME_TEST_BYPASS!, + } + await route.continue({ headers }) +}) ``` ### Client-side tag suppression behavior @@ -229,7 +236,7 @@ surrogate caches. The decision is reported in the existing protection log, for example: ```text -[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET ``` ### Structured exclusion rules diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 5ea5f6ff3..0dfc923cf 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -152,7 +152,7 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" desired shape is: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET ``` - [ ] **Step 5: Add filter-level marker tests.** Add small helpers in the diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md index c48ad2fda..d6d811780 100644 --- a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -149,10 +149,10 @@ For IP-based skips, extend the existing informational log with `client_tag=omitted`: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET ``` -The existing rule ID, reason, and request metadata remain part of the log. +The existing rule ID, reason, and method remain part of the log. Host and path are omitted to avoid placing dynamic request data in the protection logs. Client IP values are not included. Non-IP skip logs retain their current behavior and level. From 126c17fc73437adc7869d1e512901678eadec223 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 14:56:44 -0500 Subject: [PATCH 156/195] Improve publisher HTML cache policy when SSAT is inactive --- crates/trusted-server-core/src/publisher.rs | 105 ++++++++++++++------ 1 file changed, 77 insertions(+), 28 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d3410b4ed..8919f09b3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2932,33 +2932,49 @@ pub async fn handle_publisher_request( // // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, - // no per-user `tsjs.adSlots`/`tsjs.bids` are injected, so forcing private - // here would needlessly strip shared cacheability from ordinary publisher - // HTML. Applies regardless of the auction *outcome* (empty bids still inject - // per-user slot state). The separate EC-cookie cache net in the adapter's - // `finalize_response` keeps first-visit identity responses private. + // no per-user `tsjs.adSlots`/`tsjs.bids` are injected. Applies regardless of + // the auction *outcome* (empty bids still inject per-user slot state). The + // separate EC-cookie cache net in the adapter's `finalize_response` keeps + // first-visit identity responses private. let origin_content_type = response .headers() .get(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) .unwrap_or_default(); - if should_run_ad_stack && is_html_content_type(origin_content_type) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - // Every CDN-targeted cache directive, not just the browser-facing - // `Cache-Control` above: an origin emitting any of these would otherwise - // instruct an intermediary to store a synthesized per-navigation - // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover - // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) - // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field - // that overrides it there, so both are needed to close the gap on the - // Cloudflare adapter. - for directive in CDN_CACHE_HEADERS { - response.headers_mut().remove(*directive); + if is_html_content_type(origin_content_type) { + if should_run_ad_stack { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); + // Every CDN-targeted cache directive, not just the browser-facing + // `Cache-Control` above: an origin emitting any of these would otherwise + // instruct an intermediary to store a synthesized per-navigation + // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover + // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) + // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field + // that overrides it there, so both are needed to close the gap on the + // Cloudflare adapter. + for directive in CDN_CACHE_HEADERS { + response.headers_mut().remove(*directive); + } + } else { + let origin_cache_control = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase); + if !origin_cache_control + .as_deref() + .is_some_and(|value| value.contains("private") || value.contains("no-store")) + { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + } } } @@ -4697,13 +4713,16 @@ mod tests { .expect("should build conditional navigation request") } - fn queue_cacheable_html_response(stub: &StubHttpClient) { + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { stub.push_response_with_headers( 200, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", cache_control), ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), @@ -4773,7 +4792,7 @@ mod tests { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -4866,11 +4885,11 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -4916,7 +4935,7 @@ mod tests { ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), + (header::CACHE_CONTROL, "max-age=60"), (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -4947,6 +4966,36 @@ mod tests { } } + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "origin {cache_control} policy should not be weakened" + ); + } + } + #[tokio::test] async fn eligible_navigation_rejects_unexpected_origin_304() { for content_type in [None, Some("text/html; charset=utf-8")] { From 92e2a78062dc1b57194cbd8873a73f8ef7e5bdec Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 17:17:40 -0500 Subject: [PATCH 157/195] Document dedicated server-side ad template switch --- ...-server-side-ad-templates-cache-control.md | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md new file mode 100644 index 000000000..f96bdcb28 --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -0,0 +1,238 @@ +# Dedicated Server-Side Ad Templates Switch and Cache Policy Plan + +> **For agentic workers:** Implement this plan task-by-task, keeping the dedicated +> template switch separate from the global auction configuration. + +**Goal:** Add an explicit on/off switch for server-side ad templates, while +retaining the browser-facing cache policy from issue #1007: + +- Server-side ad templates active: `Cache-Control: private, no-store`. +- Server-side ad templates inactive: `Cache-Control: max-age=60`, unless the + origin already sends `private` or `no-store`. +- CDN-specific cache headers must not change when templates are inactive. + +**Issue context:** The current cache-policy change uses the runtime +`should_run_ad_stack` gate. That gate is also affected by `[auction].enabled`, +which is not the right configuration boundary for publisher templates. A +browser can call `POST /auction`, and that endpoint is a separate server-run +auction API. The new switch must disable publisher HTML/page-bids template +delivery without disabling that API. + +## Configuration decision + +Add this field to the existing `[creative_opportunities]` section: + +```toml +[creative_opportunities] +enabled = true +``` + +Use `enabled = false` to turn off server-side ad templates while retaining the +slot definitions and keeping direct `POST /auction` behavior available. + +### Compatibility rules + +- The field defaults to `true` when omitted, preserving existing behavior for + deployments that already have `[creative_opportunities]` configured. +- The section remains optional. An absent section continues to mean that the + feature is unavailable. +- Serialize the default `true` value as omitted, matching the existing + rollback-compatibility pattern for newer creative-opportunity fields. An + explicit `false` must remain serialized so the setting is not silently lost. +- `auction.enabled` remains a separate auction/orchestrator setting. Do not use + it as the dedicated template switch and do not thread the new template flag + into `POST /auction`. + +## Current cache behavior to retain + +The existing HTML policy block in `publisher.rs` must remain structurally +consistent with the current issue #952 behavior: + +1. For an eligible request that runs the server-side ad stack and receives HTML: + - Set `Cache-Control: private, no-store`. + - Remove `ETag` and `Last-Modified`. + - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, + and `Cloudflare-CDN-Cache-Control`. +2. For HTML where the server-side ad stack does not run, including an explicit + template disable: + - Read the browser-facing `Cache-Control` header. + - If its value contains `private` or `no-store`, case-insensitively, preserve + the origin value exactly. + - Otherwise set exactly `Cache-Control: max-age=60`. + - Leave validators and all CDN-specific cache headers untouched. +3. Preserve the later adapter response-privacy finalization for cookie-bearing + responses; this plan does not refactor that behavior. + +## File map + +### Configuration and compatibility + +- `crates/trusted-server-core/src/creative_opportunities.rs` + - Add `CreativeOpportunitiesConfig::enabled` with a default-true serde + implementation and documentation. + - Add a small accessor if it improves readability, but keep the source of + truth in this config type. + - Update config constructors and serialization tests. +- `crates/trusted-server-core/src/settings.rs` + - Keep `creative_opportunities` parsing and runtime preparation compatible with + the new field. + - Make `creative_opportunity_slots()` return an empty slice when the section + is absent or explicitly disabled, so all adapters receive one consistent + runtime view. + - Add TOML and environment-override coverage for `enabled = false`. +- `crates/trusted-server-core/src/config.rs` + - Extend legacy-schema tests to prove default `enabled = true` is omitted from + serialized blobs and remains readable by older binaries. + - Prove an explicit `enabled = false` is serialized, making rollback failure + loud rather than silently re-enabling templates. +- `trusted-server.example.toml` + - Document `creative_opportunities.enabled` and show how to turn templates off + without deleting slot definitions. +- `docs/guide/configuration.md` + - Add the field to the creative-opportunities reference and document the + environment override: + `TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false`. + - Clarify that this switch controls publisher HTML/page-bids template + delivery, not direct `POST /auction` callers. +- `CHANGELOG.md` + - Add an entry describing the dedicated template switch and cache behavior. + +### Publisher execution and cache policy + +- `crates/trusted-server-core/src/publisher.rs` + - Include the dedicated flag in the initial publisher eligibility decision. + - Do not match, dispatch, or inject server-side ad templates when the flag is + false, even if slots are configured and `[auction].enabled` is true. + - Apply the issue #1007 inactive-HTML cache policy in this state. + - Update skip-reason diagnostics/telemetry so `ad_templates_disabled` is + distinguishable from `auction_disabled`, consent denial, bots, prefetch, and + no matching slots. + - Update `handle_page_bids` so an explicit template disable returns the normal + empty JSON shape (`slots: []`, `bids: {}`) rather than slot definitions. Keep + the current `404` behavior for an absent `[creative_opportunities]` section. + - Extend the existing SSAT cache-policy and eligibility tests. +- `crates/trusted-server-core/src/auction/endpoints.rs` + - Do not gate `POST /auction` on the new template flag. + - Add a regression test or test fixture proving that disabling + `creative_opportunities.enabled` does not suppress a direct auction request + when providers are configured. + - Separately document/verify the existing behavior of `[auction].enabled` for + this endpoint; do not conflate that global setting with the new template + switch. + +### Adapter propagation and browser behavior + +The adapters already pass `Settings::creative_opportunity_slots()` into the +publisher/page-bids handlers. Update and verify these call sites so the central +empty-slice behavior is honored; avoid adding four divergent config checks: + +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/src/app.rs` +- `crates/trusted-server-adapter-cloudflare/src/app.rs` +- `crates/trusted-server-adapter-spin/src/app.rs` + +No route-level flag is needed if the core `Settings` accessor and handlers are +correct. Add adapter route assertions only where existing fixtures make them +useful. + +The browser runtime already defaults `window.tsjs.adSlots` and +`window.tsjs.bids` to empty values when the edge does not inject templates. If +terminology is updated, adjust these comments/tests without changing runtime +semantics: + +- `crates/trusted-server-js/lib/src/core/index.ts` +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Relevant page-bids tests under `crates/trusted-server-js/lib/test/integrations/gpt/` + +## Implementation tasks + +### Task 1: Add and serialize the dedicated setting + +- [ ] Add `enabled: bool` to `CreativeOpportunitiesConfig` with default `true`. +- [ ] Use `skip_serializing_if` so the default value does not appear in stored + config blobs; explicit `false` must serialize. +- [ ] Update all Rust struct literals in `creative_opportunities.rs` and + `publisher.rs` tests. +- [ ] Add parsing, default, false-value, and environment-override tests. +- [ ] Update the legacy compatibility tests in `config.rs`. + +### Task 2: Thread the setting through publisher eligibility + +- [ ] Update `should_run_server_side_ad_stack` to accept the dedicated template + flag as an explicit gate, with a descriptive parameter/doc comment. +- [ ] Ensure initial publisher slot matching and `Settings::creative_opportunity_slots` + do not expose slots when templates are disabled. +- [ ] Preserve the existing `[auction].enabled` and consent gates as separate + conditions. +- [ ] Add an `ad_templates_disabled` diagnostic/telemetry skip reason where the + current branch records a skipped auction. + +### Task 3: Apply the cache policy to the dedicated-off state + +- [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header + removal unchanged. +- [ ] Keep the inactive-HTML `max-age=60` behavior from issue #1007. +- [ ] Verify that explicit template disable changes only browser-facing + `Cache-Control` for cacheable HTML; preserve `ETag`, `Last-Modified`, and + every CDN-specific header. +- [ ] Verify that origin `private`, `PRIVATE`, `no-store`, and `No-Store` values + remain unchanged. + +### Task 4: Gate SPA page-bids/template delivery + +- [ ] Include `co_config.enabled` in the `ad_stack_enabled` decision in + `handle_page_bids`. +- [ ] Return empty slots and bids for an explicit disable while retaining the + endpoint and its existing response privacy headers. +- [ ] Keep the absent-section `404` behavior unchanged. +- [ ] Add tests for enabled, disabled, absent, consent-denied, bot, and prefetch + cases as appropriate; preserve existing tests for `[auction].enabled=false`. + +### Task 5: Protect direct `POST /auction` from accidental coupling + +- [ ] Add a focused endpoint test with `creative_opportunities.enabled=false` + and a recording provider. +- [ ] Assert that the provider still sees the direct auction request and that + the response remains a normal OpenRTB response. +- [ ] If the test reveals that `[auction].enabled=false` also needs a separate + product decision for `/auction`, record that as a follow-up rather than + changing it as part of the template-switch work. + +### Task 6: Update docs, examples, comments, and adapter coverage + +- [ ] Update the example config, configuration guide, and changelog. +- [ ] Update stale comments that call `[auction].enabled` the universal template + kill switch. +- [ ] Verify all four adapter call sites use the centralized disabled-slot view. +- [ ] Run JS tests if comments or tests are touched; no JS behavior change is + expected. + +## Test plan + +Use target-matched commands; do not run bare workspace tests because the +workspace contains multiple runtime targets. + +- [ ] `cargo test-axum -p trusted-server-core publisher` +- [ ] `cargo test-fastly` +- [ ] `cargo test-axum` +- [ ] `cargo test-cloudflare` +- [ ] `cargo test-spin` +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy-fastly` +- [ ] `cargo clippy-axum` +- [ ] `cargo clippy-cloudflare` +- [ ] `cargo clippy-cloudflare-wasm` +- [ ] `cargo clippy-spin-native` +- [ ] `cargo clippy-spin-wasm` +- [ ] `cd crates/trusted-server-js/lib && npx vitest run` if JS tests/comments change +- [ ] `cd docs && npm run format` if documentation formatting is required + +## Non-goals + +- Do not change CDN-specific cache policy for inactive templates. +- Do not change adapter response privacy or cookie handling. +- Do not use `auction.rewrite_creatives` as the template switch; it controls + creative URL rewriting, not whether the server-side template stack runs. +- Do not gate or disable direct `POST /auction` as part of this feature. +- Do not remove slot definitions when the switch is off; the point of the switch + is to provide a reversible runtime control. From 58054463a16ce801198f877b687b579a33a9d9a3 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 18:03:25 -0500 Subject: [PATCH 158/195] Add dedicated server-side ad template switch --- CHANGELOG.md | 1 + .../src/auction/endpoints.rs | 127 +++++++++- crates/trusted-server-core/src/config.rs | 27 ++ .../src/creative_opportunities.rs | 47 +++- crates/trusted-server-core/src/publisher.rs | 237 +++++++++++++++--- crates/trusted-server-core/src/settings.rs | 46 +++- .../trusted-server-js/lib/src/core/index.ts | 8 +- .../lib/src/integrations/gpt/index.ts | 12 +- .../test/integrations/gpt/spa_hook.test.ts | 6 +- docs/guide/configuration.md | 20 +- trusted-server.example.toml | 3 + 11 files changed, 480 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36655763c..7c43a6fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Publisher HTML now uses `Cache-Control: max-age=60` when server-side ad templates are inactive, while preserving origin `private`/`no-store` policies and CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. - **Breaking** — Integration configuration strings are no longer globally reinterpreted as JSON scalars. Operators upgrading should audit `[integrations.*]` settings and use native TOML/typed-config booleans and numbers (for example, `enabled = true`, not `enabled = "true"`); quoted numeric and boolean scalars now fail validation instead of silently converting. - **Breaking** — Sourcepoint browser module inclusion now requires explicit `[integrations.sourcepoint].enabled = true`; operators relying on the previous unconditional Sourcepoint module should enable the integration before upgrading. diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 36e083948..b80b9339c 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -554,10 +554,14 @@ mod tests { use crate::consent::types::ConsentContext; use crate::openrtb::Uid; use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, noop_services, + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, + noop_services, }; - use crate::platform::{ClientInfo, PlatformPendingRequest, PlatformResponse}; - use crate::test_support::tests::create_test_settings; + use crate::platform::{ + ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, + }; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use serde_json::json; @@ -642,6 +646,123 @@ mod tests { } } + /// Provider used to prove that direct `/auction` remains available when + /// publisher server-side ad templates are disabled. + struct TemplateSwitchProbeProvider { + calls: Arc>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for TemplateSwitchProbeProvider { + fn provider_name(&self) -> &'static str { + "template_switch_probe" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + *self.calls.lock().expect("should lock provider call count") += 1; + let request = Request::builder() + .method("POST") + .uri("https://bidder.example/auction") + .body(EdgeBody::empty()) + .expect("should build probe provider request"); + context + .services + .http_client() + .send_async(PlatformHttpRequest::new( + request, + "template-switch-probe-backend", + )) + .await + .change_context(TrustedServerError::Auction { + message: "probe provider launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.provider_name(), + Vec::new(), + 0, + )) + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("template-switch-probe-backend".to_string()) + } + } + + #[tokio::test] + async fn direct_auction_remains_available_when_templates_are_disabled() { + let settings_toml = format!( + "{}\n[auction]\nenabled = true\nproviders = [\"template_switch_probe\"]\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_toml) + .expect("should parse settings with disabled templates"); + let calls = Arc::new(Mutex::new(0)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { + calls: Arc::clone(&calls), + })); + + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"probe response".to_vec()); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build(); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("direct auction should remain available"); + + assert_eq!( + *calls.lock().expect("should lock provider call count"), + 1, + "disabling publisher templates must not disable direct /auction" + ); + assert_eq!(response.status(), StatusCode::OK); + } + #[tokio::test] async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..033e6c908 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -321,10 +321,37 @@ formats = [{ width = 300, height = 250 }] fn absent_gam_unit_template_is_accepted_by_legacy_schema() { let creative_opportunities = serialized_creative_opportunities(None); + assert!( + creative_opportunities.get("enabled").is_none(), + "default template switch should be omitted for legacy binaries" + ); serde_json::from_value::(creative_opportunities) .expect("should accept absent GAM unit template"); } + #[test] + fn disabled_creative_opportunities_flag_is_visible_to_legacy_schema() { + let mut toml = crate_test_settings_str(); + toml.push_str( + r#" + +[creative_opportunities] +enabled = false +gam_network_id = "99999" +"#, + ); + let app_config: TrustedServerAppConfig = + toml::from_str(&toml).expect("should deserialize app config wrapper"); + let creative_opportunities = serde_json::to_value(app_config) + .expect("should serialize app config wrapper") + .get("creative_opportunities") + .cloned() + .expect("should contain creative opportunities"); + + serde_json::from_value::(creative_opportunities) + .expect_err("legacy binaries should reject an explicit disabled switch"); + } + #[test] fn deploy_validation_rejects_placeholders() { let settings = Settings::from_toml( diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 10a85b3e8..a6b70e1d6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -183,10 +183,27 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str } } +const fn default_enabled() -> bool { + true +} + +const fn is_default_enabled(value: &bool) -> bool { + *value == default_enabled() +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { + /// Enables server-side ad template delivery on publisher HTML and page-bids requests. + /// + /// This does not disable the direct `POST /auction` endpoint. The default is + /// `true` so existing creative-opportunity configurations retain their behavior. + #[serde( + default = "default_enabled", + skip_serializing_if = "is_default_enabled" + )] + pub enabled: bool, /// GAM network ID used to build default unit paths. pub gam_network_id: String, /// Maximum time in milliseconds to wait for the server-side auction before @@ -244,7 +261,7 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, - /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). + /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } @@ -1139,12 +1156,39 @@ mod tests { assert_eq!(derive_section("/%%%/x", "home", 0), "_"); } + #[test] + fn enabled_defaults_true_and_is_omitted_from_serialized_config() { + let config = make_config_with_section_template(None); + assert!( + config.enabled, + "template delivery should default to enabled" + ); + let value = serde_json::to_value(&config).expect("should serialize config"); + assert!( + value.get("enabled").is_none(), + "default enabled value should be omitted for rollback compatibility" + ); + } + + #[test] + fn disabled_template_switch_is_serialized() { + let mut config = make_config_with_section_template(None); + config.enabled = false; + let value = serde_json::to_value(&config).expect("should serialize config"); + assert_eq!( + value.get("enabled"), + Some(&serde_json::Value::Bool(false)), + "explicitly disabled template delivery must remain in config blobs" + ); + } + fn make_config_with_section_template( section_root: Option<&str>, ) -> CreativeOpportunitiesConfig { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), @@ -1542,6 +1586,7 @@ mod tests { // Older binaries deserialize this struct with `deny_unknown_fields`, so // a pushed config blob must not carry `"section_root": null`. let config = CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8919f09b3..87ab8317b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1751,27 +1751,34 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -/// Returns true only when the publisher request should run the full -/// server-side ad stack: auction dispatch plus initial ad-slot injection. +#[derive(Debug, Clone, Copy)] +struct ServerSideAdStackConfig { + /// Dedicated `[creative_opportunities].enabled` switch. + ad_templates_enabled: bool, + /// Global `[auction].enabled` gate used by publisher/page-bids flows. + auction_enabled: bool, +} + +/// Returns true only when the publisher should inject and run server-side ad templates. /// -/// `auction_enabled` is the global `[auction].enabled` kill switch — when -/// false, no automatic server-side auction or ad-slot injection runs. -pub(crate) fn should_run_server_side_ad_stack( +/// This includes auction dispatch plus initial ad-slot injection. +fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, is_prefetch: bool, is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, - auction_enabled: bool, + config: ServerSideAdStackConfig, ) -> bool { is_get && is_navigation && !is_prefetch && !is_bot + && config.ad_templates_enabled && has_matched_slots && consent_allows_auction - && auction_enabled + && config.auction_enabled } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. @@ -2632,7 +2639,10 @@ pub async fn handle_publisher_request( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - let matched_slots = if is_get { + let creative_opportunities = settings.creative_opportunities.as_ref(); + let ad_templates_enabled = creative_opportunities.is_some_and(|co_config| co_config.enabled); + let ad_templates_disabled = creative_opportunities.is_some_and(|co_config| !co_config.enabled); + let matched_slots = if is_get && ad_templates_enabled { settings .creative_opportunities .as_ref() @@ -2655,7 +2665,10 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, - auction.orchestrator.is_enabled(), + ServerSideAdStackConfig { + ad_templates_enabled, + auction_enabled: auction.orchestrator.is_enabled(), + }, ); let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with @@ -2663,14 +2676,16 @@ pub async fn handle_publisher_request( // when `consent_allows_auction=false`. log::debug!( "server-side ad-stack gate: is_get={is_get} is_navigation={is_navigation} \ - is_prefetch={is_prefetch} is_bot={is_bot} matched_slots={} \ - consent_allows_auction={consent_allows_auction} orchestrator_enabled={} \ - -> should_run_auction={should_run_auction}", + is_prefetch={is_prefetch} is_bot={is_bot} ad_templates_enabled={ad_templates_enabled} \ + matched_slots={} consent_allows_auction={consent_allows_auction} \ + orchestrator_enabled={} -> should_run_auction={should_run_auction}", matched_slots.len(), auction.orchestrator.is_enabled(), ); - if matched_slots.is_empty() && settings.creative_opportunities.is_some() { + if ad_templates_disabled { + log::debug!("Server-side ad templates are disabled by configuration"); + } else if matched_slots.is_empty() && settings.creative_opportunities.is_some() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction and injection", request_path @@ -2795,7 +2810,9 @@ pub async fn handle_publisher_request( } } } else { - let skip_reason = if !auction.orchestrator.is_enabled() { + let skip_reason = if ad_templates_disabled { + "ad_templates_disabled" + } else if !auction.orchestrator.is_enabled() { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -3771,7 +3788,11 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let matched_slots = match_renderable_slots(auction.slots, co_config, &path_param); + let matched_slots = if co_config.enabled { + match_renderable_slots(auction.slots, co_config, &path_param) + } else { + Vec::new() + }; let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); @@ -3790,7 +3811,10 @@ pub async fn handle_page_bids( let is_bot = is_bot_user_agent(&req); let auction_enabled = auction.orchestrator.is_enabled(); - if !auction_enabled { + let ad_templates_enabled = co_config.enabled; + if !ad_templates_enabled { + log::debug!("page-bids: [creative_opportunities].enabled is false — skipping templates"); + } else if !auction_enabled { log::debug!("page-bids: [auction].enabled is false — skipping auction"); } else if matched_slots.is_empty() { log::debug!( @@ -3806,14 +3830,14 @@ pub async fn handle_page_bids( ); } - // The [auction].enabled kill switch and a consent denial disable the entire - // server-side ad stack. In those states the endpoint must return no slots, - // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — - // otherwise the kill switch/consent gate would stop SSP calls but still let - // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, - // keep their slot definitions (the placement structure is unchanged) but - // skip the live auction, matching the existing bot/prefetch behaviour. - let ad_stack_enabled = auction_enabled && consent_allows_auction; + // The dedicated template switch, [auction].enabled, and a consent denial + // disable the entire server-side ad stack. In those states the endpoint must + // return no slots, so the SPA hook does not assign `ts.adSlots` and call + // `adInit()` — otherwise the gate would stop SSP calls but still let the + // client create/refresh GPT slots client-side. Bot/prefetch requests, by + // contrast, keep their slot definitions (the placement structure is + // unchanged) but skip the live auction, matching the existing behavior. + let ad_stack_enabled = ad_templates_enabled && auction_enabled && consent_allows_auction; let winning_bids = if matched_slots.is_empty() { std::collections::HashMap::new() @@ -3903,7 +3927,9 @@ pub async fn handle_page_bids( } } } else { - let skip_reason = if !auction_enabled { + let skip_reason = if !ad_templates_enabled { + "ad_templates_disabled" + } else if !auction_enabled { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -4655,6 +4681,15 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + } + fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -4966,6 +5001,72 @@ mod tests { } } + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_ad_templates(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabled server-side ad templates should not bypass the origin cache" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "disabled server-side ad templates should use the short browser cache policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); + } + } + #[tokio::test] async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { let settings = settings_with_enabled_auction_and_creative_opportunities(); @@ -5450,39 +5551,69 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + let enabled_config = ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: true, + }; assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" + should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), + "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), "requests without required consent should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: false, + }, + ), "disabled [auction].enabled kill switch should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: false, + auction_enabled: true, + }, + ), + "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -8120,6 +8251,7 @@ mod tests { fn make_config() -> CreativeOpportunitiesConfig { CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, @@ -9337,6 +9469,14 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + fn settings_with_co_templates_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled templates") + } + async fn run_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, @@ -9886,6 +10026,35 @@ mod tests { ); } + #[tokio::test] + async fn disabled_server_side_ad_templates_return_no_slots_or_bids() { + // The dedicated template switch must suppress publisher/page-bids + // delivery without using the global auction switch. + let settings = settings_with_co_templates_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "disabled server-side ad templates must not return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled server-side ad templates must not produce bids" + ); + } + #[tokio::test] async fn consent_denied_returns_no_slots_or_bids() { // When consent denies the server-side auction (here: Jurisdiction diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 1021ee9ec..06152659c 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2095,13 +2095,14 @@ impl Settings { Ok(()) } - /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( &self, ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { self.creative_opportunities .as_ref() + .filter(|co| co.enabled) .map(|co| co.slot.as_slice()) .unwrap_or(&[]) } @@ -5010,6 +5011,10 @@ formats = [{ width = 300, height = 250 }] let co = settings .creative_opportunities .expect("should have creative_opportunities"); + assert!( + co.enabled, + "creative-opportunity templates should default to enabled" + ); assert_eq!(co.gam_network_id, "21765378893"); assert_eq!(co.auction_timeout_ms, Some(500)); assert_eq!( @@ -5019,6 +5024,45 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_disables_creative_opportunity_slots_when_configured_off() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"21765378893\"\n\n[[creative_opportunities.slot]]\nid = \"atf\"\npage_patterns = [\"/\"]\nformats = [{{ width = 300, height = 250 }}]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse disabled templates"); + assert!( + settings.creative_opportunity_slots().is_empty(), + "disabled template delivery should expose no runtime slots" + ); + } + + #[test] + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", + crate_test_settings_str() + ); + let env_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_var(env_key, Some("false"), || { + let settings = Settings::from_toml_and_env(&toml) + .expect("should parse template enabled environment override"); + assert!( + !settings + .creative_opportunities + .expect("should have creative opportunities") + .enabled, + "environment override should disable template delivery" + ); + }); + } + #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index 5d8e41971..b2c4e41e1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -36,10 +36,10 @@ api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; // Defensive defaults: the edge injects adSlots (head-open) and bids (before -// ) only when the server-side ad stack runs for the request. When it -// is gated off (kill switch, consent fail-closed, bots, prefetch), page code -// reading window.tsjs.bids / window.tsjs.adSlots must still see defined -// values instead of throwing. Injected scripts overwrite these wholesale. +// ) only when server-side ad templates run for the request. When template +// delivery is disabled or gated off (auction/consent, bots, prefetch), page code +// reading window.tsjs.bids / window.tsjs.adSlots must still see defined values +// instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; // Point global tsjs diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 30cfbbbdd..acea49fa6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -689,8 +689,8 @@ export function installTsAdInit(): void { ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. + // response (template switch, auction gate, or consent denial) returns no + // slots, so the loops above leave these empty. const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page @@ -971,10 +971,10 @@ export function installSpaAuctionHook(): void { // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; - // An empty page-bids response (auction kill switch or consent gate) carries - // no TS slots. Only run adInit() when there are slots to apply or prior TS - // state to sweep — otherwise a consent-denied or kill-switched navigation - // must not enter the GPT command queue and risk activating services. + // An empty page-bids response (template switch, auction, or consent gate) + // carries no TS slots. Only run adInit() when there are slots to apply or + // prior TS state to sweep — otherwise a gated navigation must not enter + // the GPT command queue and risk activating services. const hasPriorTsState = (ts.prevGptSlots?.length ?? 0) > 0 || Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 19739e3dd..36c41aa71 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -115,9 +115,9 @@ describe('installSpaAuctionHook', () => { }); it('skips adInit on an empty page-bids response with no prior TS state', async () => { - // A gated page-bids response (auction kill switch or consent denial) returns - // no slots. With no prior TS state to sweep, the hook must not call adInit() - // so a consent-denied navigation cannot activate the publisher's GPT setup. + // A gated page-bids response (template switch, auction gate, or consent + // denial) returns no slots. With no prior TS state to sweep, the hook must + // not call adInit() so a gated navigation cannot activate publisher GPT. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index aa2b0264b..ef991eea0 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1312,8 +1312,16 @@ Defines the ad slots the trusted server offers on a page: which pages each slot appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad unit it maps to (`gam_unit_path`). +`enabled` is the dedicated server-side ad-template switch. It defaults to `true` +for compatibility with existing configurations. Set it to `false` to stop +publisher HTML and SPA page-bids template delivery while retaining the slot +configuration and direct `POST /auction` endpoint. The browser-facing cache +policy for a disabled template stack is `Cache-Control: max-age=60`, unless the +origin already sends `private` or `no-store`. + ```toml [creative_opportunities] +enabled = true # set to false to disable server-side ad templates gam_network_id = "123456789" price_granularity = "dense" @@ -1332,6 +1340,13 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` +The same switch can be overridden through the legacy environment-variable +loader: + +```bash +TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false +``` + ### `gam_unit_path` templating `gam_unit_path` is a template. A publisher whose ad unit varies by site section @@ -1385,8 +1400,9 @@ publisher-specific. Startup fails if `{section}` is used without a valid `section_root`. Startup rejects a blank `gam_network_id` only when an absent path/default or a `{network_id}` template consumes it; static paths and templates without `{network_id}` do not consume it. A -`[creative_opportunities]` block with no slots is disabled, so its -`gam_network_id` is not checked. +`[creative_opportunities]` block with `enabled = false` or no slots is +inactive, so no publisher templates are delivered and its `gam_network_id` is +not checked when no slot uses it. Both knobs are config-driven, so the URL→section convention stays with the publisher: `section_segment` selects which segment names the section, and diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e78d2b255..ef5bf5487 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -155,6 +155,9 @@ ja4_endpoint_enabled = false auction_html_comment = false [creative_opportunities] +# Set to false to disable server-side ad templates while retaining slot definitions +# and direct POST /auction callers. +enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on From d0fffb9d4e886681b92a50e826cfa877f89d1a51 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 11 Aug 2026 08:04:10 -0500 Subject: [PATCH 159/195] Address DataDome review feedback --- .../trusted-server-core/src/html_processor.rs | 24 ++- .../src/integrations/datadome.rs | 26 ++- .../src/integrations/datadome/protection.rs | 199 +++++++++++++++--- .../src/integrations/registry.rs | 49 ++++- crates/trusted-server-core/src/publisher.rs | 67 +++--- .../src/response_privacy.rs | 1 - docs/guide/integrations/datadome.md | 24 ++- ...6-08-03-datadome-ip-excluded-client-tag.md | 7 - 8 files changed, 304 insertions(+), 93 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 4c827ace0..3bff588fe 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -966,7 +966,7 @@ mod tests { } #[test] - fn suppressed_datadome_tag_is_not_injected_into_processed_html() { + fn suppressed_datadome_tag_preserves_and_rewrites_publisher_tag() { let mut settings = create_test_settings(); settings .integrations @@ -991,7 +991,10 @@ mod tests { let mut processor = create_html_processor(config); let output = processor - .process_chunk(b"content", true) + .process_chunk( + br#"content"#, + true, + ) .expect("should process HTML"); let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); @@ -1000,8 +1003,21 @@ mod tests { "should omit the DataDome client configuration" ); assert!( - !html.contains("/integrations/datadome/tags.js"), - "should omit the DataDome client tag URL" + html.contains("id=\"publisher-datadome\""), + "should preserve the publisher-originated DataDome tag" + ); + assert!( + html.contains("src=\"/integrations/datadome/tags.js\""), + "should rewrite the publisher-originated DataDome tag" + ); + assert!( + !html.contains("https://js.datadome.co/tags.js"), + "should remove the original third-party DataDome URL" + ); + assert_eq!( + html.matches("/integrations/datadome/tags.js").count(), + 1, + "should leave exactly one publisher-originated DataDome tag" ); } diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 2486c16be..75e6c3fdc 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -68,6 +68,7 @@ use serde_json::Value as JsonValue; use url::Url; use validator::Validate; +use crate::constants::ENV_FASTLY_IS_STAGING; use crate::error::TrustedServerError; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, @@ -469,6 +470,17 @@ impl DataDomeIntegration { Self::try_new(config).map(|_| ()) } + fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { + if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() != Ok("1") { + return None; + } + + self.config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + } + fn validate_protection_test_bypass( config: &DataDomeConfig, ) -> Result<(), Report> { @@ -926,18 +938,26 @@ fn build( }; let integration = DataDomeIntegration::try_new(config)?; - let protection_test_bypass = integration + let protection_test_bypass_configured = integration .config .protection_test_bypass .as_ref() .is_some_and(|bypass| bypass.enabled); + let protection_test_bypass_active = integration.active_protection_test_bypass().is_some(); + if protection_test_bypass_configured && !protection_test_bypass_active { + log::warn!( + "[datadome] DataDome test bypass is configured but inactive because FASTLY_IS_STAGING is not 1" + ); + } log::info!( "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", integration.config.sdk_origin, integration.config.rewrite_sdk, integration.config.enable_protection, - if protection_test_bypass { - "enabled" + if protection_test_bypass_active { + "active" + } else if protection_test_bypass_configured { + "configured-inactive" } else { "disabled" }, diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index bc06214ff..f2803b38f 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -165,15 +165,11 @@ impl DataDomeIntegration { req: &mut Request, services: &RuntimeServices, ) -> bool { - let Some(bypass) = self - .config - .protection_test_bypass - .as_ref() - .filter(|bypass| bypass.enabled) - else { + let value = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); + let Some(bypass) = self.active_protection_test_bypass() else { return false; }; - let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { + let Some(value) = value else { return false; }; @@ -806,6 +802,23 @@ mod tests { .expect("should build filter request") } + fn filter_with_staging( + integration: &DataDomeIntegration, + settings: &Settings, + services: &RuntimeServices, + request: &mut Request, + ) -> RequestFilterDecision { + temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { + futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { + settings, + services, + request, + geo_info: None, + is_integration_route: false, + })) + }) + } + fn filter_marks_request( config: DataDomeConfig, services: &RuntimeServices, @@ -875,15 +888,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -906,6 +911,148 @@ mod tests { ); } + #[test] + fn protection_test_bypass_header_is_stripped_when_unconfigured_or_disabled() { + for protection_test_bypass in [ + None, + Some(ProtectionTestBypassConfig { + enabled: false, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ] { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass, + ..DataDomeConfig::default() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("stale-test-credential"), + ); + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass header must be stripped when the bypass is unconfigured or disabled" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "an inactive bypass must not suppress the DataDome client tag" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "an inactive bypass must still call the Protection API" + ); + } + } + + #[test] + fn protection_test_bypass_is_inactive_outside_staging() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = temp_env::with_var( + crate::constants::ENV_FASTLY_IS_STAGING, + None::<&str>, + || { + futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )) + }, + ); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass credential must be stripped outside staging" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "the bypass must not suppress the DataDome client tag outside staging" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "the bypass must still call the Protection API outside staging" + ); + } + #[test] fn protection_test_bypass_wins_over_other_exclusions() { let config = DataDomeConfig { @@ -944,15 +1091,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1007,15 +1146,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 291a54242..16cbac868 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1,4 +1,4 @@ -use std::any::Any; +use std::any::{Any, TypeId}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; @@ -98,17 +98,19 @@ pub struct IntegrationScriptContext<'a> { pub document_state: &'a IntegrationDocumentState, } +type IntegrationDocumentStateMap = BTreeMap<(&'static str, TypeId), Arc>; + /// Per-document state shared between HTML/script rewriters and post-processors. /// /// This exists to support multi-phase HTML processing without requiring a second HTML parse. #[derive(Clone, Default)] pub struct IntegrationDocumentState { - inner: Arc>>>, + inner: Arc>, } impl std::fmt::Debug for IntegrationDocumentState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let keys: Vec<&'static str> = { + let keys: Vec<(&'static str, TypeId)> = { let guard = self .inner .lock() @@ -136,7 +138,7 @@ impl IntegrationDocumentState { .inner .lock() .expect("should lock integration document state"); - let value = guard.get(integration_id)?; + let value = guard.get(&(integration_id, TypeId::of::()))?; let cloned: Arc = Arc::clone(value); cloned.downcast::().ok() } @@ -159,17 +161,15 @@ impl IntegrationDocumentState { .lock() .expect("should lock integration document state"); - if let Some(existing) = guard.get(integration_id) + let key = (integration_id, TypeId::of::()); + if let Some(existing) = guard.get(&key) && let Ok(downcast) = Arc::clone(existing).downcast::() { return downcast; } let value: Arc = Arc::new(init()); - guard.insert( - integration_id, - Arc::clone(&value) as Arc, - ); + guard.insert(key, Arc::clone(&value) as Arc); value } @@ -1405,6 +1405,37 @@ mod tests { } } + #[test] + fn document_state_keeps_multiple_types_for_one_integration() { + let state = IntegrationDocumentState::default(); + let number = state.get_or_insert_with("test", || 7_u32); + let label = state.get_or_insert_with("test", || "first".to_string()); + let repeated_number = state.get_or_insert_with("test", || 99_u32); + + assert!( + Arc::ptr_eq(&number, &repeated_number), + "repeated insertion should preserve the original typed state" + ); + assert_eq!( + *state.get::("test").expect("should retrieve number"), + 7, + "should retain numeric state" + ); + assert_eq!( + state + .get::("test") + .expect("should retrieve label") + .as_str(), + "first", + "should retain string state under the same integration ID" + ); + assert_eq!( + label.as_str(), + "first", + "should return inserted string state" + ); + } + #[test] fn default_html_post_processor_should_process_is_false() { let processor = NoopHtmlPostProcessor; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 57c4be3db..79ebf4436 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1423,6 +1423,14 @@ pub async fn publisher_response_into_streaming_response( } } +/// Removes request headers that can produce a bodyless or partial origin response. +fn strip_conditional_and_range_headers(req: &mut Request) { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + req.headers_mut().remove(header::RANGE); + req.headers_mut().remove(header::IF_RANGE); +} + /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// @@ -1464,6 +1472,8 @@ fn apply_datadome_client_tag_cache_privacy( HeaderValue::from_static("private, max-age=0"), ); } + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); for header_name in CDN_CACHE_HEADERS { response.headers_mut().remove(*header_name); } @@ -2881,11 +2891,15 @@ pub async fn handle_publisher_request( } ); - if should_run_ad_stack { - req.headers_mut().remove(header::IF_NONE_MATCH); - req.headers_mut().remove(header::IF_MODIFIED_SINCE); - req.headers_mut().remove(header::RANGE); - req.headers_mut().remove(header::IF_RANGE); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); + if should_run_ad_stack || suppress_datadome_client_side_tag { + // The origin content type is not known yet, so request hints cannot safely + // narrow this to HTML without allowing 304 or 206 responses to bypass a + // response mutation that becomes necessary after the fetch. + strip_conditional_and_range_headers(&mut req); } // Only advertise encodings the rewrite pipeline can decode and re-encode. @@ -2912,14 +2926,6 @@ pub async fn handle_publisher_request( // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. let request_method = req.method().clone(); - let suppress_datadome_client_side_tag = req - .extensions() - .get::() - .is_some(); - if suppress_datadome_client_side_tag { - req.headers_mut().remove(header::IF_NONE_MATCH); - req.headers_mut().remove(header::IF_MODIFIED_SINCE); - } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -5339,6 +5345,8 @@ mod tests { .header(header::HOST, "publisher.example") .header(header::IF_NONE_MATCH, "\"cached-page\"") .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-page\"") .body(EdgeBody::empty()) .expect("should build conditional request"); req.extensions_mut() @@ -5351,18 +5359,19 @@ mod tests { .into_iter() .next() .expect("should record one outbound request"); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), - "suppressed requests must not forward If-None-Match" - ); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), - "suppressed requests must not forward If-Modified-Since" - ); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed requests must not forward {header_name}" + ); + } } #[tokio::test] @@ -5529,6 +5538,8 @@ mod tests { .header("fastly-surrogate-control", "max-age=600") .header("cloudflare-cdn-cache-control", "max-age=600") .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); @@ -5566,6 +5577,12 @@ mod tests { response.headers().get("cdn-cache-control").is_none(), "suppressed HTML should not retain CDN-Cache-Control" ); + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "suppressed HTML should not retain {header_name}" + ); + } let mut no_store_response = Response::builder() .status(StatusCode::OK) diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 2205ae892..c2b1447e3 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,7 +22,6 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", - "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 35eda5d3e..ff2eb6aaa 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -169,7 +169,7 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. -8. The request does not contain a matching enabled `protection_test_bypass` credential. +8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. @@ -182,6 +182,7 @@ can configure a static header credential that skips only the server-side Protection API: ```toml +# Runtime activation also requires FASTLY_IS_STAGING=1. [integrations.datadome.protection_test_bypass] enabled = true credential_secret_store = "ts_secrets" @@ -189,14 +190,17 @@ credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted. Store the temporary credential in the configured Secret Store, -configure this section only while needed, protect the site with an outer access -control such as Basic Auth, and remove the section when testing finishes. Do not -enable it in production. - -The fixed `x-ts-datadome-bypass` header is compared in constant time, removed -before the request can reach DataDome or the publisher origin, and never -logged. Scope the header to the staging origin; do not attach it to every +when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. A retained +section cannot bypass protection in a production or other non-staging runtime. +Store the temporary credential in the configured Secret Store, configure this +section only while needed, protect the site with an outer access control such +as Basic Auth, and remove the section when testing finishes. + +Whenever the enabled DataDome request filter runs, the fixed +`x-ts-datadome-bypass` header is removed before configuration or credential +checks. It therefore cannot reach DataDome or the publisher origin when the +bypass is absent, disabled, inactive, or invalid. Active credentials are +compared in constant time and never logged. Scope the header to the staging origin; do not attach it to every request in a browser context because that can disclose the credential to third-party origins. With Playwright: @@ -223,7 +227,7 @@ This behavior applies to: - `protection_excluded_ip_cidr_sources`; - structured `ip_cidr` rules; - structured `ip_cidr_source` rules; and -- a matching enabled `protection_test_bypass` credential. +- a matching enabled `protection_test_bypass` credential in a staging runtime. ASN, method, path, query-parameter, static-asset, and internal-route exclusions do not automatically suppress the client-side tag. DataDome tags diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 0dfc923cf..2d9b0eae2 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -136,7 +136,6 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - [ ] **Step 3: Make `filter_protection_request` own a mutable input and pass it mutably to `is_request_protected`.** In the existing `ProtectionScopeDecision::Skip` arm: - 1. determine whether the reason is IP-based; 2. if so, insert the typed marker into `input.request.extensions_mut()`; 3. call the updated skip logger with `client_tag_omitted = true`; and @@ -160,7 +159,6 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" optional Config Store data, and a mutable request. For each case, call `filter_protection_request`, assert it returns `Continue`, and inspect the request extension: - - inline `protection_excluded_ip_cidrs` match → marker present; - `protection_excluded_ip_cidr_sources` match → marker present; - structured `ProtectionMatcherConfig::IpCidr` match → marker present; @@ -220,7 +218,6 @@ DataDomeClientTagSuppressed request extension - [ ] **Step 2: Add a boolean to the owned and borrowed publisher-processing parameter structs.** Add a clearly named field such as `suppress_datadome_client_side_tag` to: - - `OwnedProcessResponseParams`; - `ProcessResponseParams`; and - `HtmlStreamProcessorParams`. @@ -241,7 +238,6 @@ DataDomeClientTagSuppressed request extension - [ ] **Step 4: Extend `IntegrationHtmlContext`.** Add the boolean as immutable request-scoped context. Populate it at both construction sites in `html_processor.rs`: - - the streaming `` element handler; and - `HtmlWithPostProcessing::process_chunk` for full-document post-processors. @@ -249,7 +245,6 @@ DataDomeClientTagSuppressed request extension `false` by default. - [ ] **Step 5: Add plumbing tests.** - - `HtmlProcessorConfig::from_settings` defaults to non-suppressed. - A test head injector records the context flag and sees `true` when a config is built with suppression. @@ -332,7 +327,6 @@ optimization. with a processable HTML content type, suppression `true`, and cacheable origin headers (`Cache-Control`, `Surrogate-Control`, and `Fastly-Surrogate-Control`). Assert the stream response is: - - `Cache-Control: private, max-age=0`; and - missing both surrogate cache headers. @@ -343,7 +337,6 @@ optimization. 204/205/304, or responses without suppression: none has a body variation created by this feature. - [ ] **Step 3: Add non-regression cache tests.** Verify that: - - non-suppressed processed HTML keeps its existing cache headers unless another existing policy changes them; - a suppressed CSS/non-HTML stream is not made private by this feature; and From 73b700bf50afae07bfe89dbe0cd539550c18c4fb Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 12 Aug 2026 11:07:03 -0500 Subject: [PATCH 160/195] Harden DataDome exclusion and bypass handling --- .../src/integrations/datadome.rs | 9 +- .../src/integrations/datadome/protection.rs | 419 +++++++++++++++--- .../integrations/datadome/protection_scope.rs | 260 +++++++++-- .../src/platform/test_support.rs | 10 +- crates/trusted-server-core/src/publisher.rs | 256 ++++++++--- .../src/response_privacy.rs | 56 ++- docs/guide/integrations/datadome.md | 56 ++- ...6-08-03-datadome-ip-excluded-client-tag.md | 22 +- ...-datadome-ip-excluded-client-tag-design.md | 32 +- 9 files changed, 917 insertions(+), 203 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 75e6c3fdc..d95ee35ee 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -90,6 +90,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +/// Fixed request header used by the staging-only protection test bypass. pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic @@ -136,7 +137,7 @@ pub struct ProtectionTestBypassConfig { #[serde(default = "default_protection_test_bypass_secret_store")] pub credential_secret_store: String, - /// Secret name containing the temporary bypass credential. + /// Secret name containing at least 32 bytes of high-entropy bypass material. #[serde(default = "default_protection_test_bypass_secret_name")] pub credential_secret_name: String, } @@ -464,6 +465,12 @@ impl DataDomeIntegration { Ok(()) } + /// Validates `DataDome` configuration before runtime registration. + /// + /// # Errors + /// + /// Returns an error when protection, bypass, or client-tag configuration is + /// invalid. pub(crate) fn validate_config_for_startup( config: DataDomeConfig, ) -> Result<(), Report> { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index f2803b38f..665060014 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -9,6 +9,7 @@ use subtle::ConstantTimeEq as _; use url::Url; use crate::error::TrustedServerError; +use crate::http_util::is_navigation_request; use crate::integrations::{ HeaderMutation, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; @@ -16,7 +17,11 @@ use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, use crate::redacted::Redacted; use super::DataDomeIntegration; -use super::protection_scope::{ProtectionRequestFacts, ProtectionScopeDecision}; +use super::protection_scope::{ + ProtectionRequestFacts, ProtectionScopeDecision, ProtectionSkipReason, +}; + +const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; const VALIDATE_REQUEST_PATH: &str = "/validate-request"; const REQUEST_MODULE_NAME: &str = "Trusted-Server-Rust"; @@ -144,15 +149,18 @@ impl DataDomeIntegration { }; match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} - ProtectionScopeDecision::Skip { rule_id, reason } => { - let client_tag_omitted = is_ip_exclusion_reason(reason); - if client_tag_omitted { + ProtectionScopeDecision::Skip { + rule_id, + reason, + suppress_client_tag, + } => { + if suppress_client_tag { input .request .extensions_mut() .insert(super::DataDomeClientTagSuppressed); } - log_protection_skip(input, &rule_id, reason); + log_protection_skip(input, &rule_id, reason, suppress_client_tag); return false; } } @@ -165,11 +173,23 @@ impl DataDomeIntegration { req: &mut Request, services: &RuntimeServices, ) -> bool { - let value = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); - let Some(bypass) = self.active_protection_test_bypass() else { + let supplied_values = req + .headers() + .get_all(super::HEADER_DATADOME_TEST_BYPASS) + .iter() + .cloned() + .collect::>(); + req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); + if supplied_values.is_empty() { return false; - }; - let Some(value) = value else { + } + if supplied_values.len() != 1 { + log::warn!( + "[datadome] Multiple DataDome test bypass headers supplied; ignoring bypass" + ); + return false; + } + let Some(bypass) = self.active_protection_test_bypass() else { return false; }; @@ -178,10 +198,10 @@ impl DataDomeIntegration { .secret_store() .get_string(&store_name, &bypass.credential_secret_name) { - Ok(credential) if !credential.is_empty() => credential, + Ok(credential) if credential.len() >= MIN_TEST_BYPASS_CREDENTIAL_BYTES => credential, Ok(_) => { log::warn!( - "[datadome] DataDome test bypass credential is empty; ignoring bypass header" + "[datadome] DataDome test bypass credential does not meet security requirements; ignoring bypass header" ); return false; } @@ -193,7 +213,7 @@ impl DataDomeIntegration { } }; - let actual = Sha256::digest(value.as_bytes()); + let actual = Sha256::digest(supplied_values[0].as_bytes()); let expected = Sha256::digest(credential.as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -421,13 +441,16 @@ impl DataDomeIntegration { let (parts, body) = response.into_parts(); let status = parts.status; let Some(datadome_status) = datadome_response_status(&parts.headers) else { - log::warn!("[datadome] Protection API response missing X-DataDomeResponse"); + log::warn!( + "[datadome] Protection API response has missing or non-numeric verdict: api_status={} datadome_status=missing_or_invalid", + status.as_u16() + ); return RequestFilterDecision::Continue(RequestFilterEffects::default()); }; if datadome_status != status.as_u16() { log::warn!( - "[datadome] Protection API status/header mismatch: status={} header={}", + "[datadome] Protection API status/verdict mismatch: api_status={} datadome_status={}", status.as_u16(), datadome_status ); @@ -470,20 +493,14 @@ impl DataDomeIntegration { } log::warn!( - "[datadome] Protection API returned fail-open status {}", - status.as_u16() + "[datadome] Protection API returned unexpected fail-open status: api_status={} datadome_status={}", + status.as_u16(), + datadome_status ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } } -fn is_ip_exclusion_reason(reason: &str) -> bool { - matches!( - reason, - "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - ) -} - fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { log::info!( "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={}", @@ -491,14 +508,37 @@ fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { ); } -fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { - if is_ip_exclusion_reason(reason) { +fn suppression_skip_log_level(suppress_client_tag: bool, is_navigation: bool) -> log::Level { + if suppress_client_tag && is_navigation { + log::Level::Info + } else { + log::Level::Debug + } +} + +fn log_protection_skip( + input: &RequestFilterInput<'_>, + rule_id: &str, + reason: ProtectionSkipReason, + suppress_client_tag: bool, +) { + let reason = reason.as_str(); + if suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)) + == log::Level::Info + { log::info!( "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", rule_id, reason, input.request.method(), ); + } else if suppress_client_tag { + log::debug!( + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", + rule_id, + reason, + input.request.method(), + ); } else { log::debug!( "[datadome] protection decision=skipped rule={} reason={} method={}", @@ -509,6 +549,29 @@ fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &s } } +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum ProtectionResultKind { + Allowed, + Blocked, + FailedOpen, +} + +fn classify_logged_protection_result( + status: StatusCode, + datadome_status: Option, + decision: &RequestFilterDecision, +) -> ProtectionResultKind { + match decision { + RequestFilterDecision::Respond { .. } => ProtectionResultKind::Blocked, + RequestFilterDecision::Continue(_) + if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => + { + ProtectionResultKind::Allowed + } + RequestFilterDecision::Continue(_) => ProtectionResultKind::FailedOpen, + } +} + fn log_protection_result( input: &RequestFilterInput<'_>, status: StatusCode, @@ -516,23 +579,30 @@ fn log_protection_result( decision: &RequestFilterDecision, ) { let method = input.request.method(); - - match decision { - RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} route=short_circuit", + let result_kind = classify_logged_protection_result(status, datadome_status, decision); + let datadome_status = datadome_status + .map(|value| value.to_string()) + .unwrap_or_else(|| "missing_or_invalid".to_string()); + + match result_kind { + ProtectionResultKind::Blocked => log::info!( + "[datadome] protection decision=blocked api_status={} datadome_status={} method={} route=short_circuit", status.as_u16(), + datadome_status, + method, + ), + ProtectionResultKind::Allowed => log::info!( + "[datadome] protection decision=allowed api_status={} datadome_status={} method={} route=continue", + status.as_u16(), + datadome_status, + method, + ), + ProtectionResultKind::FailedOpen => log::warn!( + "[datadome] protection decision=failed_open api_status={} datadome_status={} method={} route=continue", + status.as_u16(), + datadome_status, method, ), - RequestFilterDecision::Continue(_) - if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => - { - log::info!( - "[datadome] protection decision=allowed status={} method={} route=continue", - status.as_u16(), - method, - ); - } - RequestFilterDecision::Continue(_) => {} } } @@ -830,11 +900,24 @@ mod tests { config: DataDomeConfig, services: &RuntimeServices, geo_info: Option<&GeoInfo>, + ) -> Request { + filter_marks_request_for_uri(config, services, geo_info, "https://publisher.example/page") + } + + fn filter_marks_request_for_uri( + config: DataDomeConfig, + services: &RuntimeServices, + geo_info: Option<&GeoInfo>, + uri: &str, ) -> Request { let integration = DataDomeIntegration::try_new(config).expect("should create DataDome integration"); let settings = Settings::default(); - let mut request = request_for_filter(); + let mut request = request_builder() + .method(Method::GET.as_str()) + .uri(uri) + .body(EdgeBody::empty()) + .expect("should build filter request"); let decision = futures::executor::block_on(integration.filter_protection_request( RequestFilterInput { settings: &settings, @@ -874,7 +957,7 @@ mod tests { let mut secrets = HashMap::new(); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); let services = build_services_with_secret_and_http_client( @@ -885,7 +968,7 @@ mod tests { let mut request = request_for_filter(); request.headers_mut().insert( super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); let decision = filter_with_staging(&integration, &settings, &services, &mut request); @@ -996,7 +1079,7 @@ mod tests { ); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( @@ -1012,7 +1095,7 @@ mod tests { let mut request = request_for_filter(); request.headers_mut().insert( super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); let decision = temp_env::with_var( @@ -1077,7 +1160,7 @@ mod tests { let mut secrets = HashMap::new(); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); let services = build_services_with_secret_and_http_client( @@ -1088,7 +1171,7 @@ mod tests { let mut request = request_for_filter(); request.headers_mut().insert( super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); let decision = filter_with_staging(&integration, &settings, &services, &mut request); @@ -1127,7 +1210,7 @@ mod tests { ); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( @@ -1170,6 +1253,158 @@ mod tests { ); } + #[test] + fn duplicate_test_bypass_headers_fail_closed_and_are_all_stripped() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential-32-bytes!".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + for value in [ + "temporary-test-credential-32-bytes!", + "temporary-test-credential-32-bytes!", + ] { + request.headers_mut().append( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static(value), + ); + } + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "all duplicate bypass values should be stripped" + ); + assert!(!has_client_tag_suppression_marker(&request)); + assert_eq!(http_client.recorded_backend_names().len(), 1); + } + + #[test] + fn test_bypass_credential_requires_at_least_32_bytes() { + for (credential, should_match) in [ + (Some("1234567890123456789012345678901"), false), + (Some("12345678901234567890123456789012"), true), + (Some(""), false), + (None, false), + ] { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + if let Some(credential) = credential { + secrets.insert( + "datadome_test_bypass".to_string(), + credential.as_bytes().to_vec(), + ); + } + let http_client = Arc::new(StubHttpClient::new()); + if !should_match { + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + } + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + let supplied = credential.unwrap_or("12345678901234567890123456789012"); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_str(supplied) + .expect("should build bypass header"), + ); + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + assert_eq!(has_client_tag_suppression_marker(&request), should_match); + assert_eq!( + http_client.recorded_backend_names().is_empty(), + should_match, + "only a credential meeting the minimum should skip the API" + ); + } + } + + #[test] + fn protection_result_classifier_and_suppression_log_level_cover_outcomes() { + let continue_decision = RequestFilterDecision::Continue(RequestFilterEffects::default()); + let blocked_decision = RequestFilterDecision::Respond { + response: Box::new(Response::new(EdgeBody::empty())), + effects: RequestFilterEffects::default(), + }; + + assert_eq!( + classify_logged_protection_result(StatusCode::OK, Some(200), &continue_decision), + ProtectionResultKind::Allowed + ); + assert_eq!( + classify_logged_protection_result(StatusCode::FORBIDDEN, Some(403), &blocked_decision), + ProtectionResultKind::Blocked + ); + for (status, datadome_status) in [ + (StatusCode::OK, None), + (StatusCode::OK, Some(403)), + (StatusCode::CREATED, Some(201)), + ] { + assert_eq!( + classify_logged_protection_result(status, datadome_status, &continue_decision), + ProtectionResultKind::FailedOpen + ); + } + assert_eq!(suppression_skip_log_level(true, true), log::Level::Info); + assert_eq!(suppression_skip_log_level(true, false), log::Level::Debug); + } + #[test] fn ip_exclusions_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); @@ -1257,29 +1492,89 @@ mod tests { #[test] fn non_ip_exclusions_do_not_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); - let cases = [DataDomeConfig { - enabled: true, - enable_protection: true, - protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { - id: "path".to_string(), - enabled: true, - methods: Vec::new(), - matcher: ProtectionMatcherConfig::PathExact { - paths: vec!["/page".to_string()], + let cases = [ + ( + ProtectionMatcherConfig::PathExact { + paths: vec!["/exact".to_string()], }, - }], - ..DataDomeConfig::default() - }]; + "https://publisher.example/exact", + ), + ( + ProtectionMatcherConfig::PathPrefix { + prefixes: vec!["/prefix/".to_string()], + }, + "https://publisher.example/prefix/page", + ), + ( + ProtectionMatcherConfig::PathRegex { + patterns: vec![r"^/regex/[0-9]+$".to_string()], + }, + "https://publisher.example/regex/42", + ), + ( + ProtectionMatcherConfig::QueryParamNonEmpty { + names: vec!["skip".to_string()], + }, + "https://publisher.example/page?skip=yes", + ), + ]; - for config in cases { - let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + for (matcher, uri) in cases { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "non-ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher, + }], + ..DataDomeConfig::default() + }; + let request = + filter_marks_request_for_uri(config, &noop_services_with_client_ip(ip), None, uri); assert!( !has_client_tag_suppression_marker(&request), - "non-IP exclusions should not mark the request" + "matching non-IP exclusion should not mark {uri}" ); } } + #[test] + fn overlapping_path_and_ip_exclusions_still_mark_request() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ + ProtectionExclusionRuleConfig { + id: "path-first".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }, + ProtectionExclusionRuleConfig { + id: "ip-second".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }, + ], + ..DataDomeConfig::default() + }; + + let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + + assert!( + has_client_tag_suppression_marker(&request), + "overlapping IP exclusion should suppress even when path remains the primary reason" + ); + } + #[test] fn asn_exclusions_do_not_mark_requests_for_client_tag_suppression() { let config = DataDomeConfig { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs b/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs index cce56f2fa..a83f73217 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs @@ -91,12 +91,52 @@ pub(super) struct ProtectionRequestFacts<'a> { pub(super) asn: Option, } +/// Typed reason why `DataDome` protection was skipped. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum ProtectionSkipReason { + Method, + ClientIp, + ClientIpSource, + Asn, + PathExact, + PathPrefix, + PathRegex, + QueryParamNonEmpty, + IpCidr, + IpCidrSource, +} + +impl ProtectionSkipReason { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Method => "method", + Self::ClientIp => "client_ip", + Self::ClientIpSource => "client_ip_source", + Self::Asn => "asn", + Self::PathExact => "path_exact", + Self::PathPrefix => "path_prefix", + Self::PathRegex => "path_regex", + Self::QueryParamNonEmpty => "query_param_non_empty", + Self::IpCidr => "ip_cidr", + Self::IpCidrSource => "ip_cidr_source", + } + } + + pub(super) const fn is_ip_based(self) -> bool { + matches!( + self, + Self::ClientIp | Self::ClientIpSource | Self::IpCidr | Self::IpCidrSource + ) + } +} + /// Result of evaluating whether `DataDome` protection should run. pub(super) enum ProtectionScopeDecision { Protect, Skip { rule_id: String, - reason: &'static str, + reason: ProtectionSkipReason, + suppress_client_tag: bool, }, } @@ -210,51 +250,75 @@ impl ProtectionScope { facts: &ProtectionRequestFacts<'_>, services: &RuntimeServices, ) -> ProtectionScopeDecision { - if self.excluded_methods.matches(facts.method) { - return ProtectionScopeDecision::Skip { - rule_id: "excluded-methods".to_string(), - reason: "method", - }; - } + let mut primary = self + .excluded_methods + .matches(facts.method) + .then(|| ("excluded-methods".to_string(), ProtectionSkipReason::Method)); if let Some(client_ip) = facts.client_ip { if cidrs_match(&self.excluded_ip_cidrs, client_ip) { + let (rule_id, reason) = primary.unwrap_or_else(|| { + ( + "excluded-ip-cidrs".to_string(), + ProtectionSkipReason::ClientIp, + ) + }); return ProtectionScopeDecision::Skip { - rule_id: "excluded-ip-cidrs".to_string(), - reason: "client_ip", + rule_id, + reason, + suppress_client_tag: true, }; } for source in &self.excluded_ip_cidr_sources { if source.matches(client_ip, services, self.ip_list_cache_ttl) { + let (rule_id, reason) = primary.unwrap_or_else(|| { + (source.rule_id(), ProtectionSkipReason::ClientIpSource) + }); return ProtectionScopeDecision::Skip { - rule_id: source.rule_id(), - reason: "client_ip_source", + rule_id, + reason, + suppress_client_tag: true, }; } } } - if facts - .asn - .is_some_and(|asn| self.excluded_asns.contains(&asn)) + if primary.is_none() + && facts + .asn + .is_some_and(|asn| self.excluded_asns.contains(&asn)) { - return ProtectionScopeDecision::Skip { - rule_id: "excluded-asns".to_string(), - reason: "asn", - }; + primary = Some(("excluded-asns".to_string(), ProtectionSkipReason::Asn)); } for rule in &self.exclusion_rules { + let reason = rule.matcher.reason(); + if primary.is_some() && !reason.is_ip_based() { + continue; + } if rule.matches(facts, services, self.ip_list_cache_ttl) { - return ProtectionScopeDecision::Skip { - rule_id: rule.id.clone(), - reason: rule.matcher.reason(), - }; + let suppress_client_tag = reason.is_ip_based(); + let (rule_id, reason) = primary.unwrap_or_else(|| (rule.id.clone(), reason)); + if suppress_client_tag { + return ProtectionScopeDecision::Skip { + rule_id, + reason, + suppress_client_tag, + }; + } + primary = Some((rule_id, reason)); } } - ProtectionScopeDecision::Protect + match primary { + Some((rule_id, reason)) => ProtectionScopeDecision::Skip { + rule_id, + reason, + suppress_client_tag: false, + }, + None => ProtectionScopeDecision::Protect, + } } } @@ -489,15 +553,15 @@ impl ProtectionMatcher { } } - fn reason(&self) -> &'static str { + fn reason(&self) -> ProtectionSkipReason { match self { - ProtectionMatcher::PathExact(_) => "path_exact", - ProtectionMatcher::PathPrefix(_) => "path_prefix", - ProtectionMatcher::PathRegex(_) => "path_regex", - ProtectionMatcher::QueryParamNonEmpty(_) => "query_param_non_empty", - ProtectionMatcher::Asn(_) => "asn", - ProtectionMatcher::IpCidr(_) => "ip_cidr", - ProtectionMatcher::IpCidrSource(_) => "ip_cidr_source", + ProtectionMatcher::PathExact(_) => ProtectionSkipReason::PathExact, + ProtectionMatcher::PathPrefix(_) => ProtectionSkipReason::PathPrefix, + ProtectionMatcher::PathRegex(_) => ProtectionSkipReason::PathRegex, + ProtectionMatcher::QueryParamNonEmpty(_) => ProtectionSkipReason::QueryParamNonEmpty, + ProtectionMatcher::Asn(_) => ProtectionSkipReason::Asn, + ProtectionMatcher::IpCidr(_) => ProtectionSkipReason::IpCidr, + ProtectionMatcher::IpCidrSource(_) => ProtectionSkipReason::IpCidrSource, } } } @@ -741,7 +805,8 @@ mod tests { assert!(matches!( decision, ProtectionScopeDecision::Skip { - reason: "method", + reason: ProtectionSkipReason::Method, + suppress_client_tag: false, .. } )); @@ -758,7 +823,11 @@ mod tests { assert!(matches!( decision, - ProtectionScopeDecision::Skip { reason: "asn", .. } + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Asn, + suppress_client_tag: false, + .. + } )); } @@ -783,7 +852,8 @@ mod tests { assert!(matches!( decision, ProtectionScopeDecision::Skip { - reason: "client_ip", + reason: ProtectionSkipReason::ClientIp, + suppress_client_tag: true, .. } )); @@ -817,7 +887,8 @@ mod tests { assert!(matches!( decision, ProtectionScopeDecision::Skip { - reason: "client_ip_source", + reason: ProtectionSkipReason::ClientIpSource, + suppress_client_tag: true, .. } )); @@ -840,7 +911,8 @@ mod tests { assert!(matches!( scope.evaluate(&facts("GET", "/app.JSON", None, None, None), &services), ProtectionScopeDecision::Skip { - reason: "path_regex", + reason: ProtectionSkipReason::PathRegex, + suppress_client_tag: false, .. } )); @@ -850,6 +922,119 @@ mod tests { )); } + #[test] + fn method_skip_detects_overlapping_inline_ip_exclusion() { + let mut config = config_with_protection(); + config.protection_excluded_methods = vec!["GET".to_string()]; + config.protection_excluded_ip_cidrs = vec!["192.0.2.0/24".to_string()]; + let scope = ProtectionScope::compile(&config).expect("should compile scope"); + let services = crate::platform::test_support::noop_services(); + + let decision = scope.evaluate( + &facts( + "GET", + "/page", + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + None, + ), + &services, + ); + + assert!(matches!( + decision, + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Method, + suppress_client_tag: true, + .. + } + )); + } + + #[test] + fn earlier_non_ip_skip_detects_later_structured_ip_exclusion() { + for primary_asn in [None, Some(64500)] { + let mut config = config_with_protection(); + config.protection_excluded_asns = primary_asn.into_iter().collect(); + config.protection_exclusion_rules = vec![ + ProtectionExclusionRuleConfig { + id: "path".to_string(), + enabled: primary_asn.is_none(), + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }, + ProtectionExclusionRuleConfig { + id: "ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }, + ]; + let scope = ProtectionScope::compile(&config).expect("should compile scope"); + let services = crate::platform::test_support::noop_services(); + + let decision = scope.evaluate( + &facts( + "GET", + "/page", + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + primary_asn, + ), + &services, + ); + + assert!(matches!( + decision, + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Asn | ProtectionSkipReason::PathExact, + suppress_client_tag: true, + .. + } + )); + } + } + + #[test] + fn method_scoped_structured_ip_does_not_suppress_when_scope_does_not_apply() { + let mut config = config_with_protection(); + config.protection_excluded_methods = vec!["POST".to_string()]; + config.protection_exclusion_rules = vec![ProtectionExclusionRuleConfig { + id: "get-ip".to_string(), + enabled: true, + methods: vec!["GET".to_string()], + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }]; + let scope = ProtectionScope::compile(&config).expect("should compile scope"); + let services = crate::platform::test_support::noop_services(); + + let decision = scope.evaluate( + &facts( + "POST", + "/page", + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + None, + ), + &services, + ); + + assert!(matches!( + decision, + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Method, + suppress_client_tag: false, + .. + } + )); + } + #[test] fn rule_query_param_non_empty_matches_rsc() { let mut config = config_with_protection(); @@ -870,7 +1055,8 @@ mod tests { &services ), ProtectionScopeDecision::Skip { - reason: "query_param_non_empty", + reason: ProtectionSkipReason::QueryParamNonEmpty, + suppress_client_tag: false, .. } )); diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 10389c787..917f1bf50 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -769,6 +769,14 @@ pub(crate) fn build_services_with_backend_and_http_client( pub(crate) fn build_services_with_secret_and_http_client( secret_store: impl PlatformSecretStore + 'static, http_client: Arc, +) -> RuntimeServices { + build_services_with_secret_http_client_and_client_ip(secret_store, http_client, None) +} + +pub(crate) fn build_services_with_secret_http_client_and_client_ip( + secret_store: impl PlatformSecretStore + 'static, + http_client: Arc, + client_ip: Option, ) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(NoopConfigStore)) @@ -778,7 +786,7 @@ pub(crate) fn build_services_with_secret_and_http_client( .http_client(http_client) .geo(Arc::new(NoopGeo)) .client_info(ClientInfo { - client_ip: None, + client_ip, tls_protocol: None, tls_cipher: None, ..ClientInfo::default() diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 66b8a9553..4bed98327 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -59,7 +59,7 @@ use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_eta use crate::integrations::IntegrationRegistry; use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::CDN_CACHE_HEADERS; +use crate::response_privacy::enforce_synthesized_html_cache_privacy; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -1424,6 +1424,22 @@ pub async fn publisher_response_into_streaming_response( } } +/// Returns whether a request can render an HTML document context. +fn is_html_document_request(req: &Request) -> bool { + if let Some(destination) = req + .headers() + .get("sec-fetch-dest") + .and_then(|value| value.to_str().ok()) + { + return matches!( + destination.trim().to_ascii_lowercase().as_str(), + "document" | "embed" | "fencedframe" | "frame" | "iframe" | "object" + ); + } + + is_navigation_request(req) +} + /// Removes request headers that can produce a bodyless or partial origin response. fn strip_conditional_and_range_headers(req: &mut Request) { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -1454,29 +1470,11 @@ fn apply_datadome_client_tag_cache_privacy( suppress_datadome_client_side_tag: bool, content_type: &str, ) { - if !suppress_datadome_client_side_tag - || !response_carries_body(method, response.status()) - || !is_html_content_type(content_type) + if suppress_datadome_client_side_tag + && response_carries_body(method, response.status()) + && is_html_content_type(content_type) { - return; - } - - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|value| value.contains("private") || value.contains("no-store")); - if !already_uncacheable { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - } - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - for header_name in CDN_CACHE_HEADERS { - response.headers_mut().remove(*header_name); + enforce_synthesized_html_cache_privacy(response); } } @@ -2900,10 +2898,11 @@ pub async fn handle_publisher_request( .extensions() .get::() .is_some(); - if should_run_ad_stack || suppress_datadome_client_side_tag { - // The origin content type is not known yet, so request hints cannot safely - // narrow this to HTML without allowing 304 or 206 responses to bypass a - // response mutation that becomes necessary after the fetch. + if should_run_ad_stack || (suppress_datadome_client_side_tag && is_html_document_request(&req)) + { + // HTML document contexts whose output may be synthesized must not + // receive a cached 304 or partial 206. Non-document subresources contain + // no executable injected tag, so retain their validators and ranges. strip_conditional_and_range_headers(&mut req); } @@ -3015,23 +3014,7 @@ pub async fn handle_publisher_request( .and_then(|h| h.to_str().ok()) .unwrap_or_default(); if should_run_ad_stack && is_html_content_type(origin_content_type) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - // Every CDN-targeted cache directive, not just the browser-facing - // `Cache-Control` above: an origin emitting any of these would otherwise - // instruct an intermediary to store a synthesized per-navigation - // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover - // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) - // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field - // that overrides it there, so both are needed to close the gap on the - // Cloudflare adapter. - for directive in CDN_CACHE_HEADERS { - response.headers_mut().remove(*directive); - } + enforce_synthesized_html_cache_privacy(&mut response); } let content_type = response @@ -4218,7 +4201,8 @@ mod tests { use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; use crate::platform::test_support::{ - StubHttpClient, build_services_with_http_client, noop_services, + NoopSecretStore, StubHttpClient, build_services_with_http_client, + build_services_with_secret_http_client_and_client_ip, noop_services, noop_services_with_telemetry_sink, }; use crate::test_support::tests::create_test_settings; @@ -5386,7 +5370,7 @@ mod tests { } #[tokio::test] - async fn suppressed_publisher_request_removes_conditional_validators() { + async fn suppressed_navigation_removes_conditional_and_range_headers() { let settings = create_test_settings(); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( @@ -5401,6 +5385,7 @@ mod tests { .method(Method::GET) .uri("https://publisher.example/page") .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") .header(header::IF_NONE_MATCH, "\"cached-page\"") .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") .header(header::RANGE, "bytes=0-18") @@ -5427,7 +5412,105 @@ mod tests { headers .iter() .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed requests must not forward {header_name}" + "suppressed navigations must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn suppressed_iframe_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"frame".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/frame") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "iframe") + .header(header::IF_NONE_MATCH, "\"cached-frame\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-frame\"") + .body(EdgeBody::empty()) + .expect("should build conditional iframe request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed iframe documents must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn suppressed_subresource_preserves_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"video".to_vec(), + vec![("content-type", "video/mp4")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/video.mp4") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "video") + .header(header::IF_NONE_MATCH, "\"cached-video\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-video\"") + .body(EdgeBody::empty()) + .expect("should build conditional subresource request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for (header_name, expected) in [ + (header::IF_NONE_MATCH, "\"cached-video\""), + (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), + (header::RANGE, "bytes=0-18"), + (header::IF_RANGE, "\"cached-video\""), + ] { + assert_eq!( + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) + .map(|(_, value)| value.as_str()), + Some(expected), + "suppressed subresources should preserve {header_name}" ); } } @@ -5547,6 +5630,77 @@ mod tests { ); } + #[tokio::test] + async fn datadome_filter_marker_survives_into_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": true, + "protection_excluded_ip_cidrs": ["192.0.2.0/24"], + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"content".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_secret_http_client_and_client_ip( + NoopSecretStore, + Arc::clone(&stub) as Arc, + Some("192.0.2.10".parse().expect("should parse client IP")), + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let filter_outcome = registry + .filter_request(crate::integrations::RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + }) + .await + .expect("should run DataDome filter"); + assert!(matches!( + filter_outcome, + crate::integrations::RequestFilterRegistryOutcome::Continue(_) + )); + let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let response = buffer_publisher_response_async( + publisher_response, + &Method::GET, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &services, + ) + .await + .expect("should buffer publisher response"); + let html = response_body_string(response); + + assert!(!html.contains("window.ddjskey")); + assert!(!html.contains("/integrations/datadome/tags.js")); + assert_eq!( + stub.recorded_backend_names().len(), + 1, + "only the publisher origin should be called" + ); + } + #[test] fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { let mut settings = create_test_settings(); @@ -5613,8 +5767,8 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("private, max-age=0"), - "suppressed HTML should be private" + Some("private, no-store"), + "suppressed HTML should be private and non-storable" ); assert!( response.headers().get("surrogate-control").is_none(), @@ -5658,8 +5812,8 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store"), - "suppressed HTML should preserve an existing no-store policy" + Some("private, no-store"), + "suppressed HTML should use the exact synthesized-HTML policy" ); } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index c2b1447e3..21ba9f20b 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -24,6 +24,27 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "cloudflare-cdn-cache-control", ]; +fn strip_cdn_cache_headers(response: &mut Response) { + for name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*name); + } +} + +/// Forces synthesized HTML to be private and non-storable. +/// +/// Use this exact policy whenever Trusted Server changes an origin HTML +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. +pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); + strip_cdn_cache_headers(response); +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -41,9 +62,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - for name in CDN_CACHE_HEADERS { - response.headers_mut().remove(*name); - } + strip_cdn_cache_headers(response); // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. let already_uncacheable = response @@ -149,6 +168,37 @@ mod tests { s } + #[test] + fn synthesized_html_is_forced_no_store_without_validators_or_cdn_headers() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "private, max-age=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_synthesized_html_cache_privacy(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store", + "synthesized HTML should always be non-storable" + ); + for header_name in [header::ETAG.as_str(), header::LAST_MODIFIED.as_str()] + .into_iter() + .chain(CDN_CACHE_HEADERS.iter().copied()) + { + assert!( + !response.headers().contains_key(header_name), + "synthesized HTML should remove {header_name}" + ); + } + } + #[test] fn downgrades_public_cache_control_on_cookie_response() { let settings = settings_with_response_headers(&[("cache-control", "public, max-age=600")]); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index ff2eb6aaa..0c2d8ae6c 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -86,7 +86,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | | `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | | `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | -| `protection_test_bypass` | object | omitted | Temporary static-header bypass for access-controlled staging tests | +| `protection_test_bypass` | object | omitted | Staging-only fixed-header bypass; secret must contain at least 32 bytes | | `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | | `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | | `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | @@ -190,19 +190,25 @@ credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. A retained +when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. +`FASTLY_IS_STAGING` is supplied at runtime by Fastly (`1` in staging and `0` in +production); it is not compiled into or promoted with the Wasm artifact. Verify +staging through the `X-TS-ENV: staging` response signal and the integration +activation log, and verify production omits that response signal. A retained section cannot bypass protection in a production or other non-staging runtime. -Store the temporary credential in the configured Secret Store, configure this -section only while needed, protect the site with an outer access control such -as Basic Auth, and remove the section when testing finishes. - -Whenever the enabled DataDome request filter runs, the fixed -`x-ts-datadome-bypass` header is removed before configuration or credential -checks. It therefore cannot reach DataDome or the publisher origin when the -bypass is absent, disabled, inactive, or invalid. Active credentials are -compared in constant time and never logged. Scope the header to the staging origin; do not attach it to every -request in a browser context because that can disclose the credential to -third-party origins. With Playwright: +Store a randomly generated credential containing at least 32 bytes of +high-entropy material in the configured Secret Store, configure this section +only while needed, protect the site with an outer access control such as Basic +Auth, and remove the section when testing finishes. + +Whenever the enabled DataDome request filter runs on the Fastly adapter, the +fixed `x-ts-datadome-bypass` header is removed before configuration or +credential checks. It therefore cannot reach DataDome or the publisher origin +through that path when the bypass is absent, disabled, inactive, or invalid. +Active credentials are compared in constant time and never logged. Duplicate +header values fail closed. Scope the header to the staging origin; do not attach +it to every request in a browser context because that can disclose the +credential to third-party origins. With Playwright: ```ts await context.route('https://staging.example.com/**', async (route) => { @@ -229,15 +235,25 @@ This behavior applies to: - structured `ip_cidr_source` rules; and - a matching enabled `protection_test_bypass` credential in a staging runtime. -ASN, method, path, query-parameter, static-asset, and internal-route -exclusions do not automatically suppress the client-side tag. DataDome tags -already present in publisher HTML are not removed or changed by this behavior, -and `/integrations/datadome/tags.js` remains available when requested directly. +Method, ASN, path, query-parameter, static-asset, and internal-route exclusions +alone do not suppress the client-side tag. However, a simultaneous matching IP +exclusion suppresses it regardless of which first-match rule and reason are +logged. DataDome tags already present in publisher HTML are not removed or +changed by this behavior, and `/integrations/datadome/tags.js` remains available +when requested directly. Because the processed HTML differs by client IP or test credential, -tag-suppressed HTML is marked `private, max-age=0` and removed from shared -surrogate caches. The decision is reported in the existing protection log, for -example: +tag-suppressed HTML is marked `private, no-store`, has origin validators +removed, and has shared-surrogate cache directives removed. This response-time +policy cannot invalidate tag-bearing HTML already held by a shared cache in +front of Trusted Server. Guaranteed suppression requires bypassing or purging +that cache, or avoiding shared caching ahead of Trusted Server. + +IP-exclusion suppression skips are logged at `info` for navigations and `debug` +for subresources; matching test-bypass events remain at `info` as security audit +events. Protection API result logs classify `allowed`, `blocked`, and +`failed_open` outcomes and use distinct `api_status` and `datadome_status` +fields. For example: ```text [datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 2d9b0eae2..9f2c1674f 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -40,10 +40,9 @@ request-filter wiring. 3. **Private marker:** communicate the decision with a typed request extension, never a request/response header. The marker cannot leak to the origin or client. -4. **Precise scope:** tag suppression is keyed only on decision reasons - `client_ip`, `client_ip_source`, `ip_cidr`, and `ip_cidr_source`. +4. **Precise scope:** tag suppression uses typed scope metadata and applies whenever an IP exclusion overlaps the primary first-match skip reason. 5. **Cache safety:** an HTML response with the tag omitted differs by client IP. - A suppressed processed HTML response must be `private, max-age=0` and have + A suppressed processed HTML response must be `private, no-store` and have `Surrogate-Control` and `Fastly-Surrogate-Control` removed. Do not alter cache headers when the response is not processed HTML, because this feature does not alter that body. @@ -183,9 +182,11 @@ cargo test-fastly datadome::protection cargo test-fastly datadome::protection_scope ``` -**Acceptance:** only the four IP decision reasons add the private marker and -produce the augmented informational skip log; all other exclusion and fail-open -paths keep their current tag behavior. +**Acceptance:** typed scope metadata adds the private marker whenever an +applicable IP exclusion matches, including overlap with an earlier primary skip +reason. Navigation suppression uses the augmented informational log, +subresources use debug, and non-overlapping/fail-open paths keep their current +tag behavior. --- @@ -327,7 +328,7 @@ optimization. with a processable HTML content type, suppression `true`, and cacheable origin headers (`Cache-Control`, `Surrogate-Control`, and `Fastly-Surrogate-Control`). Assert the stream response is: - - `Cache-Control: private, max-age=0`; and + - `Cache-Control: private, no-store`; and - missing both surrogate cache headers. - [ ] **Step 2: Apply privacy only in the `ResponseRoute::Stream` HTML arm.** @@ -349,9 +350,10 @@ optimization. cargo test-fastly publisher ``` -**Acceptance:** a shared cache cannot replay an IP-excluded client's tagless -HTML to a non-excluded visitor, while unchanged responses retain their existing -cacheability. +**Acceptance:** newly synthesized tagless HTML is private and non-storable, +while unchanged responses retain their existing cacheability. A shared cache in +front of Trusted Server that already holds tag-bearing HTML must be bypassed or +purged for guaranteed suppression. --- diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md index d6d811780..0bc7a856d 100644 --- a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -112,16 +112,11 @@ found in publisher HTML. That behavior must remain unchanged. ### 1. Capture an IP-exclusion marker at the request filter The request filter must attach a typed, internal request-scoped marker when the -existing protection-scope evaluation returns a skip for one of these reasons: - -- `client_ip` -- `client_ip_source` -- `ip_cidr` -- `ip_cidr_source` - -The marker must be attached only after the existing scope decision confirms the -IP exclusion. It must not be inferred from request headers or recomputed later -in the HTML pipeline. +existing protection-scope evaluation reports `suppress_client_tag` metadata. +The scope preserves the current first-match rule ID and typed reason for +logging, while independently checking applicable IP exclusions when an earlier +method, ASN, path, or query rule already matched. The marker must not be +inferred from request headers or recomputed later in the HTML pipeline. The request-filter API currently exposes an immutable request view. Add the smallest internal mechanism needed for a filter to attach a typed request @@ -134,19 +129,17 @@ suppressed; the existing skip log supplies the rule ID and reason. The marker must not be attached for: -- `OPTIONS` or other excluded methods before scope evaluation; - internal or integration routes; -- ASN exclusions; -- path, query, or other non-IP structured exclusions; -- unmatched IP rules; +- method, ASN, path, or query exclusions without an overlapping IP match; +- unmatched or method-inapplicable IP rules; - Protection API fail-open behavior; or - requests where `enable_protection` is false and the request filter does not run. ### 2. Enrich the existing skip log -For IP-based skips, extend the existing informational log with -`client_tag=omitted`: +For navigation skips that suppress the tag, extend the existing informational +log with `client_tag=omitted` (subresource suppression skips use `debug`): ```text [datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET @@ -184,8 +177,11 @@ responses, which should retain their current processing. A processed HTML response differs by client IP when the generated tag is suppressed. In the `PublisherResponse::Stream` path, when suppression is active -and the response is HTML, set `Cache-Control: private, max-age=0` and remove -`Surrogate-Control` and `Fastly-Surrogate-Control` before the body is streamed. +and the response is HTML, use the shared synthesized-HTML policy: +`Cache-Control: private, no-store`, no origin validators, and no CDN-targeted +cache headers. This response-time policy cannot invalidate tag-bearing HTML +already held by a fronting shared cache; guaranteed suppression requires +bypassing or purging that cache. This matches the existing per-user ad-stack cache policy. It prevents Fastly or another shared cache from replaying a tag-suppressed response to a visitor whose From 806b5644ce6ea5a47725a5dca13831381324b0cc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 13 Aug 2026 11:31:41 +0530 Subject: [PATCH 161/195] Document GPT diagnostics attribution and correlation evidence Complete the public guide against the behavior merged in #997. Cover the per-auction hb_auction_id correlation token and the targeting boundary that keeps it out of the ad request, badge delivery labels, creative-bridge failure triggers, Ad Manager identifier normalization, and the adInit refresh context. Add troubleshooting for unattributed and competing refreshes, unconfirmed candidate delivery, and missing correlation evidence. Link the guide from the GPT integration page, use the exact ts_console=1 activation example, and drop the integrations overview claim that diagnostics make no attribution claims. --- docs/guide/integrations-overview.md | 12 +- docs/guide/integrations/gpt-diagnostics.md | 152 ++++++++++++++++++--- docs/guide/integrations/gpt.md | 2 + 3 files changed, 142 insertions(+), 24 deletions(-) diff --git a/docs/guide/integrations-overview.md b/docs/guide/integrations-overview.md index 1e837e63f..95f3d2011 100644 --- a/docs/guide/integrations-overview.md +++ b/docs/guide/integrations-overview.md @@ -186,17 +186,19 @@ enabled = true ### GPT Runtime Diagnostics -**What it does:** Observes documented GPT lifecycle callbacks and presents directly observed slot, timing, coverage, binding, and visibility facts in a local browser console. +**What it does:** Observes documented GPT lifecycle callbacks and Trusted Server integration evidence, then presents directly observed slot, request-path, delivery, timing, coverage, binding, and visibility facts in a local browser console. **Key Features:** - Explicit browser-session `ts_console` activation through a host-only HttpOnly cookie - Conditional standalone delivery only on active HTML documents -- Initial and refresh request-cycle history -- Conservative unmatched and ambiguous callback reporting +- Initial and refresh request-cycle history, with observed request paths and replacements +- Delivery states derived only from observed Trusted Server creative evidence +- Source-neutral Ad Manager identifiers and response classes reported by GPT +- Conservative unmatched, ambiguous, and attribution issue reporting - Exact DOM binding and non-layout-changing viewport badges - Versioned local JSON export with no diagnostic upload -- No creative-provenance or auction attribution claims +- No inferred demand ownership: a filled slot alone never proves Trusted Server delivery **Configuration:** @@ -207,7 +209,7 @@ enabled = true **Endpoints:** None. The feature observes GPT in the browser and makes no diagnostic network request. -**When to use:** You need to debug GPT request, response, render, load, viewability, refresh, and slot-binding behavior without changing ad delivery. +**When to use:** You need to debug GPT request, response, render, load, viewability, refresh, delivery-attribution, and slot-binding behavior without changing ad delivery. **Learn more:** [GPT Runtime Diagnostics](./integrations/gpt-diagnostics.md) diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index 78c657bf6..f3ac9779b 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -19,7 +19,7 @@ Server candidate, a PUC markup request, a successfully posted markup response, a GPT slot load are separate steps in an evidence ladder. This feature requires zero publisher-code changes. Activation remains the existing -server integration configuration plus `?ts_console=true`; it does not require new +server integration configuration plus `?ts_console=1`; it does not require new publisher JavaScript, React, Next.js, DOM, or GAM configuration. The diagnostics integration is independent of the @@ -36,11 +36,36 @@ The module is unavailable unless explicitly enabled for the deployment: enabled = true ``` -Deployment configuration only makes the module available. Inactive browser sessions -receive no diagnostics module. When activated, the standalone content-hashed module -loads synchronously after the core bundle so it can install listeners before -publisher GPT request code. The standalone static response is cookie-independent and -remains publicly cacheable; active HTML responses are private and non-storeable. +Deployment configuration makes the module available; it does not activate any browser +session. Inactive browser sessions receive no diagnostics module. When activated, the +standalone content-hashed module loads synchronously after the core bundle so it can +install listeners before publisher GPT request code. The standalone static response is +cookie-independent and remains publicly cacheable; active HTML responses are private +and non-storeable. + +### Auction correlation token + +Enabling the integration has one server-side effect that does not depend on browser +activation. For each server-side auction that produced winning bids, Trusted Server +mints a fresh correlation token and publishes it as `hb_auction_id` on each winning bid +in `window.tsjs.bids`: + +```text +ts-auc-2f8c1d5a4b7e4c0f9a3d6b1e8c5f2a7d +``` + +- The token is generated per auction from a random UUID. It is not derived from the Edge + Cookie ID, the auction request ID, or any other visitor identifier, and it does not + repeat across auctions. +- It is **not** a GAM targeting key. Of the Trusted Server bid fields, only `hb_pb`, + `hb_bidder`, `hb_adid`, `hb_cache_host`, `hb_cache_path`, and `ts_initial` are applied + as slot targeting alongside the slot's configured targeting, so the token never enters + the ad request. +- It is absent when the integration is disabled, and absent for any auction that + produced no winning bids. +- It is published on every document whose auction produced winning bids, including + documents with no active console session, because the console reads it from the same + page bid state the GPT integration already consumes. ## Activate or Deactivate a Browser Session @@ -56,7 +81,13 @@ Open a page with one of these exact, case-sensitive query directives: For example: ```text -https://publisher.example.com/article?ts_console=true +https://publisher.example.com/article?ts_console=1 +``` + +Deactivate the same browser session with the matching directive: + +```text +https://publisher.example.com/article?ts_console=0 ``` An exact directive establishes or clears the host-only, `Secure`, `HttpOnly`, @@ -124,10 +155,11 @@ arguments, result, and synchronous throw. A `refresh()` call that omits its slot list, or passes `null` or `undefined` for it, refreshes every slot; the observer reads GPT's current slot list for diagnostics only. A stale refresh function reference captured before installation bypasses that boundary and remains -`unattributed`. Prebid sets a scoped, -synchronous diagnostics context while delegating its own refresh, so nesting does not -mislabel a Prebid refresh as `competing`. Diagnostics never suppresses or changes a -GPT request. +`unattributed`. Prebid and the Trusted Server `adInit` refresh each set a scoped, +synchronous diagnostics context while delegating their own refresh, so a nested +`pubads.refresh` is not mislabeled `publisher_refresh` or `competing`. Both clear that +context even when the delegated refresh throws, and the Prebid wrapper restores the +exact prior value. Diagnostics never suppresses or changes a GPT request. For a direct observation, the optional opaque auction ID is retained only after trimming to a non-empty value no longer than 256 UTF-8 bytes. No auction payload, @@ -191,10 +223,12 @@ the state. A matched creative attempt can report these safe, non-terminal categories: -- `missing_render_source` -- `cache_fetch_failed` -- `invalid_cache_payload` -- `response_post_failed` +| Failure | Observed at the bridge | +| ----------------------- | ---------------------------------------------------------------------------------------------- | +| `missing_render_source` | The bid carried neither inline markup nor a complete PBS Cache host and path. | +| `cache_fetch_failed` | The PBS Cache fetch was rejected or returned a non-OK status. | +| `invalid_cache_payload` | The cache response was read but held no renderable creative, so nothing was posted. | +| `response_post_failed` | `port.postMessage` threw while posting markup, on either the inline or the cached-markup path. | Failures are deduplicated and retain first-observed order. Detailed URLs, cache IDs, payloads, markup, and error objects remain only in existing operational logging and do @@ -219,6 +253,11 @@ line-item backfill alike, so they classify as `reservation` only when GPT also reported the render as explicitly non-backfill. On their own they remain `unclassified_non_empty` rather than becoming an unsupported conclusion. +Identifiers are retained only as positive whole numbers, and the yield-group and company +lists keep at most eight IDs each. GPT reports these fields only for reservation and +backfill ads served by PubAdsService, so an absent identifier is a fact about the render +rather than a gap in observation. + ## Attribution Issues and Callback Coverage Creative-correlation problems are exported separately from GPT callback issues. The @@ -270,6 +309,28 @@ A concise viewport badge appears only when a slot: - Has a unique, connected exact binding. - Has a non-zero rectangle intersecting the viewport. +A badge summarizes the slot's most recent request cycle: the GPT result (Filled, Empty, +Rendered (fill unknown), or Pending), a short delivery label, a `Competing paths` +marker when the request path is `competing`, the rendered size, and the request-to- +response, response-to-render, and render-to-viewable durations that are available. It +adds `Incomplete sequence` when a callback proved a missing or invalid earlier step. + +Badge delivery labels are the same derived states the panel and export report, shortened +to fit: + +| Delivery state | Badge label | +| ------------------------------ | ----------------------- | +| `trusted_server_response_sent` | TS response sent | +| `trusted_server_selected` | TS selected | +| `pending` | TS candidate (pending) | +| `candidate_unconfirmed` | TS unconfirmed | +| `no_candidate` | No TS candidate | +| `unknown` | Delivery unknown | +| `not_applicable` | No delivery label shown | + +The badge never re-derives delivery from raw timestamps; it labels the state the store +already resolved, so a badge cannot disagree with the panel or the export. + Missing elements and duplicate DOM or GPT slot IDs remain visible in the panel as Unbound or Ambiguous and receive no badge. If DOM uniqueness cannot be verified because selector support is unavailable or throws, the export reports @@ -340,10 +401,13 @@ The allowlisted export contains: It does not contain raw targeting, bid IDs, bid prices, bidder identity, creative markup, cache URLs, cache payloads, cache or bridge error details, cookies, user -identifiers, query strings, or URL fragments. The exported -`trustedServerAuctionId` is a token minted fresh for each server-side auction: it -is not derived from the Edge Cookie ID or any other visitor identifier, and it does -not repeat across auctions, so it cannot be joined back to a visitor. +identifiers, query strings, or URL fragments. The exported `trustedServerAuctionId` +is the `hb_auction_id` value described in +[Auction correlation token](#auction-correlation-token): minted fresh for each +server-side auction, not derived from the Edge Cookie ID or any other visitor +identifier, and never repeated across auctions, so it cannot be joined back to a +visitor. Diagnostics retain it only after trimming to a non-empty value of at most +256 UTF-8 bytes. Captured records are memory-only. Diagnostics do not add an upload, diagnostics network request, `localStorage`, `sessionStorage`, IndexedDB, or other persistence. @@ -419,6 +483,56 @@ documented callbacks do not expose a request-cycle identifier. Avoid overlap in controlled tests, or use the issue record as evidence that correlation was not possible. +### A refresh is `unattributed` or `competing` + +`unattributed` means no request-path evidence was still eligible when GPT emitted +`slotRequested`. Each source's marker lives five seconds and is consumed once, so a +request more than five seconds after the observation, a refresh function reference the +publisher captured before installation, and any path Trusted Server does not observe +all stay `unattributed`. Diagnostics never fill that gap from timing, element IDs, or +targeting names. + +`competing` means two or more sources contributed evidence for the same request. It is +a warning that competition or overwrite is possible, not a statement about which values +GPT sent. To narrow it in a controlled test, trigger one path at a time and leave more +than five seconds between refreshes. + +### Delivery stays `candidate_unconfirmed` + +The cycle rendered explicitly non-empty with a Trusted Server candidate, but no matched +creative markup request arrived within five seconds of `slotRenderEnded`. Read the +cycle's other facts before concluding anything: + +- `responseClass` and the GAM identifiers show what Ad Manager reported delivering. +- A creative-bridge failure category on the same cycle shows the bridge was reached and + failed. +- An attribution issue at the same time shows the request arrived but could not be + correlated. +- No evidence at all is consistent with a different GAM result, a targeting overwrite, + and a PUC configuration or ID mismatch alike. + +A late positive observation upgrades the state, so re-read the panel rather than +exporting immediately after render. + +### Correlation evidence is missing + +Attribution issues record why creative evidence could not be attached. They never +increment callback coverage and never produce a delivery claim: + +| Reason | What was observed | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `creative_request_without_slot` | The markup request carried no auction slot ID, or no retained association mapped it to a GPT slot. | +| `creative_request_without_cycle` | The slot had no retained request cycle inside the 30-second attempt window, or its most recent cycle was already reported empty. | +| `creative_request_ambiguous_cycle` | An earlier non-empty cycle for the same slot was still in window before render, so no cycle was chosen. | +| `creative_request_on_empty_cycle` | GPT later reported the matched cycle empty, so the attempt was dropped instead of claiming delivery. | +| `creative_attempt_capacity` | The 128-attempt bound was reached with every retained attempt still live. | +| `creative_attempt_unknown` | A response or failure referenced an attempt no longer retained. | +| `creative_attempt_expired` | The attempt passed its 30-second lifetime before its response or failure was observed. | +| `creative_attempt_evicted` | The attempt's slot or request cycle was dropped first, by a retention bound or by GPT reporting that cycle empty. | + +Repeated issues on a busy page usually mean retention bounds, not delivery failure. +Reduce refresh overlap or capture a shorter session, then re-read the cycle. + ## Limits The integration observes six documented PubAdsService events, wraps diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index f38f68231..de0df03c4 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -144,6 +144,7 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut - Check the proxy responses have `200` status (look for `X-GPT-Proxy: true` header) - Verify the `script_url` config points to the correct GPT endpoint - Review server logs for upstream fetch failures +- Open [GPT Runtime Diagnostics](./gpt-diagnostics.md) with `?ts_console=1` to see the observed request, render, load, and delivery evidence per slot ## Implementation @@ -152,6 +153,7 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut ## Next Steps +- Use [GPT Runtime Diagnostics](/guide/integrations/gpt-diagnostics) to inspect GPT lifecycle and Trusted Server delivery evidence in the browser - Review [Integrations Overview](/guide/integrations-overview) for comparison with other integrations - Check [Configuration Reference](/guide/configuration) for advanced options - Learn about [First-Party Proxy](/guide/first-party-proxy) architecture From 297325ac84e385fd45cbe3fc42e0862d3c511b5d Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:44:26 -0700 Subject: [PATCH 162/195] Add GAM ts cohort attribution design spec --- ...-07-15-gam-ts-cohort-attribution-design.md | 886 ++++++++++++++++++ 1 file changed, 886 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md diff --git a/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md new file mode 100644 index 000000000..525c1e874 --- /dev/null +++ b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md @@ -0,0 +1,886 @@ +# GAM `ts=true` attribution for Trusted Server A/B traffic + +Date: 2026-07-15 + +Status: Design + +## Problem + +A publisher will route a small, cookie-sticky A/B cohort through Trusted Server +while the control cohort continues through the existing production path. The +publisher wants Google Ad Manager (GAM) reporting to identify impressions and +clicks generated on pages served through Trusted Server and compare them with +the unmodified production cohort. + +Trusted Server currently adds the slot-level key-value `ts_initial=1` while it +prepares initial GPT slots. That key has a different lifecycle and meaning from +the experiment marker: + +- it identifies the initial slot request prepared by Trusted Server; +- it is cleared before later client-side refresh auctions; and +- it is set on matched slots rather than every GPT request on the page. + +The experiment needs a document-delivery marker. Every request issued by the +document-local GPT PubAds service after marker installation in an HTML document +successfully rewritten by Trusted Server must contain `ts=true`, including +initial requests, publisher-owned slots, lazy slots, and refreshes. Production +documents cannot be modified, so the control cohort remains unmarked. + +## Goals + +1. Add `ts=true` to every in-scope GPT PubAds request made during the lifetime + of an HTML document successfully rewritten by Trusted Server. +2. Leave production/control pages unchanged. +3. Preserve the existing `ts_initial=1` slot-ownership and refresh lifecycle. +4. Support GAM reports that count treatment impressions and clicks and derive + the control counts within the same experiment scope. +5. Avoid changing auction eligibility, ad delivery, consent behavior, or page + performance. +6. Define the data-quality checks needed when an unmarked request is used as the + control baseline. + +## Non-goals + +- Implement or change the cookie-based A/B router. The experiment infrastructure + owns sticky cohort assignment and routes only the treatment cohort through + Trusted Server. +- Prove that a Trusted Server server-side bid won the GAM auction. `ts=true` + means that the page was delivered through Trusted Server, regardless of + whether the winning demand was a server-side bid, a direct GAM line item, Ad + Exchange, or backfill. +- Mark GAM traffic outside a successfully rewritten document's local GPT PubAds + service. IMA/video SDK requests, direct tags, and server-side GAM requests + require separate instrumentation and are outside this design. A nested + document's GPT instance is in scope only when that nested HTML response is + independently routed through Trusted Server and satisfies the same rewrite, + ordering, CSP, and audit prerequisites. +- Replace `ts_initial`, `hb_*`, line-item, bidder, or creative reporting. +- Add a client-side analytics beacon or a Trusted Server telemetry event. +- Make GAM click tracking more complete. The marker only segments clicks that + GAM already records. +- Provide billing-grade or causal experiment analysis from GAM alone. + +## Assumptions and prerequisites + +- The GPT integration is enabled on every page routed into the treatment cohort. + A page served through Trusted Server without the GPT integration does not + receive the GPT head bootstrap and cannot satisfy this design. +- The response enters and successfully completes Trusted Server HTML rewriting, + contains a literal `` element, and reaches that element before any + publisher script issues a GAM request. Pass-through or buffered-unmodified + responses, rewrite failures, and origin markup that omits `` cannot + satisfy the marker guarantee. +- The publisher Content Security Policy allows Trusted Server's bare inline + scripts to execute. Initial `adSlots`, the GPT enable flag, the GPT bootstrap, + and the `bids`/`adInit` invocation are all nonce-less inline scripts; Trusted + Server does not currently propagate a publisher nonce or update CSP hashes. A + policy that blocks those scripts makes the initial TS ad stack inert and is + ineligible at launch even if it allows the synchronous first-party TSJS + bundle. +- Each in-scope HTML document uses one document-local GPT PubAds service. Nested + documents are not implicitly covered by a marked parent: each nested HTML + response must be independently routed through Trusted Server and rewritten to + receive the marker. IMA/video, direct-tag, server-side GAM, and any nested GPT + inventory whose document is not independently rewritten must be excluded from + the experiment and paired reports. +- Treatment and control traffic use the same GAM network and comparable + inventory. Report filters can isolate the pages and time window eligible for + the experiment. +- The experiment owner can obtain the expected treatment allocation from the + cookie router, even though Trusted Server does not read or emit that cookie. +- Publisher code and Trusted Server creative-opportunity slot configuration do + not reuse `ts` for another meaning, set a slot-level `ts` value, or clear + page-level targeting after Trusted Server targeting runs. The deployment audit + must inspect `trusted-server.toml` targeting maps and search publisher code + for `setTargeting`, `setConfig`, and `clearTargeting` uses that could + overwrite or remove the reserved key. If such behavior exists, it must be + resolved before launch; silently filtering operator targeting or wrapping + publisher GPT APIs is out of scope. + +## Existing behavior + +The GPT integration has two related pieces: + +1. `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` is injected at + the start of ``. It creates the GPT command queue early and installs + the minimal `window.tsjs.adInit` implementation used before the richer bundle + is available. +2. `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` installs the + richer GPT integration and applies slot-level auction targeting. + +Both initial-render paths set `ts_initial=1` on slots handled by `adInit`. The +Prebid refresh integration includes `ts_initial` in its list of stale +slot-targeting keys and clears it before subsequent client-side refresh +auctions. SPA cleanup also clears stale `ts_initial` targeting before applying +new route state. + +This behavior is correct for `ts_initial` and must not change. It is not +sufficient for a page-level treatment marker because it does not cover all GPT +slots and intentionally does not persist across refreshes. + +## Decision + +Add a separate page-level GPT key-value: + +```text +ts=true +``` + +The early GPT bootstrap will enqueue page-level targeting before publisher GPT +commands execute. The enqueue must occur after `window.tsjs` is initialized but +before the existing `if (ts.adInit) return;` guard: + +```text +(function () { + if (typeof window === "undefined") return; + var ts = (window.tsjs = window.tsjs || {}); + var tag = (window.googletag = window.googletag || { cmd: [] }); + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { + try { + if (typeof googletag.setConfig === "function") { + googletag.setConfig({ targeting: { ts: "true" } }); + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap. + } + }); + + if (ts.adInit) return; + // Existing initial-load detector and adInit stub follow. +})(); +``` + +The exact implementation must follow the repository's JavaScript formatting and +defensive checks. The important contract is that the page-level targeting +command is queued by the head bootstrap before the origin page can queue its GPT +setup or request ads. Page attribution is independent of whether the bootstrap +needs to install `ts.adInit`, so the existing guard may skip only the ad-init +stub and detector setup, never the marker enqueue. An unavailable targeting API +may skip only the marker callback; it must not prevent later queued publisher or +Trusted Server callbacks from running. + +The existing initial-load detector immediately below the new marker must reuse +the initialized `tag.cmd` reference rather than repeat its current +`(window.googletag = window.googletag || { cmd: [] }).cmd` expression. This +keeps queue initialization and both bootstrap callbacks on one consistent path. + +Moving queue initialization above the `ts.adInit` guard intentionally creates a +standard `window.googletag` command-queue stub on every rewritten, GPT-enabled +page, including a page where `ts.adInit` already exists and GPT never loads. The +stub is inert by itself and preserves the marker-before-guard guarantee; it is +not an accidental behavior to remove during implementation review. + +The TypeScript GPT bundle will defensively enqueue the same page-level targeting +at module initialization after the existing flag-gated shim block and before +`installTsAdInit()`, only when the existing publisher-page bundle tag carries a +non-executable GPT activation attribute. The HTML pipeline adds that attribute +when the GPT integration is enabled. A pre-existing `ts.adInit` is already +covered by placing the bootstrap marker before the guard and is not a reason for +the fallback. + +The fallback exists to preserve delivery-path attribution if the inline +bootstrap unexpectedly stops executing while the synchronous first-party bundle +still runs before publisher GPT. It does not recover `adSlots`, bids, the +`adInit` invocation, or the initial TS auction: those are also nonce-less inline +scripts. A fallback-only marker therefore still truthfully means "page delivered +through Trusted Server," but it also indicates a deployment state that was +ineligible at launch. Synthetic validation must treat that state as a +measurement incident, pause interpretation, and exclude the affected time window +from both paired reports if the incident contaminates collected results. GAM +cannot distinguish fallback-only pages from normally executing treatment pages +because both intentionally use the same marker. + +Neither targeting path can cover a response that was not HTML-rewritten, markup +without ``, a policy that blocks both injected paths, or a publisher GPT +request issued before the injected head content runs. Those are deployment +eligibility and coverage-validation concerns, not runtime conditions the +targeting code can repair. + +The implementation uses GPT's current page-level `googletag.setConfig` API +rather than the deprecated `pubads().setTargeting()` API. See +[GPT configuration API migration](https://developers.google.com/publisher-tag/guides/config-migration). +Page-level targeting is the right scope because GPT applies it to all slots +associated with the `pubads` service. Once installed, it remains effective for +initial, lazy, and refreshed requests for the life of the page. Existing slot +targeting may add or override other keys without requiring Trusted Server to +discover every publisher slot. + +GPT merges page-level targeting per key across `setConfig` calls. Enqueuing +`ts=true` from both Trusted Server paths is therefore idempotent, and a +publisher call that sets an unrelated targeting key preserves `ts`. The explicit +clear operations are a per-key `null`, a whole-targeting `null`, or the +equivalent legacy `pubads().clearTargeting()` calls. See +[GPT key-value targeting](https://developers.google.com/publisher-tag/guides/key-value-targeting). + +`ts` is intentionally not added to the slot-targeting cleanup arrays. Those +arrays manage per-auction state. Clearing page-level `ts` during refresh or SPA +navigation would incorrectly move a treatment page into the unmarked control +cohort. + +## Attribution contract + +### Treatment + +An in-scope GAM request is in the treatment cohort when it contains: + +```text +ts=true +``` + +The marker means: + +> The containing page was delivered through Trusted Server. + +It does not mean: + +- a Trusted Server bidder returned a bid; +- a Trusted Server bid won; +- Trusted Server rendered the winning creative; or +- the request was the first impression for the slot. + +### Control + +The production path cannot be changed. Within the exact experiment inventory, +time window, and publisher scope, an unmarked GAM request is treated as control. + +This is an inference rather than an explicit `ts=false` assertion. A treatment +request that loses its marker would be misclassified as control. The rollout +therefore requires coverage checks that compare the observed GAM treatment share +with the A/B router's expected cookie cohort share. + +### Relationship to `ts_initial` + +| Key | Scope | Lifetime | Meaning | +| -------------- | ---------- | ---------------------------- | ----------------------------------------- | +| `ts=true` | Page-level | Entire browser page lifetime | Page was delivered through Trusted Server | +| `ts_initial=1` | Slot-level | Initial TS-managed request | Initial slot request was prepared by TS | + +The two keys answer different questions and coexist. No code or report should +infer that one is an alias for the other. The value contract is exactly +`ts=true`: do not emit, accept, or report any other value (for example `ts=1`), +and do not dual-write an alternative key name such as `trusted_server`. + +## Request lifecycle + +```text +Sticky A/B cookie + -> control: browser receives production page + -> publisher GPT runs without the `ts` key + -> treatment: request is routed through Trusted Server + -> GPT head bootstrap queues page-level `ts=true` + -> GPT bundle defensively queues the same marker + -> GPT library drains the command queue + -> publisher and TS define/display/refresh slots + -> every in-scope PubAds request carries `ts=true` +``` + +The marker covers: + +- Trusted Server-defined initial slots; +- publisher-defined slots reused by Trusted Server; +- publisher slots that are not part of a Trusted Server creative opportunity; +- slots created lazily after initial page load; +- publisher-initiated refreshes; +- Prebid-managed refreshes; and +- SPA route changes within the same browser document. + +All bullets refer to slots using the same document-local GPT PubAds service. +Requests from IMA/video SDKs, direct tags, or server-side GAM integrations are +not covered merely because the containing document is marked. A nested GPT +instance is marked only when Trusted Server separately rewrites that nested +document and injects the attribution paths into its own ``. + +A full browser navigation creates a new page and repeats cookie-based routing. +The new page receives the marker only when that navigation is routed through +Trusted Server. + +## Component changes + +### Early GPT bootstrap + +`crates/trusted-server-core/src/integrations/gpt_bootstrap.js` owns the +behavior. It will set the page-level key in its earliest GPT command callback, +before the `ts.adInit` early-return guard. The targeting code stays inside the +existing raw bootstrap script returned by `head_inserts`; it must not add a +third head insert. + +The operation must be idempotent. Calling +`googletag.setConfig({ targeting: { ts: 'true' } })` more than once with the +same value is harmless, but the bootstrap should avoid adding a new global state +machine solely for deduplication. + +The bootstrap already binds the local variable `ts` to the `window.tsjs` +namespace, so the targeting key `ts` and that variable are unrelated names that +sit only a few lines apart. Add a clarifying comment at the targeting call so a +maintainer does not read the key as the namespace. The value is the string +`'true'`, never the boolean `true`: GPT targeting values must be strings. + +### TypeScript GPT bundle fallback + +`crates/trusted-server-js/lib/src/integrations/gpt/index.ts` will add a small +`installTrustedServerPageTargeting()` helper and call it during GPT module +initialization after the existing flag-gated `installGptShim()` block and before +`installTsAdInit()` when the publisher-page bundle's activation attribute is +present. The helper creates or reuses the standard GPT command queue, enqueues +the same defensive `setConfig({ targeting: { ts: 'true' } })` call, and does not +read the experiment cookie or wait for an auction. Extend the local `GoogleTag` +interface with optional `setConfig?(config: Record): void` so +the fallback remains defensive when the API is unavailable. + +The bootstrap remains the primary path because it is injected first. The bundle +call is a redundant fallback and must not delay module initialization, create a +request, or add slot-level targeting. A plain GPT module import without the +activation attribute must preserve the existing runtime-gating contract and must +not create `window.googletag`. + +### Non-executable bundle activation + +The publisher HTML pipeline in +`crates/trusted-server-core/src/html_processor.rs`, using a separate, +publisher-page-only tag helper in `crates/trusted-server-core/src/tsjs.rs`, will +add a `data-ts-gpt-enabled="true"` attribute to the existing synchronous +`#trustedserver-js` bundle tag when GPT is enabled. The attribute is data, not +an inline executable, so CSP can block the inline GPT head inserts while still +allowing the external bundle to detect that it owns page attribution. + +At module initialization, the GPT bundle captures `document.currentScript` and +requires that executing synchronous script to carry `data-ts-gpt-enabled="true"` +before the fallback may create a GPT stub. It must fail closed when the +executing script cannot be identified. Do not authorize activation through a +global `#trustedserver-js` lookup: the generic unified tag uses the same ID in +creative and test contexts, and duplicate IDs could select the wrong element. +Binding the signal to the executing tag keeps the activation decision explicit +and testable without relying on an inline global flag. + +The existing `window.__tsjs_gpt_enabled` flag continues to activate +`installGptShim()` when inline scripts run. It cannot activate the CSP fallback +because the server sets it from an inline head insert—the execution path CSP may +block. Migrating shim activation to the data attribute is out of scope; module +initialization preserves the current flag-gated shim installation, then runs the +attribute-gated page-targeting helper, then installs `ts.adInit` and the +remaining GPT bundle hooks. + +This signal must be limited to the publisher-page bundle generated from the +enabled integration registry. Do not infer activation merely because the GPT +module exists in an all-modules bundle: creative and test tooling can load that +bundle outside the publisher GPT integration. Do not add a new script tag or +change the integration's existing head-insert count. + +Extend the `tsjs.rs` and `html_processor.rs` tests to prove that the existing +publisher-page bundle tag gains the activation attribute only when GPT is in the +enabled immediate module set, remains a single external tag, and omits the +attribute for non-GPT publisher bundles and generic all-modules tags. Bundle +tests must also prove that an unrelated or duplicate element with +`id="trustedserver-js"` cannot activate the fallback. + +### GPT Rust integration tests + +`crates/trusted-server-core/src/integrations/gpt.rs` already tests the embedded +bootstrap returned by `head_inserts`. Extend those tests to prove that: + +- the bootstrap contains page-level `ts=true` targeting; +- the marker enqueue appears before the `if (ts.adInit) return;` guard; +- the targeting setup is queued before `ts.adInit` can issue `display` or + `refresh`; +- the existing `ts_initial` marker remains present; and +- the enabled integration without `slim_prebid_url` still emits exactly the + existing two head inserts, proving the marker was added to the bootstrap + instead of a new tag. + +### Bootstrap execution tests + +Add `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts` +as a Vitest/jsdom behavioral test for the raw bootstrap. The test reads +`crates/trusted-server-core/src/integrations/gpt_bootstrap.js` with Node's +`readFileSync`, resolves the source path relative to `import.meta.url` rather +than the process working directory, and creates an isolated +`new JSDOM(html, { runScripts: 'outside-only' })` realm. Before evaluating the +exact source with that realm's `window.eval`, the test must assert that +`globalThis === window` and `typeof global === 'undefined'` inside the realm. +This is required because Vitest's default jsdom `window.eval` runs in Node's +realm under the current configuration. The harness supplies a minimal mocked GPT +command queue and `pubads` service. It must not evaluate the source in Node's +global context, copy the bootstrap into a test fixture, or add a JavaScript +runtime dependency to Rust. + +The harness must prove that: + +- the attribution callback is queued before a publisher callback added after the + injected bootstrap; +- draining the queue calls `googletag.setConfig` with page-level `ts=true` + before the publisher callback runs; +- a pre-existing `ts.adInit` does not prevent the attribution callback from + being queued or executed; +- a publisher callback queued after the bootstrap still runs when + `googletag.setConfig` throws; +- an unavailable or throwing `googletag.setConfig` does not prevent the existing + `disableInitialLoad` wrapper from being installed; +- `ts.adInit` remains installed when attribution setup is unavailable or throws; + and +- calling the wrapped `disableInitialLoad` still records + `ts.gptInitialLoadDisabled`. + +### Bundle fallback tests + +Extend `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts` using +its existing dynamic-import and `vi.resetModules()` pattern. Prove that module +initialization with the bundle activation attribute queues page-level `ts=true` +after any existing flag-gated shim installation and before installing +`ts.adInit`, that it reuses an existing GPT command queue, and that unavailable +or throwing `setConfig` does not stop the remaining GPT module installers. +Retain the existing assertion that a plain module import without an activation +signal does not create `window.googletag`. A duplicate call after the bootstrap +must remain safe and must not create another script or network request. + +### Slot cleanup constraints + +No refresh-lifecycle change is required. In particular: + +- do not add `ts` to `TS_REFRESH_TARGETING_KEYS`; +- do not add `ts` to `TS_BASE_TARGETING_KEYS`; +- do not rename or remove `TS_INITIAL_TARGETING_KEY`; and +- do not copy `ts` onto individual slots. + +Leaving these components unchanged is part of the design: slot cleanup cannot +remove a page-level key set through `googletag.setConfig`. + +### Documentation + +Document the distinction between page-level `ts=true` and slot-level +`ts_initial=1` in `docs/guide/integrations/gpt.md`, near the existing command +queue documentation. Include the GAM setup and reporting preconditions below; do +not add this current integration to a planned-future GAM document. + +## GAM configuration + +GAM configuration is a deployment prerequisite and must be completed before the +experiment starts because key-value reporting is not retroactive. + +The request contract is `ts=true`: key name `ts`, predefined value `true`. The +key name is provisional pending a GAM preflight (see issue #1027). The GPT/GAM +`CustomTargetingKey.name` (the code sent in the ad request) is documented as +limited to 10 characters in the SOAP/REST API, which would rule out a fuller +name such as `trusted_server` (14); other Help Center material implies 20, and +these describe different provisioning surfaces, so the enforced limit must be +confirmed in the target network before the contract is fixed. If the confirmed +limit permits a longer name, a more descriptive, less collision-prone key is +preferred over `ts`. Because `ts` is short, it carries a real collision risk with +common publisher timestamp or cache-buster keys, which makes the cross-system +collision audit a hard launch gate, not a formality. If any publisher, Trusted +Server configuration, or GAM object already uses `ts`, the experiment must stop +until the collision is removed or the contract is explicitly revised everywhere +before treatment traffic begins. + +1. In **Inventory > Key-values**, create or verify a key whose request name is + the finalized marker key (`ts` pending the preflight in issue #1027). When the + key is created, confirm the enforced key-name and value length limits on the + provisioning surface actually used: the SOAP/REST + [`CustomTargetingKey`](https://developers.google.com/ad-manager/api/reference/v202511/CustomTargetingService.CustomTargetingKey) + documents a 10-character key-`name` limit and a 40-character value limit, but + the UI surface may differ, so verify against the target network rather than + assuming. +2. Use a predefined value named `true`. +3. Enable `ts` as a dedicated reportable Enhanced key-value dimension. If the + network does not support Enhanced key-value dimensions, use a report filtered + to the single legacy key-value `ts=true`; never sum unfiltered legacy + **Key-values** dimension rows. +4. Reserve `ts` for Trusted Server page attribution. +5. Audit existing publisher GPT code and every GAM object that consumes custom + targeting for an existing `ts` key before deployment. This includes line + items, proposal line items, rules, protections, yield configuration, and any + network-specific custom-targeting surface. +6. Audit every `CreativeOpportunitySlot.targeting` map from all effective + `trusted-server.toml` configuration sources. The arbitrary operator-supplied + map is copied to GPT slots, where a slot-level `ts` value would override the + page-level marker. Any occurrence is a launch blocker; do not silently + discard it because that could change established operator targeting. +7. Audit publisher code for every operation that can remove or supersede the + marker after initial GPT setup. Search for `setConfig({ targeting: null })`, + a `ts: null` or different `ts` value, `pubads().clearTargeting()` with no key + or with `ts`, and slot-level `ts` targeting. Account for equivalent calls + assembled dynamically. + +The audit is a hard precondition. If `ts` already has another meaning, or any +GAM object targets or acts on `ts=true`, the experiment owner must resolve the +collision before deployment. The measurement marker is not intended to change ad +eligibility, pricing, protection, or routing. A pre-existing targeting consumer +for `ts=true` would make the A/B test measure a traffic or demand change at the +same time as Trusted Server delivery. + +Undefined values do not appear in standard key-value reports even when the key +is reportable, so value `true` must exist before treatment traffic begins. See +[Add key-values](https://support.google.com/admanager/answer/9796369) and +[Report on targeting keys](https://support.google.com/admanager/answer/14528835). + +## Reporting and comparison + +### Report scope + +Every comparison must apply identical filters for: + +- publisher/network; +- experiment start and end time; +- sites or inventory included in the cookie experiment; +- ad units and formats; +- geography and device categories, when used; and +- any consent or traffic-quality exclusions. + +Do not compare the TS cohort with all unmarked network traffic unless all that +traffic is eligible for the same experiment. Likewise, exclude smoke tests, +direct hits, operations traffic, and any other TS-served page outside the cookie +experiment. The marker identifies the delivery path, not the router's cohort +assignment, so all such requests also carry `ts=true` when the GPT integration +runs. + +The route owner must use router or access logs to prove that non-experiment TS +traffic is absent from the eligible inventory during the measurement window. If +such traffic cannot be prevented and has no independent inventory or reportable +dimension, GAM cannot remove it from Report B because its marker is identical to +the cohort marker; the experiment must not launch. Record the owner, query, +expected zero threshold, and response procedure in the experiment runbook. + +### Cohort calculations + +Use two reports with identical date boundaries, time zone, inventory filters, +traffic-quality filters, and metric definitions: + +1. **Report A — experiment total.** Do not include **Placement**, legacy + **Key-values**, **Targeting**, **Yield group**, or another dimension that can + represent one event more than once. This report provides one non-duplicated + total for every metric in the eligible experiment scope. +2. **Report B — TS treatment.** Use the dedicated Enhanced `ts` dimension + filtered to `ts=true`. If Enhanced key-value dimensions are unavailable, use + the legacy **Key-values** dimension filtered to exactly `ts=true` and do not + sum any other key-value rows. Do not add **Placement**, **Targeting**, + **Yield group**, or any unrelated dimension that can represent the filtered + treatment event more than once. + +The legacy **Key-values** dimension can emit the same impression or click on +multiple rows when a request contains multiple key-values. It therefore cannot +provide Report A or a summable totals row. See +[Avoid double counting report totals](https://support.google.com/admanager/answer/7642799). + +For this paired report scope, define: + +```text +total_impressions = Report A impressions +ts_impressions = Report B impressions +prod_impressions = total_impressions - ts_impressions + +total_clicks = Report A GAM-recorded clicks +ts_clicks = Report B GAM-recorded clicks +prod_clicks = total_clicks - ts_clicks +``` + +If the selected GAM report exposes an explicit unassigned or `(not set)` row, +that row may be used only as a cross-check. The paired Report A minus Report B +calculation remains the control definition because production cannot send an +explicit value. The experiment owner must retain both report definitions with +the results so later analysis can verify that their filters and metrics match. +Export both reports after the same GAM reporting-latency and invalid-traffic +adjustment window. If GAM restates one report, rerun the pair before applying +the subtraction. + +Use total metrics when the goal includes all GAM demand sources. GAM's +`Ad server impressions` and `Ad server clicks` metrics exclude Ad Exchange and +AdSense, so those narrower metrics should only be used when that exclusion is +intentional. GAM counts impressions and clicks according to its own tracking +rules; adding `ts=true` does not create new impression or click trackers. See +[Counting impressions and clicks](https://support.google.com/admanager/answer/2521337). + +Both reports must use the same non-targeted impression and click metric names. +Do not use targeted-impression or targeted-click metrics for this attribution: +`ts` is intentionally forbidden from line-item targeting, so metrics limited to +keys used for targeting do not represent the requested delivery-path cohort. +Record the exact selected GAM metric names with the saved report definitions +before launch. + +### Unequal cohort sizes + +The treatment cohort is intentionally small, so raw TS and production totals are +not directly comparable. Reports should show the raw counts, but experiment +conclusions should compare normalized measures where compatible metrics are +available: + +- impressions per GAM ad request; +- fill rate; +- clicks per impression (CTR); and +- revenue per thousand impressions or requests. + +Impressions per routed pageview or per assigned visitor require a denominator +from the A/B router or site analytics. GAM alone cannot identify unmarked +production pageviews that made no ad request. Any cross-system experiment +analysis is outside the implementation but should use the same time and +eligibility filters. + +### Data-quality checks + +During the experiment, monitor: + +1. observed `ts=true` ad-request or impression share versus the router's + expected treatment allocation; +2. scheduled synthetic marker presence on initial, lazy, and refreshed treatment + requests; +3. scheduled synthetic marker absence on production requests; +4. non-experiment traffic served through Trusted Server; +5. unexpected `ts` values or line-item targeting; +6. report freshness and GAM invalid-traffic adjustments. + +A gap between expected and observed treatment share is a measurement incident, +not evidence of production performance, until missing-marker and request-volume +differences are ruled out. Because router assignment and GAM delivery normally +use page or visitor counts versus ad-request or impression counts, this share +comparison is a diagnostic rather than direct proof of marker coverage. It can +measure coverage directly only when the router or site analytics supplies a +matched request- or page-level denominator. + +Checks 2–3 use a scheduled synthetic browser crawl of representative experiment +URLs. The crawler supplies known treatment and control cookies, captures GAM +network requests, and triggers initial, lazy, and refreshed slots. A failed +marker assertion is an operational measurement incident. This is external +validation rather than a site beacon or Trusted Server telemetry event; if the +experiment owner cannot operate the crawl, checks 2–3 become documented manual +samples and must not be represented as continuous production metrics. + +On treatment URLs with matched creative opportunities, the crawler must also +detect the fallback-only CSP state: capture CSP violations and verify that the +injected `adSlots`, `bids`, and initial `adInit` handoff executed. A page that +has `ts=true` only because the external bundle ran, while those inline scripts +were blocked, remains correctly marked as TS-delivered but raises a measurement +incident. Since GAM cannot separate those requests afterward, the incident owner +must pause interpretation and exclude the affected time range from both reports +when clean boundaries can be established; otherwise the experiment result is +invalid. + +## Failure handling + +The marker is best-effort instrumentation and must never block ads or page +delivery. + +- If GPT never loads, there is no GAM request to classify. +- If a response is not successfully HTML-rewritten, has no literal ``, or + issues an in-scope GPT request before the injected head content runs, neither + targeting path can mark that request. Such traffic is ineligible for the + experiment and must be detected before launch or excluded from analysis. +- If CSP blocks Trusted Server's nonce-less inline scripts, the initial TS ad + stack is inert and the page is ineligible even when the first-party bundle + queues the attribution marker. The fallback prevents a TS-delivered page from + leaking into the inferred control cohort; it does not make the deployment + healthy. If CSP blocks both inline scripts and the bundle, attribution also + fails. +- If `googletag.setConfig` is unavailable when a queued command runs, the + targeting step is a defensive no-op and must not throw. Supported treatment + deployments must use a GPT version with the configuration API; browser/GAM + validation detects an unsupported or missing API before experiment launch. +- If publisher code or a Trusted Server creative-opportunity targeting map + applies slot-level `ts`, GPT gives the slot-level value precedence. The + deployment audit prevents this collision; runtime filtering or interception is + out of scope because it could silently alter established targeting behavior. +- If publisher code calls `setConfig({ targeting: null })`, sets `ts: null` or a + different value, calls legacy `pubads().clearTargeting()` for all keys or for + `ts`, or applies slot-level `ts`, the effective marker can be removed or + superseded. The publisher-code audit and refresh validation are required + because this design deliberately does not intercept those APIs. +- If the marker is absent on a treatment request, GAM classifies it with the + unmarked baseline. Coverage monitoring is the mitigation. +- GAM configuration or reporting failures do not affect ad serving. + +No retry, beacon, cookie read, backend request, or persistent client state is +added by this feature. + +## Privacy and consent + +`ts=true` contains no unique user identifier, cookie value, page URL, or auction +data. It describes only the delivery path of the current document. Because only +the cookie-sticky treatment cohort is routed through Trusted Server for this +experiment, the value also reveals treatment-path membership for that GAM +request. It is therefore cohort information even though it does not expose the +assignment cookie or identify a person by itself. + +The implementation does not read the experiment cookie. Routing happens before +Trusted Server handles the request. Existing consent gates continue to decide +whether GAM requests or auctions occur. The marker does not create an ad request +that would otherwise be suppressed. Before enabling the key, the experiment +owner must complete the publisher's privacy/data-governance review for sending +this treatment-path attribute to GAM and confirm that existing consent and +data-use terms cover it. + +## Testing strategy + +### Automated tests + +1. Extend Rust GPT head-insert tests to assert page-level `ts=true` targeting is + in the existing raw bootstrap, occurs before the `ts.adInit` guard and any + bootstrap `display()` or `refresh()` call, and does not change the expected + head-insert count. +2. Extend the TSJS tag and HTML processor tests to prove the non-executable GPT + activation attribute appears only on the enabled publisher-page bundle and + adds no script tag. +3. Add the Vitest/jsdom raw-bootstrap harness described above. Exercise the + bootstrap with `googletag.setConfig` available, unavailable, and throwing, + and prove a later publisher callback still runs in every case. Set + `window.tsjs.adInit` before evaluation and prove the marker still runs. +4. Extend the existing GPT bundle tests to prove module initialization queues + the fallback marker and remains non-blocking when `setConfig` is unavailable + or throws. +5. Retain assertions for `ts_initial=1` to prevent accidental replacement. +6. Retain refresh tests proving stale `ts_initial` and `hb_*` slot targeting is + cleared. Add an explicit assertion or source-level invariant that page-level + `ts` is not included in slot cleanup lists. +7. Extend creative-opportunity configuration tests to demonstrate that an + operator targeting map is forwarded verbatim, documenting why the deployment + audit must reject a configured `ts` key rather than assuming the client + overwrites or filters it. +8. Run the project-required target-matched Rust and JavaScript checks for the + touched files. + +### Browser/GAM validation + +Before experiment launch: + +1. Load a treatment page using a known treatment cookie. +2. Confirm the initial in-scope GAM request contains `ts=true` using GPT + Publisher Console, Delivery Inspector, or the browser network panel. +3. Trigger a lazy slot and a refresh; confirm both requests still contain + `ts=true`. +4. Load the equivalent production page with a control cookie and confirm the key + is absent. +5. Confirm `ts_initial=1` remains limited to its existing initial-slot + lifecycle. +6. Validate the deployed CSP by proving `adSlots`, the GPT bootstrap, `bids`, + and the initial `adInit` handoff execute on a representative page with + matched slots. A page that runs only the external bundle is ineligible even + if the fallback marker appears. +7. Set an unrelated page-level targeting key after `ts=true` and confirm both + keys remain on a later request. Treat an explicit page-level or per-key clear + as a failed publisher-code audit, not supported behavior. +8. Validate that IMA/video, direct-tag, server-side GAM, and nested GPT + inventory without an independently TS-rewritten document is absent from the + experiment and paired report scope. Directly validate any independently + rewritten nested documents that are intentionally included. +9. Run a short GAM report and verify treatment totals appear under `ts=true` + while overall totals remain unchanged apart from normal reporting latency. + +## Rollout + +1. Audit response eligibility, including HTML rewriting, `` ordering, CSP, + and publisher GPT calls that could precede or remove the marker. +2. Audit the `ts` key across publisher GPT code, effective `trusted-server.toml` + creative-opportunity targeting maps, and every GAM custom-targeting consumer. +3. Prove through router or access logs that non-experiment traffic is excluded + from the TS route or independently separable in both paired GAM reports. +4. Exclude IMA/video, direct-tag, server-side GAM, and nested GPT inventory + whose document is not independently rewritten. Inventory in intentionally + rewritten nested documents must pass the same request and report validation + as the top-level document. +5. Create and enable the reportable GAM key and predefined value. +6. Deploy the Trusted Server marker before assigning experiment traffic. +7. Provision the scheduled synthetic crawl, assign an incident owner, and obtain + one successful treatment/control run covering initial, lazy, refreshed, and + CSP execution checks. +8. Validate treatment and control requests manually. +9. Start the small cookie-sticky cohort. +10. Compare observed GAM treatment share with the router allocation before using + the results for performance decisions. +11. Monitor normalized metrics over a sufficiently large window; do not infer a + treatment effect from unequal raw totals. + +Rollback stops adding the page-level key to newly loaded documents. Already-open +documents—including long-lived SPA sessions—retain page-level targeting and may +continue issuing marked lazy or refreshed requests after the code rollback. +Record the rollback timestamp, end the experiment reports at the last clean +pre-rollback boundary, and exclude the post-rollback drain interval from both +cohorts. The drain ends only after router/access logs and GAM show no remaining +`ts=true` traffic for one complete agreed reporting interval and a fresh +synthetic navigation confirms that new documents are unmarked. If marked traffic +persists, the interval remains excluded rather than being inferred as control. +Historical GAM rows recorded while the key was active remain valid, and the GAM +key may stay defined and reportable for historical analysis. + +## Alternatives considered + +### Reuse `ts_initial=1` + +Rejected because the key is slot-level, covers only TS-managed initial slots, +and is deliberately cleared on refresh. Changing its lifecycle would also break +its existing ownership semantics. + +### Add slot-level `ts=true` in `adInit` + +Rejected because it would miss publisher-owned or lazy slots that do not pass +through `adInit`, and existing refresh cleanup could remove it. It would measure +auction participation rather than page delivery. + +### Set `ts=true` only in the bootstrap + +Rejected as the sole path. Placing the enqueue before the existing `ts.adInit` +guard correctly handles a pre-installed ad-init implementation. The bundle is +not needed for that case. It remains useful when the inline script unexpectedly +stops executing but the synchronous first-party bundle still runs: without the +fallback, a TS-delivered treatment page would be silently inferred as control. +The fallback does not rescue the simultaneously blocked TS ad-stack scripts, so +that state is an incident rather than an eligible deployment mode. + +### Use a longer descriptive key name such as `trusted_server` + +A descriptive name would lower the collision risk that the short `ts` key +carries. Rejected because GAM limits a custom-targeting key name to 10 +characters, so `trusted_server` (14) cannot be created as a reportable key. The +short `ts` name is therefore mandatory, and the cross-system collision audit is +the compensating control. Do not dual-write `ts` alongside any longer alias: two +names for one cohort would increase GAM setup and audit surface and permit +silent drift between reports. Only `ts=true` is valid. + +### Configure `ts=true` in creative-opportunity slot targeting + +Rejected because creative-opportunity targeting applies only to matched slots. +The experiment requirement covers every request from each successfully rewritten +document's local GPT PubAds service. + +### Rewrite `cust_params` on GAM network requests + +Rejected because it depends on GPT's internal request construction and encoding, +adds interception risk, and duplicates a supported GPT targeting API. + +### Mark production explicitly with `ts=false` + +Preferred in a fully controlled experiment, but unavailable because the +production path cannot be changed. The design documents the resulting unmarked +baseline limitation and requires coverage checks. + +## Acceptance criteria + +1. For a deployment that satisfies the documented GPT-integration, HTML-rewrite, + `` ordering, CSP, reserved-key, request-scope, and targeting-cleanup + prerequisites—and in which a Trusted Server marker callback runs before the + first request—every request from that rewritten document's local GPT PubAds + service carries `ts=true`, including initial, lazy, refreshed, + publisher-owned, and SPA-route requests. A nested document is covered only + when its own HTML response independently satisfies the same prerequisites. +2. Production/control pages remain unmodified and do not carry `ts` from this + feature. +3. `ts_initial=1` retains its current slot-level initial-request lifecycle. +4. The new key is not cleared by Prebid refresh or SPA slot cleanup. +5. No publisher code, effective Trusted Server creative-opportunity targeting + map, or GAM custom-targeting consumer changes ad eligibility, pricing, + protection, routing, or the marker value because of the measurement key. +6. The marker adds no unique identifier, cookie value, network request, or + blocking work. Its disclosure of treatment-path membership to GAM has passed + the publisher's privacy/data-governance review. +7. GAM can report treatment impressions and clicks under `ts=true`, and the + control counts can be derived within the same experiment scope using + identical non-targeted metrics. +8. Known treatment requests are validated directly for marker presence, and the + observed GAM treatment share is compared diagnostically with the router's + expected cohort allocation before experiment results are interpreted. +9. Non-experiment TS traffic is absent from the route during the measurement + window or excluded from both paired GAM reports with identical filters. +10. Only `ts=true` is emitted and reported; no other value (for example `ts=1`) + and no alternative key name (for example `trusted_server`) alias or + dual-write compatibility path exists. +11. CSP-compatible inline execution is proven before launch. A fallback-only + page remains attributed to treatment but raises an incident and cannot be + treated as a healthy experiment page. +12. Rollback reporting excludes already-open documents until observed marked + traffic has drained according to the documented boundary rule. From 2650aa27294fa726456cc47f75a124aab5db7254 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 13 Aug 2026 17:03:58 -0500 Subject: [PATCH 163/195] Show observed slot size in GPT diagnostics --- .../trusted-server-js/lib/src/core/types.ts | 6 + .../src/integrations/gpt_diagnostics/api.ts | 2 + .../integrations/gpt_diagnostics/badges.ts | 5 +- .../src/integrations/gpt_diagnostics/index.ts | 5 + .../integrations/gpt_diagnostics/overlay.ts | 5 +- .../gpt_diagnostics/slot_size_observer.ts | 146 ++++++++++++++++ .../src/integrations/gpt_diagnostics/store.ts | 40 +++++ .../integrations/gpt_diagnostics/api.test.ts | 1 + .../gpt_diagnostics/badges.test.ts | 2 +- .../gpt_diagnostics/overlay.test.ts | 2 +- .../slot_size_observer.test.ts | 158 ++++++++++++++++++ .../gpt_diagnostics/store.test.ts | 32 ++++ docs/guide/integrations/gpt-diagnostics.md | 18 +- 13 files changed, 417 insertions(+), 5 deletions(-) create mode 100644 crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..464203046 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -207,7 +207,13 @@ export interface GptDiagnosticsRequestCycle { viewableAtMs?: number; durations: GptDiagnosticsDurations; isEmpty?: boolean; + /** Exact size fact GPT reported in its `slotRenderEnded` callback. */ size?: Size; + /** + * Outer CSS box observed on the uniquely bound, connected slot element after + * a filled GPT render. This is not an assertion about internal creative pixels. + */ + observedSlotSize?: Size; isBackfill?: boolean; slotContentChanged?: boolean; incompleteSequence: boolean; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts index 99f876b3f..8871165dd 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts @@ -64,6 +64,7 @@ function cloneExportSnapshot(snapshot: GptDiagnosticsExportV1): GptDiagnosticsEx ...cycle, durations: { ...cycle.durations }, size: cycle.size ? [...cycle.size] : undefined, + observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, adManager: cycle.adManager ? { ...cycle.adManager, @@ -192,6 +193,7 @@ export class GptDiagnosticsApiController { ...cycle, durations: { ...cycle.durations }, size: cycle.size ? [...cycle.size] : undefined, + observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, adManager: cycle.adManager ? { ...cycle.adManager, diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts index 57fce3d85..ab39f1f09 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts @@ -115,7 +115,10 @@ function badgeText(cycle: GptDiagnosticsRequestCycle): string { const delivery = deliveryLabel(cycle); if (delivery) firstLine.push(delivery); if (cycle.requestPath === 'competing') firstLine.push('Competing paths'); - if (cycle.size) firstLine.push(`${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.size) firstLine.push(`GPT ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.observedSlotSize) { + firstLine.push(`Box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + } const timingLine: string[] = []; const response = formatMilliseconds(cycle.durations.requestToResponseMs); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts index 75bf97823..d7271710c 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts @@ -7,6 +7,7 @@ import { GptDiagnosticsBindingManager } from './binding'; import { GptDiagnosticsObserver } from './observer'; import type { GptObserverWindow } from './observer'; import { GptDiagnosticsOverlay } from './overlay'; +import { GptDiagnosticsSlotSizeObserver } from './slot_size_observer'; import { GptDiagnosticsStore } from './store'; interface GptDiagnosticsRuntime { @@ -44,6 +45,7 @@ export function installGptDiagnosticsRuntime( let bindings: GptDiagnosticsBindingManager | undefined; let badges: GptDiagnosticsBadgeManager | undefined; let overlay: GptDiagnosticsOverlay | undefined; + let slotSizeObserver: GptDiagnosticsSlotSizeObserver | undefined; let apiController: GptDiagnosticsApiController | undefined; try { @@ -59,6 +61,7 @@ export function installGptDiagnosticsRuntime( window: target, document: target.document, }); + slotSizeObserver = new GptDiagnosticsSlotSizeObserver(store, bindings, { window: target }); overlay = new GptDiagnosticsOverlay(store, bindings, { window: target, document: target.document, @@ -83,6 +86,7 @@ export function installGptDiagnosticsRuntime( apiController?.destroy(); overlay?.destroy(); badges?.destroy(); + slotSizeObserver?.destroy(); bindings?.destroy(); delete target.__tsjs_gpt_diagnostics_runtime; }, @@ -95,6 +99,7 @@ export function installGptDiagnosticsRuntime( apiController?.destroy(); overlay?.destroy(); badges?.destroy(); + slotSizeObserver?.destroy(); bindings?.destroy(); log.warn('gpt diagnostics: runtime installation failed', error); return undefined; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts index e99f1345b..ca5d8dd6d 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts @@ -277,7 +277,10 @@ function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { if (cycle.loadAtMs !== undefined) facts.push('GPT slot onload observed'); if (cycle.viewableAtMs !== undefined) facts.push('GPT impressionViewable observed'); if (cycle.incompleteSequence) facts.push('Incomplete sequence'); - if (cycle.size) facts.push(`Rendered size ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.size) facts.push(`GPT reported size ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.observedSlotSize) { + facts.push(`Observed slot box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + } if (cycle.isBackfill !== undefined) facts.push(`Backfill ${cycle.isBackfill ? 'yes' : 'no'}`); if (cycle.slotContentChanged !== undefined) { facts.push(`Slot content changed ${cycle.slotContentChanged ? 'yes' : 'no'}`); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts new file mode 100644 index 000000000..ab49a451a --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts @@ -0,0 +1,146 @@ +import type { Size } from '../../core/types'; + +import type { GptDiagnosticsBindingManager } from './binding'; +import type { GptDiagnosticsStoreSnapshot } from './store'; + +interface SlotSizeStore { + snapshot(): GptDiagnosticsStoreSnapshot; + recordObservedSlotSize(runtimeSlotNumber: number, requestNumber: number, size: Size): void; + subscribe(listener: () => void): () => void; +} + +interface SlotSizeBindings { + get: GptDiagnosticsBindingManager['get']; + subscribe(listener: () => void): () => void; +} + +type SlotSizeWindow = Window & { + ResizeObserver?: typeof ResizeObserver; +}; + +interface SlotSizeObserverOptions { + window?: SlotSizeWindow; + scheduleFrame?: (callback: () => void) => void; +} + +interface ObservedCycle { + runtimeSlotNumber: number; + requestNumber: number; +} + +function defaultScheduleFrame(callback: () => void): void { + if (typeof requestAnimationFrame === 'function') { + requestAnimationFrame(() => callback()); + } else { + queueMicrotask(callback); + } +} + +function latestFilledCycle( + slot: GptDiagnosticsStoreSnapshot['slots'][number] +): ObservedCycle | undefined { + const cycle = slot.requests[slot.requests.length - 1]; + if (!cycle || cycle.isEmpty !== false || cycle.renderAtMs === undefined) return undefined; + return { runtimeSlotNumber: slot.runtimeSlotNumber, requestNumber: cycle.requestNumber }; +} + +/** + * Observes the outer CSS boxes of uniquely bound elements after filled GPT renders. + * + * Measurements remain separately labelled from GPT's reported creative size and + * are conditionally written with the runtime-slot and request-cycle identity that + * was current when the measurement was scheduled. + */ +export class GptDiagnosticsSlotSizeObserver { + private readonly store: SlotSizeStore; + private readonly bindings: SlotSizeBindings; + private readonly window: SlotSizeWindow; + private readonly scheduleFrame: (callback: () => void) => void; + private readonly unsubscribeStore: () => void; + private readonly unsubscribeBindings: () => void; + private resizeObserver?: ResizeObserver; + private refreshScheduled = false; + private destroyed = false; + + constructor( + store: SlotSizeStore, + bindings: SlotSizeBindings, + options: SlotSizeObserverOptions = {} + ) { + this.store = store; + this.bindings = bindings; + this.window = options.window ?? (window as unknown as SlotSizeWindow); + this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.unsubscribeStore = this.store.subscribe(this.scheduleRefresh); + this.unsubscribeBindings = this.bindings.subscribe(this.scheduleRefresh); + this.refresh(); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + this.unsubscribeStore(); + this.unsubscribeBindings(); + this.resizeObserver?.disconnect(); + } + + private readonly scheduleRefresh = (): void => { + if (this.destroyed || this.refreshScheduled) return; + this.refreshScheduled = true; + this.scheduleFrame(() => { + this.refreshScheduled = false; + this.refresh(); + }); + }; + + private refresh(): void { + if (this.destroyed) return; + this.resizeObserver?.disconnect(); + const observations = new Map(); + const ResizeObserverConstructor = this.window.ResizeObserver; + if (typeof ResizeObserverConstructor === 'function') { + this.resizeObserver = new ResizeObserverConstructor((entries) => { + for (const entry of entries) { + const element = entry.target; + if (!(element instanceof this.window.HTMLElement)) continue; + const cycle = observations.get(element); + if (cycle) this.scheduleMeasure(element, cycle); + } + }); + } + + for (const slot of this.store.snapshot().slots) { + const cycle = latestFilledCycle(slot); + const binding = this.bindings.get(slot.runtimeSlotNumber); + if (!cycle || binding.binding.status !== 'bound' || !binding.element?.isConnected) continue; + observations.set(binding.element, cycle); + this.resizeObserver?.observe(binding.element); + this.scheduleMeasure(binding.element, cycle); + } + } + + private scheduleMeasure(element: HTMLElement, cycle: ObservedCycle): void { + this.scheduleFrame(() => this.measure(element, cycle)); + } + + private measure(element: HTMLElement, cycle: ObservedCycle): void { + const binding = this.bindings.get(cycle.runtimeSlotNumber); + if (binding.binding.status !== 'bound' || binding.element !== element || !element.isConnected) { + return; + } + + const rectangle = element.getBoundingClientRect(); + if ( + !Number.isFinite(rectangle.width) || + !Number.isFinite(rectangle.height) || + rectangle.width < 0 || + rectangle.height < 0 + ) { + return; + } + this.store.recordObservedSlotSize(cycle.runtimeSlotNumber, cycle.requestNumber, [ + rectangle.width, + rectangle.height, + ]); + } +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts index 0324a56cc..2917dc206 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts @@ -249,6 +249,7 @@ function copyCycle(cycle: MutableRequestCycle, nowMs: number): GptDiagnosticsReq ...cycle, durations: derivedDurations(cycle), size: cycle.size ? ([...cycle.size] as Size) : undefined, + observedSlotSize: cycle.observedSlotSize ? ([...cycle.observedSlotSize] as Size) : undefined, adManager: cycle.adManager ? { ...cycle.adManager, @@ -666,6 +667,45 @@ export class GptDiagnosticsStore { ); } + /** + * Retain an outer CSS box only when this exact slot and request cycle still + * identify a filled render. Async DOM measurements use this guard so a prior + * render cannot alter a later refresh cycle. + */ + recordObservedSlotSize(runtimeSlotNumber: number, requestNumber: number, size: Size): void { + if ( + !Number.isSafeInteger(requestNumber) || + requestNumber <= 0 || + !Number.isFinite(size[0]) || + !Number.isFinite(size[1]) || + size[0] < 0 || + size[1] < 0 + ) { + return; + } + + const record = this.slots.get(runtimeSlotNumber); + const cycle = record?.requests.find((candidate) => candidate.requestNumber === requestNumber); + if ( + !cycle || + record.requests[record.requests.length - 1] !== cycle || + cycle.isEmpty !== false || + cycle.renderAtMs === undefined + ) { + return; + } + + const observedSlotSize: Size = [size[0], size[1]]; + if ( + cycle.observedSlotSize?.[0] === observedSlotSize[0] && + cycle.observedSlotSize[1] === observedSlotSize[1] + ) { + return; + } + cycle.observedSlotSize = observedSlotSize; + this.notify(); + } + recordSlotOnload(slot: GptDiagnosticsSlotLike): void { const timestampMs = this.timestamp(); this.matchCycle( diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts index 2e2ae2d2b..6ac993df7 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts @@ -348,6 +348,7 @@ describe('GptDiagnosticsApiController', () => { 'incompleteSequence', 'isBackfill', 'isEmpty', + 'observedSlotSize', 'renderAtMs', 'requestNumber', 'requestPath', diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts index 675e4f442..56de45842 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts @@ -260,7 +260,7 @@ describe('GptDiagnosticsBadgeManager', () => { renderToViewableMs: 1000, }, }) - ).toBe('Filled · 728×90\nResponse 276 ms · Render 42 ms\nViewable after 1 s'); + ).toBe('Filled · GPT 728×90\nResponse 276 ms · Render 42 ms\nViewable after 1 s'); expect( gptDiagnosticsBadgeTextForTest({ requestNumber: 1, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts index b89a8f507..e3d80dae0 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts @@ -457,7 +457,7 @@ describe('GptDiagnosticsOverlay', () => { expect(root!.textContent).toContain('/example/site/filled-slot'); expect(root!.textContent).toContain('Empty'); expect(root!.textContent).toContain('Previous requests (1)'); - expect(root!.textContent).toContain('Rendered size 300×250'); + expect(root!.textContent).toContain('GPT reported size 300×250'); expect(root!.textContent).toContain('Backfill yes'); expect(root!.textContent).toContain('GPT slot onload observed'); expect(root!.textContent).toContain('GPT impressionViewable observed'); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts new file mode 100644 index 000000000..86ad532c7 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { GptDiagnosticsRequestCycle } from '../../../src/core/types'; +import { GptDiagnosticsSlotSizeObserver } from '../../../src/integrations/gpt_diagnostics/slot_size_observer'; +import type { GptDiagnosticsStoreSnapshot } from '../../../src/integrations/gpt_diagnostics/store'; + +class ResizeObserverMock { + static instances: ResizeObserverMock[] = []; + readonly observe = vi.fn(); + readonly disconnect = vi.fn(); + + constructor(readonly callback: ResizeObserverCallback) { + ResizeObserverMock.instances.push(this); + } + + emit(element: Element): void { + this.callback([{ target: element } as ResizeObserverEntry], this as unknown as ResizeObserver); + } +} + +function cycle(requestNumber: number, isEmpty: boolean | undefined): GptDiagnosticsRequestCycle { + return { + requestNumber, + isEmpty, + renderAtMs: 1, + durations: {}, + incompleteSequence: false, + }; +} + +function snapshot(requests: GptDiagnosticsRequestCycle[]): GptDiagnosticsStoreSnapshot { + return { + gptObserved: true, + slots: [ + { + runtimeSlotNumber: 1, + slotElementId: 'ad-slot-example', + requests, + }, + ], + callbackIssues: [], + attributionIssues: [], + coverage: { + slotRequested: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotResponseReceived: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotRenderEnded: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotOnload: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + impressionViewable: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotVisibilityChanged: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + }, + metadata: { + droppedCallbacks: 0, + droppedAttributionIssues: 0, + evictedSlots: 0, + evictedRequestCycles: 0, + }, + }; +} + +describe('GptDiagnosticsSlotSizeObserver', () => { + afterEach(() => { + ResizeObserverMock.instances = []; + document.body.replaceChildren(); + }); + + it('keeps GPT 1×1 distinct from the observed outer box and updates it on resize', () => { + const element = document.createElement('div'); + document.body.append(element); + const getBoundingClientRect = vi.spyOn(element, 'getBoundingClientRect'); + getBoundingClientRect.mockReturnValue({ width: 728, height: 90 } as DOMRect); + const requests = [cycle(1, false)]; + requests[0].size = [1, 1]; + const store = { + snapshot: () => snapshot(requests), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [728, 90]); + expect(requests[0].size).toEqual([1, 1]); + + getBoundingClientRect.mockReturnValue({ width: 970, height: 250 } as DOMRect); + ResizeObserverMock.instances.at(-1)!.emit(element); + expect(store.recordObservedSlotSize).toHaveBeenLastCalledWith(1, 1, [970, 250]); + observer.destroy(); + }); + + it.each(['unbound', 'ambiguous'] as const)('does not observe %s slots', (status) => { + const element = document.createElement('div'); + document.body.append(element); + const store = { + snapshot: () => snapshot([cycle(1, false)]), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status }, element, visible: false }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).not.toHaveBeenCalled(); + expect(ResizeObserverMock.instances.at(-1)!.observe).not.toHaveBeenCalled(); + observer.destroy(); + }); + + it('cannot apply a delayed prior-cycle measurement to a later refresh', () => { + const element = document.createElement('div'); + document.body.append(element); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + width: 300, + height: 250, + } as DOMRect); + const requests = [cycle(1, false)]; + const listeners: Array<() => void> = []; + const store = { + snapshot: () => snapshot(requests), + recordObservedSlotSize: vi.fn(), + subscribe: (listener: () => void) => { + listeners.push(listener); + return () => undefined; + }, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + const frames: Array<() => void> = []; + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => frames.push(callback), + }); + const firstObserver = ResizeObserverMock.instances[0]; + + requests.push(cycle(2, false)); + listeners[0](); + frames.shift()!(); + firstObserver.emit(element); + while (frames.length > 0) frames.shift()!(); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [300, 250]); + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 2, [300, 250]); + observer.destroy(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts index 4c6721f3a..2d29d8b18 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts @@ -519,6 +519,38 @@ describe('GptDiagnosticsStore', () => { expect(cycle.responseClass).toBe('reservation'); }); + it('retains an observed outer slot box separately from GPT reported size', () => { + const store = new GptDiagnosticsStore({ now: () => 10 }); + const slot = fakeSlot('ad-slot-outer-box'); + + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false, size: [1, 1] }); + store.recordObservedSlotSize(1, 1, [728, 90]); + + const cycle = store.snapshot().slots[0].requests[0]; + expect(cycle.size).toEqual([1, 1]); + expect(cycle.observedSlotSize).toEqual([728, 90]); + }); + + it('rejects a stale prior-cycle outer-box measurement after a refresh', () => { + const store = new GptDiagnosticsStore({ now: () => 10 }); + const slot = fakeSlot('ad-slot-stale-outer-box'); + + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false }); + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false }); + store.recordObservedSlotSize(1, 1, [300, 250]); + store.recordObservedSlotSize(1, 2, [970, 250]); + + const requests = store.snapshot().slots[0].requests; + expect(requests[0].observedSlotSize).toBeUndefined(); + expect(requests[1].observedSlotSize).toEqual([970, 250]); + }); + it('separates a fill without Ad Manager identifiers from a reservation', () => { const store = new GptDiagnosticsStore({ now: () => 10 }); const slot = fakeSlot('ad-slot-default'); diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index 78c657bf6..d19376b4b 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -89,7 +89,7 @@ Each request cycle can show: - GPT slot-onload, impression-viewable, and visibility observations. - Non-negative request-to-response, response-to-render, render-to-load, and render-to-viewable durations. -- Rendered size, backfill, and slot-content-change facts exposed by GPT. +- GPT-reported rendered size, a separately labelled observed outer slot box when safely bound, backfill, and slot-content-change facts. - Current DOM binding status and viewport intersection. Elapsed time alone never changes a pending GPT request to Incomplete. Incomplete @@ -276,6 +276,22 @@ because selector support is unavailable or throws, the export reports `dom_uniqueness_unverifiable`. Framework replacement of an element with a new unique element using the same exact ID is rebound automatically. +For an explicitly filled render, diagnostics can also retain `observedSlotSize`: the +current outer CSS box of the uniquely bound, connected slot element. This is measured +after `slotRenderEnded`. When `ResizeObserver` is available, it remains current +while that same request cycle is latest for the GPT slot; otherwise it is the most +recently sampled box. It is displayed separately from `size`, which remains the exact +`slotRenderEnded.size` fact GPT reported. The observed box may differ from GPT's +reported size (for example, a flexible APS creative can report `1×1` while its +allocated outer slot box is larger). It is a publisher-page layout measurement, not a +claim about universal internal creative-pixel dimensions. Empty, unbound, missing, or +ambiguous slots do not report an observed box; delayed measurements from an older +cycle are rejected after a refresh. + +Cross-origin and SafeFrame boundaries prevent diagnostics from inspecting iframe +content. It does not inspect iframe content or alter the APS sandbox, so it cannot +use this field to prove the inner creative's pixels. + Badges and the panel live in a closed Shadow DOM. Diagnostics do not add attributes, classes, or inline styles to publisher slot elements. From fa9326900bb43621fdb4202bda5b10de3beb590a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 16:22:38 +0530 Subject: [PATCH 164/195] Merge ESI #1013 into rc/202608 --- .github/workflows/test.yml | 16 + Cargo.lock | 113 +- Cargo.toml | 1 + .../trusted-server-adapter-fastly/Cargo.toml | 2 + .../trusted-server-adapter-fastly/src/app.rs | 40 +- .../src/esi_assembly.rs | 287 + .../trusted-server-adapter-fastly/src/main.rs | 62 +- .../src/template_cache.rs | 505 ++ crates/trusted-server-core/Cargo.toml | 1 + .../benches/html_processor_bench.rs | 8 +- .../src/creative_opportunities.rs | 315 +- .../trusted-server-core/src/html_processor.rs | 147 +- .../src/integrations/datadome/protection.rs | 65 +- .../src/integrations/gpt_bootstrap.js | 6 +- .../src/integrations/gpt_diagnostics.rs | 77 + .../trusted-server-core/src/platform/mod.rs | 13 + .../src/platform/template_assembly.rs | 77 + .../src/platform/template_cache.rs | 1096 +++ .../trusted-server-core/src/platform/types.rs | 74 + crates/trusted-server-core/src/publisher.rs | 7449 ++++++++++++++++- .../src/response_privacy.rs | 112 +- .../trusted-server-js/lib/src/core/types.ts | 12 +- .../lib/src/integrations/gpt/index.ts | 16 +- .../integrations/gpt/gpt_bootstrap.test.ts | 30 + .../gpt/schedule_initial_ad_init.test.ts | 63 + docs/guide/configuration.md | 116 + ...2026-08-08-1009-measurement-and-stage-0.md | 1044 +++ .../2026-08-08-1009-measurement-findings.md | 503 ++ .../2026-08-10-1009-esi-validation-spike.md | 967 +++ .../2026-08-12-1009-esi-merge-hardening.md | 294 + .../2026-08-14-1009-esi-parser-assembly.md | 104 + ...08-esi-cacheable-root-validation-design.md | 864 ++ ...11-1009-streaming-assembly-architecture.md | 260 + ...6-08-12-1009-esi-merge-hardening-design.md | 195 + ...6-08-14-1009-esi-parser-assembly-design.md | 153 + scripts/c2-local-test.sh | 607 ++ trusted-server.example.toml | 25 + 37 files changed, 15276 insertions(+), 443 deletions(-) create mode 100644 crates/trusted-server-adapter-fastly/src/esi_assembly.rs create mode 100644 crates/trusted-server-adapter-fastly/src/template_cache.rs create mode 100644 crates/trusted-server-core/src/platform/template_assembly.rs create mode 100644 crates/trusted-server-core/src/platform/template_cache.rs create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-findings.md create mode 100644 docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md create mode 100644 docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md create mode 100644 docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md create mode 100644 docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md create mode 100644 docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md create mode 100644 docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md create mode 100644 docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md create mode 100755 scripts/c2-local-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb0233b03..4f0bb6cfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,11 @@ jobs: run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT shell: bash + - name: Retrieve Node.js version + id: node-version + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + shell: bash + - name: Set up Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: @@ -45,9 +50,20 @@ jobs: if: steps.cache-viceroy.outputs.cache-hit != 'true' run: cargo install viceroy --version "${{ steps.viceroy-version.outputs.viceroy-version }}" --locked --force + - name: Use Node.js for the served-seam contract + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + - name: Run tests run: cargo test-fastly + - name: Run C2 ESI local harness + run: ./scripts/c2-local-test.sh esi + + - name: Run inline control harness + run: ./scripts/c2-local-test.sh inline + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index ed1b9c7a3..5c4748b41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -254,6 +254,15 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -573,7 +582,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", ] [[package]] @@ -583,7 +603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -916,6 +936,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1041,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1186,7 +1215,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1690,6 +1719,26 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "esi" +version = "0.7.1" +source = "git+https://github.com/stackpop/esi.git?rev=4c53feab4d22ad9a84641b4c46f3f63bc6d197e2#4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" +dependencies = [ + "atoi", + "base64", + "bytes", + "chrono", + "fastly", + "html-escape", + "log", + "md5", + "nom 8.0.0", + "percent-encoding", + "rand 0.10.2", + "regex", + "thiserror 2.0.18", +] + [[package]] name = "etcetera" version = "0.10.0" @@ -2034,6 +2083,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -2188,6 +2238,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html-escape" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" + [[package]] name = "html5ever" version = "0.35.0" @@ -2914,6 +2970,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.8.2" @@ -2984,6 +3046,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num" version = "0.4.3" @@ -3477,7 +3548,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -3775,6 +3846,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3813,6 +3895,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rcgen" version = "0.13.2" @@ -4079,7 +4167,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -4494,7 +4582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4506,7 +4594,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -4518,7 +4606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -5286,9 +5374,11 @@ dependencies = [ "base64", "bytes", "chrono", + "derive_more", "edgezero-adapter-fastly", "edgezero-core", "error-stack", + "esi", "fastly", "fern", "futures", @@ -5386,6 +5476,7 @@ dependencies = [ "hex", "hmac", "http", + "httpdate", "iab_gpp", "jose-jwk", "log", @@ -6338,7 +6429,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", diff --git a/Cargo.toml b/Cargo.toml index a5a63ca3a..ab5638f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ getrandom = "0.2" hex = "0.4.3" hmac = "0.12.1" http = "1.4.0" +httpdate = "1.0.3" http-body-util = "0.1" hyper = "1" hyper-util = "0.1" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b6bc0f1a1..3d42ae388 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -15,9 +15,11 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } +derive_more = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } +esi = { git = "https://github.com/stackpop/esi.git", rev = "4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" } fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d6090c983..6be93b4b3 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -257,6 +257,11 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) + // Spike-only (#1009). Constructed unconditionally, but only read when the + // assembly mode is a shared-template one — which defaults to Inline, so this + // is inert until an operator opts in. + .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) + .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) .http_client(Arc::new(FastlyPlatformHttpClient)) .geo(Arc::new(FastlyPlatformGeo)) @@ -1239,12 +1244,15 @@ mod tests { use super::{ AppState, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, - TrustedServerApp, build_state_from_settings, startup_error_router, + TrustedServerApp, build_per_request_services, build_state_from_settings, + startup_error_router, }; use bytes::Bytes; use edgezero_core::body::Body; + use edgezero_core::context::RequestContext; use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Mutex; @@ -1379,6 +1387,36 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + #[test] + fn per_request_services_register_the_fastly_template_assembler() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = RequestContext::new( + empty_request(Method::GET, "/article"), + PathParams::default(), + ); + + let services = build_per_request_services(&state, &context); + let template = format!( + "article{}", + trusted_server_core::publisher::AD_ASSEMBLY_SEAM + ); + let fragment = b""; + let assembled = services + .template_assembler() + .assemble(template.as_bytes(), fragment) + .expect("Fastly services should provide ESI assembly"); + + assert_eq!( + assembled, + template + .replace( + trusted_server_core::publisher::AD_ASSEMBLY_SEAM, + std::str::from_utf8(fragment).expect("fragment should be UTF-8") + ) + .into_bytes() + ); + } + /// Builds a router whose `AppState` uses a registry containing the given /// request filters (and no routes), so dispatch-level request-filter /// behavior can be exercised without a real integration. diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs new file mode 100644 index 000000000..4a92c6f78 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -0,0 +1,287 @@ +//! Fastly cold-response assembly backed by the repaired `stackpop/esi` parser. +//! +//! C2 stores an inert marker. This module creates one synthetic ESI include only in a +//! request-private working copy, resolves it from an already-built fragment, and never +//! performs an HTTP request. + +use std::io::Cursor; + +use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; +use fastly::http::StatusCode; +use fastly::{Request, Response}; +use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; +use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + +const INTERNAL_FRAGMENT_PATH: &str = "/_ts/internal/reader-ad-state"; +const SYNTHETIC_ESI_INCLUDE: &[u8] = b""; + +/// Why the Fastly ESI adapter refused or failed to assemble a document. +#[derive(Debug, derive_more::Display)] +enum EsiAssemblyError { + /// The inert seam marker was missing or repeated. + #[display("expected exactly one inert seam marker, found {count}")] + InvalidMarkerCount { count: usize }, + /// Publisher bytes contained ESI instructions outside TS's synthetic seam. + #[display("publisher-authored ESI directives are not allowed")] + PublisherEsiDirective, + /// The parser dispatched a URL other than TS's one synthetic fragment. + #[display("unexpected fragment request path `{path}` (query present: {has_query})")] + UnexpectedFragmentRequest { path: String, has_query: bool }, + /// The pinned parser could not process the document. + #[display("ESI processing failed: {message}")] + Processing { message: String }, + /// The parser changed bytes outside the one synthetic include. + #[display("ESI output was not an exact seam substitution")] + OutputMismatch, +} + +impl core::error::Error for EsiAssemblyError {} + +/// ESI configuration with every cache- and recursion-sensitive option explicit. +fn assembly_configuration() -> Configuration { + Configuration::default() + .with_escaped(false) + .with_default_dca(DcaMode::None) + .with_inherit_parent_dca(false) + .with_max_include_depth(1) + .with_edge_control(false) + .with_caching(CacheConfig { + is_includes_cacheable: false, + includes_default_ttl: None, + includes_force_ttl: None, + is_rendered_cacheable: false, + rendered_cache_control: false, + rendered_ttl: None, + }) +} + +fn contains_esi_directive(bytes: &[u8]) -> bool { + bytes + .windows(b" Result<(Vec, usize), EsiAssemblyError> { + let marker = AD_ASSEMBLY_SEAM.as_bytes(); + let positions = template + .windows(marker.len()) + .enumerate() + .filter_map(|(at, window)| (window == marker).then_some(at)) + .collect::>(); + if positions.len() != 1 { + return Err(EsiAssemblyError::InvalidMarkerCount { + count: positions.len(), + }); + } + if contains_esi_directive(template) { + return Err(EsiAssemblyError::PublisherEsiDirective); + } + + let at = positions[0]; + let mut working = + Vec::with_capacity(template.len() - marker.len() + SYNTHETIC_ESI_INCLUDE.len()); + working.extend_from_slice(&template[..at]); + working.extend_from_slice(SYNTHETIC_ESI_INCLUDE); + working.extend_from_slice(&template[at + marker.len()..]); + Ok((working, at)) +} + +fn completed_fragment_response( + request: &Request, + fragment: &[u8], +) -> Result { + let path = request.get_path().to_string(); + let has_query = request.get_url().query().is_some(); + if path != INTERNAL_FRAGMENT_PATH || has_query { + return Err(EsiAssemblyError::UnexpectedFragmentRequest { path, has_query }); + } + + Ok(PendingFragmentContent::CompletedRequest(Box::new( + Response::from_status(StatusCode::OK) + .with_header( + fastly::http::header::CONTENT_TYPE, + "text/html; charset=utf-8", + ) + .with_body(fragment.to_vec()), + ))) +} + +fn assemble_with_observer( + template: &[u8], + fragment: &[u8], + on_dispatch: F, +) -> Result, EsiAssemblyError> +where + F: Fn() + 'static, +{ + let (working, seam_at) = template_with_synthetic_include(template)?; + let fragment_len = fragment.len(); + let fragment_response = fragment.to_vec(); + let dispatcher = move |request, _index| { + on_dispatch(); + completed_fragment_response(&request, &fragment_response) + .map_err(|error| esi::ESIError::FragmentRequestError(error.to_string())) + }; + let mut processor = Processor::new(None, assembly_configuration()); + let mut output = Vec::with_capacity(template.len() + fragment_len); + processor + .process_stream(Cursor::new(working), &mut output, Some(&dispatcher), None) + .map_err(|error| EsiAssemblyError::Processing { + message: error.to_string(), + })?; + let expected_len = template.len() - AD_ASSEMBLY_SEAM.len() + fragment_len; + let output_tail_at = seam_at + fragment_len; + let template_tail_at = seam_at + AD_ASSEMBLY_SEAM.len(); + if output.len() != expected_len + || output[..seam_at] != template[..seam_at] + || &output[seam_at..output_tail_at] != fragment + || output[output_tail_at..] != template[template_tail_at..] + { + return Err(EsiAssemblyError::OutputMismatch); + } + Ok(output) +} + +fn assemble(template: &[u8], fragment: &[u8]) -> Result, EsiAssemblyError> { + assemble_with_observer(template, fragment, || {}) +} + +/// Fastly implementation of the core cold-response assembly boundary. +pub struct FastlyTemplateAssembler; + +impl PlatformTemplateAssembler for FastlyTemplateAssembler { + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError> { + assemble(template, fragment).map_err(|error| TemplateAssemblyError::Failed { + message: error.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + + const FRAGMENT: &[u8] = b""; + + fn template(body: &str) -> Vec { + format!("{body}{AD_ASSEMBLY_SEAM}").into_bytes() + } + + #[test] + fn a_script_larger_than_the_parser_chunk_survives_exactly() { + let script = format!( + "", + "x".repeat(120_000) + ); + let document = template(&script); + let dispatches = Arc::new(AtomicUsize::new(0)); + let observed_dispatches = Arc::clone(&dispatches); + + let assembled = assemble_with_observer(&document, FRAGMENT, move || { + observed_dispatches.fetch_add(1, Ordering::Relaxed); + }) + .expect("should assemble a document with a large script"); + + let seam_at = document + .windows(AD_ASSEMBLY_SEAM.len()) + .position(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) + .expect("should find seam"); + let mut expected = Vec::new(); + expected.extend_from_slice(&document[..seam_at]); + expected.extend_from_slice(FRAGMENT); + expected.extend_from_slice(&document[seam_at + AD_ASSEMBLY_SEAM.len()..]); + + assert_eq!( + assembled, expected, + "ESI must alter only the synthetic seam" + ); + assert_eq!(dispatches.load(Ordering::Relaxed), 1); + } + + #[test] + fn missing_and_repeated_markers_are_rejected_before_parsing() { + let missing = assemble(b"plain", FRAGMENT) + .expect_err("should reject a missing marker"); + let repeated = assemble( + format!("{AD_ASSEMBLY_SEAM}{AD_ASSEMBLY_SEAM}").as_bytes(), + FRAGMENT, + ) + .expect_err("should reject repeated markers"); + + assert!(matches!( + missing, + EsiAssemblyError::InvalidMarkerCount { count: 0 } + )); + assert!(matches!( + repeated, + EsiAssemblyError::InvalidMarkerCount { count: 2 } + )); + } + + #[test] + fn every_publisher_esi_directive_form_is_rejected_case_insensitively() { + for directive in [ + "", + "secret", + "x", + "$(HTTP_HOST)", + "text", + "", + ] { + let error = assemble(&template(directive), FRAGMENT) + .expect_err("should reject publisher-authored ESI"); + + assert!(matches!(error, EsiAssemblyError::PublisherEsiDirective)); + } + } + + #[test] + fn fragment_esi_is_emitted_verbatim_and_never_reparsed() { + let fragment = b""; + + let assembled = assemble(&template("article"), fragment).expect("should assemble"); + + assert!( + assembled + .windows(fragment.len()) + .any(|window| window == fragment), + "fragment bytes must remain data" + ); + } + + #[test] + fn dispatcher_rejects_every_url_except_the_synthetic_internal_one() { + let unexpected = fastly::Request::get("https://example.com/not-the-seam"); + let with_query = + fastly::Request::get("https://example.com/_ts/internal/reader-ad-state?publisher=1"); + + assert!(matches!( + completed_fragment_response(&unexpected, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + assert!(matches!( + completed_fragment_response(&with_query, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + } + + #[test] + fn configuration_cannot_cache_or_reparse_reader_state() { + let configuration = assembly_configuration(); + + assert!(!configuration.cache.is_includes_cacheable); + assert!(configuration.cache.includes_default_ttl.is_none()); + assert!(configuration.cache.includes_force_ttl.is_none()); + assert!(!configuration.cache.is_rendered_cacheable); + assert!(!configuration.cache.rendered_cache_control); + assert!(configuration.cache.rendered_ttl.is_none()); + assert_eq!(configuration.default_dca, DcaMode::None); + assert!(!configuration.inherit_parent_dca); + assert_eq!(configuration.max_include_depth, 1); + assert!(!configuration.enable_edge_control); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 39d35b198..07c042ea8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -29,11 +29,13 @@ mod app; mod backend; mod compat; mod ec_kv; +mod esi_assembly; mod logging; mod management_api; mod middleware; mod platform; mod rate_limiter; +mod template_cache; mod tinybird; use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; @@ -328,14 +330,7 @@ fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, ) { - if let Some(effects) = request_filter_effects { - effects.apply_to_response(&mut response); - } - - // Final cache guard: EC finalization and request-filter effects may have - // added a per-user Set-Cookie after `apply_finalize_headers` ran, so - // re-apply the privacy downgrade before send. - crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + apply_terminal_response_effects(&mut response, request_filter_effects); let (parts, body) = response.into_parts(); @@ -364,6 +359,26 @@ fn send_edgezero_response( } } +/// Apply every late response mutation, then restore privacy invariants before headers commit. +fn apply_terminal_response_effects( + response: &mut HttpResponse, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = + trusted_server_core::response_privacy::is_private_or_no_store(response.headers()); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + trusted_server_core::response_privacy::enforce_private_no_store(response); + } + + // Final cache guard: EC finalization and request-filter effects may have + // added a per-user Set-Cookie after `apply_finalize_headers` ran, so + // re-apply the privacy downgrade before send. + crate::middleware::enforce_set_cookie_cache_privacy(response); +} + const FALLBACK_UNAVAILABLE: &str = "unavailable"; const FALLBACK_NOT_SENT: &str = "not sent"; const FALLBACK_NONE: &str = "none"; @@ -485,6 +500,7 @@ mod tests { use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use trusted_server_core::integrations::HeaderMutation; fn test_settings() -> Settings { Settings::from_toml( @@ -557,6 +573,36 @@ mod tests { ); } + #[test] + fn late_filter_effects_cannot_make_an_assembled_response_public() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("etag", "\"reader-document\"") + .body(EdgeBody::empty()) + .expect("should build response"); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + assert!(response.headers().get("etag").is_none()); + } + #[test] #[allow(clippy::panic)] fn entry_point_finalize_skips_geo_lookup_for_401() { diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs new file mode 100644 index 000000000..3f168f574 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -0,0 +1,505 @@ +//! Fastly Core Cache backing for the shared transformed-template cache (C2). +//! +//! Only the Fastly adapter implements this; every other adapter uses +//! `UnavailableTemplateCache`, so the ESI assembly mode stays portable and only +//! the caching is Fastly-only. +//! +//! **Why Core Cache and not read-through caching.** Read-through with `after_send` + +//! `set_body_transform` looks like a better fit — it keeps HTTP semantics and derives +//! TTL and surrogate keys from origin headers for free. It is unreachable here: +//! Viceroy 0.17 stubs the entire HTTP Cache ABI and the SDK converts that into a +//! *send error*, so setting `after_send` makes every publisher origin fetch fail +//! under `fastly compute serve`, `cargo test-fastly` and the parity suite. It is also +//! silently dead whenever the origin request is in pass mode, and its closure bounds +//! (`Fn + Send + Sync`) are incompatible with a platform layer that is `!Send` by +//! construction. Recorded in the spike plan's Task 3 Step 4 so nobody re-proposes it. +//! +//! Spike-only. Remove with the spike. + +use fastly::cache::core::{CacheKey, Found, Transaction}; +use std::io::Write as _; +use std::time::Duration; +use trusted_server_core::platform::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, +}; + +/// Surrogate key attached to every stored template, so a single purge clears them +/// all. This is the rollback lever: without it, backing out a bad template means +/// waiting for the TTL. +const PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; + +/// Fastly Core Cache implementation of the C2 template cache. +#[derive(Default)] +pub struct FastlyTemplateCache; + +impl FastlyTemplateCache { + /// Create the Fastly Core Cache implementation. + /// + /// Entry lifetime is supplied per insert after core validates origin freshness + /// and applies the operator's configured safety ceiling. + #[must_use] + pub const fn new() -> Self { + Self + } +} + +fn backend_error(message: impl Into) -> TemplateCacheError { + TemplateCacheError::Backend { + message: message.into(), + } +} + +enum ReadFoundError { + Invalid(TemplateCacheMiss), + Backend(TemplateCacheError), +} + +fn read_found(found: &Found, key: &TemplateCacheKey) -> Result { + if found.is_stale() { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::NotFound)); + } + + let metadata = TemplateMetadata::decode(&found.user_metadata()).ok_or( + ReadFoundError::Invalid(TemplateCacheMiss::UnreadableMetadata), + )?; + if metadata.schema_version != key.schema_version { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::SchemaMismatch)); + } + if found + .known_length() + .is_some_and(|length| length != metadata.body_len) + { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + + let body = found + .to_stream() + .map_err(|error| { + ReadFoundError::Backend(backend_error(format!( + "opening cached template body failed: {error:?}" + ))) + })? + .into_bytes(); + if body.len() as u64 != metadata.body_len { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + Ok(TemplateEntry { metadata, body }) +} + +struct FastlyTemplateReservation { + transaction: Transaction, + surrogate_keys: Vec, +} + +impl PlatformTemplateCacheReservation for FastlyTemplateReservation { + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied", + metadata.body_len, + body.len() + ))); + } + + let mut writer = self + .transaction + .insert(max_age) + .surrogate_keys(self.surrogate_keys.iter().map(String::as_str)) + .known_length(body.len() as u64) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + writer + .write_all(&body) + .map_err(|e| backend_error(format!("writing template body failed: {e}")))?; + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.transaction + .cancel_insert_or_update() + .map_err(|e| backend_error(format!("cancelling cache reservation failed: {e:?}"))) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for FastlyTemplateCache { + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + let transaction = Transaction::lookup(CacheKey::from(key.to_cache_key().into_bytes())) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + if transaction.must_insert_or_update() { + return Ok(TemplateCacheLookup::Reserved( + TemplateCacheReservation::new(Box::new(FastlyTemplateReservation { + transaction, + surrogate_keys: key.surrogate_keys(), + })), + )); + } + + let found = transaction.found().ok_or_else(|| { + backend_error("transaction returned neither a hit nor an insert obligation") + })?; + Ok(match read_found(&found, key) { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(ReadFoundError::Invalid(miss)) => TemplateCacheLookup::Invalid(miss), + Err(ReadFoundError::Backend(error)) => return Err(error), + }) + } + + async fn get(&self, key: &TemplateCacheKey) -> Result { + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // A plain lookup, not a transaction: a read that does not intend to insert + // must not take an insert obligation it will never discharge, which would + // block every other client waiting on the same key until they time out. + let found = fastly::cache::core::lookup(cache_key) + .execute() + .map_err(|_| TemplateCacheMiss::NotFound)? + .ok_or(TemplateCacheMiss::NotFound)?; + + read_found(&found, key).map_err(|error| match error { + ReadFoundError::Invalid(miss) => miss, + ReadFoundError::Backend(error) => { + // This legacy method cannot expose a backend error. Production uses + // `lookup_or_reserve`, which preserves it for bounded diagnostics. + log::warn!("c2_template_cache legacy read failed: {error}"); + TemplateCacheMiss::NotFound + } + }) + } + + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied; storing \ + this would make every read a truncation miss", + metadata.body_len, + body.len() + ))); + } + + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // Transactional insert so a cold key under load transforms once rather than + // once per concurrent request. + let tx = Transaction::lookup(cache_key) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + // Order matters. A STALE entry sets *both* `found()` and + // `must_insert_or_update()`. Testing `found()` first would return early on + // the stale bytes and never discharge the obligation, leaving every + // concurrent waiter blocked until timeout. + if !tx.must_insert_or_update() { + // Someone else already inserted a fresh entry. Nothing to do, and + // nothing to discharge. + return Ok(()); + } + + // `Transaction::insert` takes `self`, so from here there is no handle left to + // cancel the insert with. A write that fails part-way therefore cannot be + // retracted — which is why `TemplateMetadata::body_len` exists and `get` + // checks it. The metadata is written before the body, so a truncated entry + // still carries the length it was supposed to have. + let surrogate_keys = key.surrogate_keys(); + let mut writer = tx + .insert(max_age) + .surrogate_keys(surrogate_keys.iter().map(String::as_str)) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + + if let Err(e) = writer.write_all(&body) { + // Deliberately not calling `finish()`. An unfinished entry has no known + // length, and even if it is observable, `get`'s length check rejects it. + return Err(backend_error(format!("writing template body failed: {e}"))); + } + + // Required. Without it the object never completes and its length stays + // unknown, so readers see a partial or absent entry. + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + + Ok(()) + } + + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(&key.url_surrogate_key()) + .map_err(|e| backend_error(format!("purging invalid template failed: {e:?}"))) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(PURGE_ALL_SURROGATE_KEY) + .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::creative_opportunities::AssemblyMode; + use trusted_server_core::platform::TEMPLATE_SCHEMA_VERSION; + + /// Distinct per test, so tests sharing the process cache cannot collide. + fn key(url: &str) -> TemplateCacheKey { + TemplateCacheKey { + url: url.to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![trusted_server_core::platform::VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "fp".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + fn metadata_for(body: &[u8]) -> TemplateMetadata { + TemplateMetadata { + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: body.len() as u64, + } + } + + /// The trait is `async_trait(?Send)` and this crate has no async test runtime, + /// so drive the futures directly. + fn run(fut: impl core::future::Future) -> T { + futures::executor::block_on(fut) + } + + fn cache() -> FastlyTemplateCache { + FastlyTemplateCache::new() + } + + #[test] + fn a_stored_template_reads_back_intact() { + let cache = cache(); + let key = key("https://example.com/roundtrip"); + let body = b"template".to_vec(); + let metadata = metadata_for(&body); + + run(cache.put(&key, &metadata, body.clone(), Duration::from_secs(60))) + .expect("should store"); + + let entry = run(cache.get(&key)).expect("should read back"); + assert_eq!(entry.body, body, "bytes must survive the round trip"); + assert_eq!(entry.metadata, metadata, "metadata must survive too"); + } + + #[test] + fn transactional_lookup_reserves_before_insert_then_hits() { + let cache = cache(); + let key = key("https://example.com/pre-origin-reservation"); + let body = b"collapsed".to_vec(); + let metadata = metadata_for(&body); + + let reservation = match run(cache.lookup_or_reserve(&key)).expect("lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation, + _ => panic!("a cold transactional lookup must assign the insert obligation"), + }; + reservation + .insert(&metadata, body.clone(), Duration::from_secs(17)) + .expect("reservation should insert"); + + match run(cache.lookup_or_reserve(&key)).expect("warm lookup should work") { + TemplateCacheLookup::Hit(entry) => assert_eq!(entry.body, body), + _ => panic!("the next transactional lookup must see the inserted template"), + } + } + + #[test] + fn an_absent_key_is_a_miss_not_an_error() { + let miss = + run(cache().get(&key("https://example.com/never-stored"))).expect_err("should miss"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn a_different_assembly_mode_does_not_read_the_same_entry() { + // The arms emit different bytes. If they shared an entry, one would serve + // the other's template. + let cache = cache(); + let esi = key("https://example.com/mode-split"); + let mut inline = esi.clone(); + inline.assembly_mode = AssemblyMode::Inline; + + let body = b"esi-template".to_vec(); + run(cache.put(&esi, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + assert_eq!( + run(cache.get(&inline)).err(), + Some(TemplateCacheMiss::NotFound), + "inline must not read the ESI arm's template" + ); + } + + #[test] + fn a_schema_bump_reads_a_miss_rather_than_a_stale_shape() { + let cache = cache(); + let key_v1 = key("https://example.com/schema"); + let body = b"old-shape".to_vec(); + run(cache.put(&key_v1, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + // A deploy that changes the transform bumps the constant. The old entry must + // not be assembled against. + let mut key_v2 = key_v1.clone(); + key_v2.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + + assert_eq!( + run(cache.get(&key_v2)).err(), + Some(TemplateCacheMiss::NotFound), + "a bumped schema changes the key, so the old entry is simply not found" + ); + } + + #[test] + fn a_stale_but_present_entry_reads_as_a_miss_rather_than_being_served() { + // Stale-while-revalidate is a real option and deliberately not taken: it is a + // state machine `cache::core` does not implement for you, and serving stale here + // means serving a template built by an older transform or an older JS bundle. + // + // The entry has to be *present and stale*, not merely expired. A zero TTL with no + // `stale_while_revalidate` window is simply absent, so a test written that way + // passes without ever reaching `is_stale()` — verified: reverting the staleness + // check left that version green. The revalidate window is what keeps the object + // readable while stale, so this actually exercises the branch. + let key = key("https://example.com/stale"); + let body = b"stale-template".to_vec(); + let metadata = metadata_for(&body); + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + let mut writer = fastly::cache::core::insert(cache_key, Duration::from_secs(0)) + .stale_while_revalidate(Duration::from_secs(60)) + .user_metadata(metadata.encode().into()) + .execute() + .expect("should begin insert"); + writer.write_all(&body).expect("should write body"); + writer.finish().expect("should finish insert"); + + let miss = run(cache().get(&key)).expect_err("a stale template must not be served"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn purge_all_clears_stored_templates() { + // The rollback lever. Without this, backing out a bad template means waiting + // for the TTL. + let cache = cache(); + let key = key("https://example.com/purge"); + let body = b"template".to_vec(); + run(cache.put(&key, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + run(cache.get(&key)).expect("should be present before purge"); + + run(cache.purge_all()).expect("should purge"); + + assert!( + run(cache.get(&key)).is_err(), + "purge must clear the template, or rollback is TTL-bound" + ); + } + + #[test] + fn a_second_put_on_a_fresh_entry_is_a_no_op() { + // Exercises the `must_insert_or_update` early return: a concurrent writer + // that finds a fresh entry must neither error nor overwrite. + let cache = cache(); + let key = key("https://example.com/second-put"); + let first = b"first".to_vec(); + run(cache.put( + &key, + &metadata_for(&first), + first.clone(), + Duration::from_secs(60), + )) + .expect("first put stores"); + + let second = b"second".to_vec(); + run(cache.put( + &key, + &metadata_for(&second), + second, + Duration::from_secs(60), + )) + .expect("second put should be a no-op, not an error"); + + assert_eq!( + run(cache.get(&key)).expect("should read").body, + first, + "a fresh entry must not be overwritten by a racing writer" + ); + } + + #[test] + fn the_cache_round_trips_through_the_platform_trait_object() { + // Every other test here calls `FastlyTemplateCache` concretely. The publisher + // never does — it reaches the cache as a `dyn PlatformTemplateCache` behind + // `RuntimeServices`. That join is what `app.rs` wires, and until this test it + // was only type-checked, never executed. + let cache: std::sync::Arc = std::sync::Arc::new(cache()); + let key = key("https://example.com/via-trait-object"); + let body = b"template".to_vec(); + + run(cache.put( + &key, + &metadata_for(&body), + body.clone(), + Duration::from_secs(60), + )) + .expect("should store"); + + assert_eq!( + run(cache.get(&key)).expect("should read back").body, + body, + "the trait object must reach the same Core Cache the concrete type does" + ); + } + + #[test] + fn a_length_mismatch_is_refused_at_write_rather_than_stored() { + // Storing metadata whose length disagrees with the body would make every + // subsequent read a truncation miss — a cache that silently never hits. + // Catch it at the write instead. + let cache = cache(); + let key = key("https://example.com/length-mismatch"); + let mut metadata = metadata_for(b"12345"); + metadata.body_len = 999; + + let err = run(cache.put(&key, &metadata, b"12345".to_vec(), Duration::from_secs(60))) + .expect_err("a length mismatch must be refused"); + assert!( + matches!(err, TemplateCacheError::Backend { .. }), + "expected a backend error, got {err:?}" + ); + } +} diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index c035799e9..e44d46f77 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -29,6 +29,7 @@ glob = { workspace = true } hex = { workspace = true } hmac = { workspace = true } http = { workspace = true } +httpdate = { workspace = true } iab_gpp = { workspace = true } jose-jwk = { workspace = true } log = { workspace = true } diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 7c1303dd4..de968301c 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -1,5 +1,7 @@ use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; -use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; +use trusted_server_core::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor, +}; use trusted_server_core::integrations::IntegrationRegistry; use trusted_server_core::streaming_processor::StreamProcessor as _; @@ -13,6 +15,10 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + // The benchmark measures URL rewriting, not ad injection, and + // `ad_slots_script` is `None` here — matching the previous behaviour, + // which inferred no body-close work from that. + body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cd11e1c14..fca317440 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -16,6 +16,8 @@ use crate::settings::vec_from_seq_or_map; const MAX_DYNAMIC_GAM_UNIT_PATH_BYTES: usize = 100; const MAX_SECTION_BYTES: usize = 100; +const DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS: u32 = 60; +const MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS: u32 = 86_400; /// A single parsed segment of a [`gam_unit_path`](CreativeOpportunitySlot::gam_unit_path) template. #[derive(Debug, Clone)] @@ -191,6 +193,36 @@ const fn is_default_enabled(value: &bool) -> bool { *value == default_enabled() } +/// How per-user ad state reaches the page. +/// +/// `Inline` is the shipped behaviour: the auction result is injected before +/// `` and the root document is therefore uncacheable. `Esi` stores a +/// request-neutral shared template and fills its per-request byte seam at the edge. +/// +/// Spike-only, for the #1009 ESI validation. Remove with the spike. +/// +/// # Why the template must be request-neutral +/// +/// Under `Esi` the template is shared across visitors, so +/// nothing whose *presence* depends on the request may appear in it — not merely +/// nothing whose *value* does. `tsjs.adSlots` is the trap: its content is derived +/// from config and path, but whether it is emitted at all is gated on consent, +/// bot classification, prefetch status and the auction kill switch. A template +/// filled by the first request would freeze that request's decision for every +/// later reader. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + /// Inject bids inline before ``. Root uncacheable. Shipped behaviour. + #[default] + Inline, + /// Serve a shared template; assemble its inert marker with an exact byte split. + /// + /// The operator-facing spelling remains `esi` for continuity, but no general + /// purpose ESI parser executes on this path. + Esi, +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -261,11 +293,104 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, + /// How per-user ad state reaches the page. Absent means + /// [`AssemblyMode::Inline`], the shipped behaviour. + /// + /// `Option` rather than a bare enum, and `skip_serializing_if`, deliberately: + /// these structs use `deny_unknown_fields`, so a pushed key makes an older + /// binary fail configuration load. Keeping it absent when unset means a + /// deployment that never sets it stays rollback-compatible. + /// + /// Spike-only. See [`AssemblyMode`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assembly_mode: Option, + /// Request headers the origin varies on, which the shared-template cache key must + /// cover. + /// + /// Operator-stated because a cache **lookup happens before the fetch**, so on a cold + /// key the origin's `Vary` is not yet known. See `VarySpec` for why the alternatives + /// (two-phase lookup, or storing the list and re-keying) were not taken. + /// + /// **Unset or empty means no operator-stated header is covered, so any origin + /// `Vary` other than structurally covered `Accept-Encoding` disqualifies the + /// response.** `Cookie` may never be configured: a per-cookie object violates the + /// reader-neutral template contract. This fail-closed default prevents a deployment + /// that has not stated what its origin varies on from gaining a shared cache by + /// omission. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_vary: Option>, + /// Maximum time a reader-neutral transformed template may remain in C2. + /// + /// This is a safety ceiling, not freshness authorization. The origin must still + /// provide positive shared freshness, and the stored lifetime is the smaller of + /// the origin's remaining edge freshness and this value. Defaults to 60 seconds + /// and may be configured from 1 second through 1 day. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_max_age_seconds: Option, + /// Operator assertion that the origin's HTML does not depend on request cookies. + /// + /// Unset or `false` disqualifies **every cookie-bearing request** from the shared + /// template cache, in both directions. That is safe and it is also very nearly a + /// disable switch: Trusted Server sets its own identity cookie, so essentially every + /// repeat visitor carries one. Left at the default, the cache can only ever serve + /// first-ever page views and cookie-less clients. + /// + /// Setting `true` asserts the origin serves the same HTML with or without cookies. + /// It is not taken on trust alone — if the origin ever declares `Vary: Cookie`, the + /// response is refused regardless of this flag or the configured key. So a wrong + /// assertion is caught whenever the origin is honest about it, and this only widens + /// the window where the origin personalizes *silently*. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_is_cookie_independent: Option, /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } +impl CreativeOpportunitiesConfig { + /// Resolved assembly mode, defaulting to [`AssemblyMode::Inline`] when unset. + #[must_use] + pub fn assembly_mode(&self) -> AssemblyMode { + self.assembly_mode.unwrap_or_default() + } + + /// Whether a cookie-bearing request may participate in the shared cache. + /// + /// Defaults to `false`, which is the conservative reading and also the one that + /// makes the cache almost inert on real traffic. See + /// [`Self::origin_is_cookie_independent`]. + #[must_use] + pub fn origin_is_cookie_independent(&self) -> bool { + self.origin_is_cookie_independent.unwrap_or(false) + } + + /// Headers the cache key covers, per operator config. + /// + /// Unset yields an empty operator spec, so any origin `Vary` other than the + /// structurally covered `Accept-Encoding` reads as a gap and the response is never + /// cached. Failing closed is deliberate: an unconfigured deployment should not + /// acquire a shared cache silently. + #[must_use] + pub fn template_cache_vary(&self) -> crate::platform::VarySpec { + crate::platform::VarySpec::new(self.template_cache_vary.clone().unwrap_or_default()) + } + + /// Safety ceiling for one shared transformed-template cache entry. + #[must_use] + pub fn template_cache_max_age(&self) -> std::time::Duration { + std::time::Duration::from_secs(u64::from( + self.template_cache_max_age_seconds + .unwrap_or(DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS), + )) + } +} + impl CreativeOpportunitiesConfig { /// Derives the `{section}` value for `path` under this config's section /// policy ([`section_root`](Self::section_root) and @@ -333,10 +458,32 @@ impl CreativeOpportunitiesConfig { /// Returns an error string when [`gam_network_id`](Self::gam_network_id) is /// blank but consumed by a default path or `{network_id}` template; when a /// slot has an invalid identifier, page pattern set, format list, or - /// dimensions; when a `{section}` template lacks a valid + /// dimensions; when `template_cache_max_age_seconds` falls outside 1–86,400; + /// when a `{section}` template lacks a valid /// [`section_root`](Self::section_root); or when configured values make a /// dynamic path exceed 100 UTF-8 bytes. pub fn validate_runtime(&self) -> Result<(), String> { + if self + .template_cache_max_age_seconds + .is_some_and(|seconds| !(1..=MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS).contains(&seconds)) + { + return Err(format!( + "template_cache_max_age_seconds must be between 1 and {MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS}" + )); + } + + if let Some(names) = &self.template_cache_vary { + crate::platform::VarySpec::try_new(names.clone()).map_err(|name| { + format!("template_cache_vary contains invalid HTTP header name `{name}`") + })?; + if names.iter().any(|name| name.eq_ignore_ascii_case("cookie")) { + return Err( + "template_cache_vary must not include Cookie; C2 templates are reader-neutral" + .to_string(), + ); + } + } + // A network ID is required only when a slot renders the default // `//` path or substitutes `{network_id}`. Static // and `{slot_id}`/`{section}`-only templates leave it inert. @@ -1197,6 +1344,10 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: vec![slot], } @@ -1595,6 +1746,10 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), }; @@ -1872,6 +2027,164 @@ mod tests { ); } + #[test] + fn assembly_mode_defaults_to_inline_when_absent() { + // Arrange: the minimal config an existing deployment would have. + let toml = r#" + gam_network_id = "99999" + "#; + + // Act + let config: CreativeOpportunitiesConfig = + toml::from_str(toml).expect("should deserialize without assembly_mode"); + + // Assert + assert_eq!( + config.assembly_mode, None, + "an absent key should stay absent rather than materializing a value" + ); + assert_eq!( + config.assembly_mode(), + AssemblyMode::Inline, + "should resolve to the shipped inline behaviour" + ); + } + + #[test] + fn assembly_mode_deserializes_each_variant() { + for (raw, expected) in [("inline", AssemblyMode::Inline), ("esi", AssemblyMode::Esi)] { + let toml = format!( + r#" + gam_network_id = "99999" + assembly_mode = "{raw}" + "# + ); + let config: CreativeOpportunitiesConfig = + toml::from_str(&toml).unwrap_or_else(|e| panic!("should parse {raw}: {e}")); + assert_eq!( + config.assembly_mode(), + expected, + "should resolve `{raw}` to {expected:?}" + ); + } + + let removed_mode = r#" + gam_network_id = "99999" + assembly_mode = "client_fill" + "#; + assert!( + toml::from_str::(removed_mode).is_err(), + "client_fill is outside #1009's ESI byte-seam design and must be rejected" + ); + } + + #[test] + fn template_cache_vary_rejects_invalid_header_names() { + let config: CreativeOpportunitiesConfig = toml::from_str( + r#" + gam_network_id = "99999" + template_cache_vary = ["rsc", "not a header"] + "#, + ) + .expect("shape should deserialize before runtime validation"); + let err = config + .validate_runtime() + .expect_err("invalid field names must fail configuration validation"); + assert!(err.contains("not a header"), "unexpected error: {err}"); + + let cookie_key: CreativeOpportunitiesConfig = toml::from_str( + r#" + gam_network_id = "99999" + template_cache_vary = ["Cookie"] + "#, + ) + .expect("shape should deserialize before runtime validation"); + let err = cookie_key + .validate_runtime() + .expect_err("per-cookie templates violate the reader-neutral C2 contract"); + assert!(err.contains("Cookie"), "unexpected error: {err}"); + } + + #[test] + fn template_cache_max_age_accepts_a_positive_value_up_to_one_day() { + for seconds in [1_u32, 1_200, 86_400] { + let config: CreativeOpportunitiesConfig = toml::from_str(&format!( + r#" + gam_network_id = "99999" + template_cache_max_age_seconds = {seconds} + "# + )) + .unwrap_or_else(|error| panic!("{seconds}s should deserialize: {error}")); + + config + .validate_runtime() + .unwrap_or_else(|error| panic!("{seconds}s should validate: {error}")); + let serialized = serde_json::to_value(config).expect("should serialize config"); + assert_eq!( + serialized + .get("template_cache_max_age_seconds") + .and_then(serde_json::Value::as_u64), + Some(u64::from(seconds)), + "the configured ceiling must survive typed configuration" + ); + } + } + + #[test] + fn template_cache_max_age_rejects_zero_and_more_than_one_day() { + for seconds in [0_u32, 86_401] { + let config: CreativeOpportunitiesConfig = toml::from_str(&format!( + r#" + gam_network_id = "99999" + template_cache_max_age_seconds = {seconds} + "# + )) + .unwrap_or_else(|error| panic!("shape should deserialize before validation: {error}")); + + let error = config + .validate_runtime() + .expect_err("an unsafe template-cache ceiling must fail startup validation"); + assert!( + error.contains("template_cache_max_age_seconds"), + "unexpected validation error: {error}" + ); + } + } + + #[test] + fn unset_template_cache_max_age_is_omitted_for_rollback_compatibility() { + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + assert_eq!( + config.template_cache_max_age(), + std::time::Duration::from_secs(60), + "an absent ceiling must preserve the spike's existing lifetime" + ); + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("template_cache_max_age_seconds"), + "an unset new key must not break rollback to an older binary: {serialized}" + ); + } + + #[test] + fn unset_assembly_mode_is_omitted_from_serialized_config() { + // `deny_unknown_fields` means a pushed key breaks config load on an older + // binary. A deployment that never sets this must not gain the key just by + // round-tripping through a newer one. + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("assembly_mode"), + "unset assembly_mode must not be serialized, got:\n{serialized}" + ); + } + #[test] fn prebid_slot_params_deserializes_without_bidders_field() { let json = r#"{"bidders": {}}"#; diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3bff588fe..711c4e31d 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use lol_html::{ - EndTagHandler, Settings as RewriterSettings, element, + EndTagHandler, Settings as RewriterSettings, doc_comments, element, end, html_content::{ContentType, EndTag}, text, }; @@ -156,6 +156,29 @@ impl StreamProcessor for HtmlWithPostProcessing { fn reset(&mut self) {} } +/// What the `` seam injects. +/// +/// This is a decision, not a side effect of whether the `` script exists. +/// An earlier shape gated body-close injection on `ad_slots_script.is_some()`, +/// which coupled two independent choices: once a shared-template mode stopped +/// emitting the head script, body-close injection silently stopped too. +/// +/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// §6.7. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum BodyCloseInjection { + /// Emit nothing because no slots matched under the inline path. + #[default] + None, + /// Read the auction result from `ad_bids_state` and inject it, falling back to + /// an empty payload. Today's shipped behaviour. + InlineBids, + /// Emit this markup verbatim — an inert marker the assembly step splits on. + /// Must be identical for every request that reaches the transform, or the + /// cached template is not shared-safe. + Marker(String), +} + /// Configuration for HTML processing #[derive(Clone)] pub struct HtmlProcessorConfig { @@ -176,6 +199,9 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// What the `` seam injects. Decided by the caller rather than inferred + /// from [`Self::ad_slots_script`]. + pub body_close: BodyCloseInjection, /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. pub suppress_datadome_client_side_tag: bool, } @@ -199,6 +225,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, } } @@ -221,6 +248,17 @@ impl HtmlProcessorConfig { self } + /// Set what the `` seam injects. + /// + /// Separate from [`with_ad_state`](Self::with_ad_state) because the two are + /// independent decisions: a shared-template mode emits no head script and + /// still needs a body-close marker. + #[must_use] + pub fn with_body_close(mut self, body_close: BodyCloseInjection) -> Self { + self.body_close = body_close; + self + } + /// Attach the request-scoped conditional diagnostics decision. #[must_use] pub fn with_gpt_diagnostics(mut self, decision: Option) -> Self { @@ -318,9 +356,44 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); let ad_slots_script = config.ad_slots_script.clone(); + let body_close = config.body_close.clone(); let ad_bids_state = config.ad_bids_state.clone(); let gpt_diagnostics = config.gpt_diagnostics.clone(); + // A publisher can legitimately emit the same inert comment text as the reserved + // C2 seam, including after ``. Neutralize source comments while they are + // parsed; markup injected by the body end-tag handler is output, not reparsed, so + // the transform-owned marker remains the only exact copy. + let mut document_content_handlers = Vec::new(); + if let BodyCloseInjection::Marker(marker) = &body_close + && let Some(reserved) = marker + .strip_prefix("")) + { + let reserved = reserved.to_string(); + let escaped = format!("x{reserved}"); + document_content_handlers.push(doc_comments!(move |comment| { + if comment.text() == reserved { + comment.set_text(&escaped)?; + } + Ok(()) + })); + } + if let BodyCloseInjection::Marker(marker) = &body_close { + let marker = marker.clone(); + let injected_bids = Arc::clone(&injected_bids); + document_content_handlers.push(end!(move |document_end| { + // HTML fragments and malformed-but-renderable documents may never expose a + // body end tag. Always mint a transform-owned terminal seam in that case; + // otherwise source bytes equal to the reserved marker could be mistaken for + // ownership by the post-transform exact-count validator. + if !injected_bids.swap(true, Ordering::SeqCst) { + document_end.append(&marker, ContentType::Html); + } + Ok(()) + })); + } + let mut element_content_handlers = vec![ // Inject unified tsjs bundle once at the start of element!("head", { @@ -385,29 +458,42 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso element!("body", { let state = ad_bids_state.clone(); let injected_bids = injected_bids.clone(); - let has_slots = ad_slots_script.is_some(); + let body_close = body_close.clone(); move |el| { - if !has_slots { + if matches!(body_close, BodyCloseInjection::None) { return Ok(()); } let state = state.clone(); let injected_bids = injected_bids.clone(); + let body_close = body_close.clone(); if let Some(handlers) = el.end_tag_handlers() { let handler: EndTagHandler<'static> = Box::new(move |end_tag: &mut EndTag<'_>| { if injected_bids.swap(true, Ordering::SeqCst) { return Ok(()); } - let script_guard = state.lock().expect("should lock bid state"); - let bids_script = match &*script_guard { - Some(s) => s.clone(), - None => build_empty_bids_script(), + let markup = match &body_close { + // Verbatim, and identical on every request that + // reaches the transform — that is what makes the + // cached template shared-safe. + BodyCloseInjection::Marker(marker) => marker.clone(), + BodyCloseInjection::InlineBids => { + let script_guard = state.lock().expect("should lock bid state"); + match &*script_guard { + Some(s) => s.clone(), + None => build_empty_bids_script(), + } + } + // Unreachable: the element handler returned early + // above. Kept exhaustive rather than using `_` so a + // new variant is a compile error here. + BodyCloseInjection::None => return Ok(()), }; - end_tag.before(&bids_script, ContentType::Html); + end_tag.before(&markup, ContentType::Html); Ok(()) }); handlers.push(handler); - } else { + } else if matches!(body_close, BodyCloseInjection::InlineBids) { // No end tag (implicitly closed or EOF ``): lol_html // cannot attach an end-tag handler, so tsjs.bids/adInit() are // never injected even though adSlots was injected at ``. @@ -659,6 +745,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } let rewriter_settings = RewriterSettings { + document_content_handlers, element_content_handlers, ..RewriterSettings::default() }; @@ -698,6 +785,7 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), @@ -1599,6 +1687,7 @@ mod tests { #[test] fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1675,6 +1764,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1712,6 +1802,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1750,6 +1841,7 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), @@ -1802,6 +1894,7 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1832,6 +1925,7 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1853,6 +1947,41 @@ mod tests { ); } + #[test] + fn bodyless_marker_mode_emits_an_owned_terminal_seam_even_after_source_bytes() { + const MARKER: &str = ""; + let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::Marker(MARKER.to_string()), + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + }; + let source = + format!(r#""#); + + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(source.as_bytes(), true) + .expect("should process bodyless HTML"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + + assert_eq!( + html.matches(MARKER).count(), + 2, + "one source occurrence plus one transform-owned seam must reach normalization" + ); + assert!( + html.ends_with(MARKER), + "the transform-owned fallback must be unambiguously terminal" + ); + } + #[test] fn response_size_does_not_grow_disproportionately() { // Processing must not expand HTML by more than 1.1× (accounts for the diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 665060014..75de88afb 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -58,14 +58,21 @@ impl DataDomeIntegration { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + let request_method = input.request.method().clone(); match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { - log::error!("[datadome] Protection setup failed open: {err:?}"); + log::error!( + "[datadome] protection decision=failed_open api_status=none datadome_status=unavailable method={} route=continue failure=setup error={err:?}", + request_method, + ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } Err(ProtectionRequestError::Runtime(err)) => { - log::warn!("[datadome] Protection API failed open: {err:?}"); + log::warn!( + "[datadome] protection decision=failed_open api_status=none datadome_status=unavailable method={} route=continue failure=runtime error={err:?}", + request_method, + ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } } @@ -183,15 +190,15 @@ impl DataDomeIntegration { if supplied_values.is_empty() { return false; } + let Some(bypass) = self.active_protection_test_bypass() else { + return false; + }; if supplied_values.len() != 1 { log::warn!( "[datadome] Multiple DataDome test bypass headers supplied; ignoring bypass" ); return false; } - let Some(bypass) = self.active_protection_test_bypass() else { - return false; - }; let store_name = StoreName::from(bypass.credential_secret_store.as_str()); let credential = match services @@ -522,31 +529,21 @@ fn log_protection_skip( reason: ProtectionSkipReason, suppress_client_tag: bool, ) { - let reason = reason.as_str(); - if suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)) - == log::Level::Info - { - log::info!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", - rule_id, - reason, - input.request.method(), - ); - } else if suppress_client_tag { - log::debug!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", - rule_id, - reason, - input.request.method(), - ); + let level = + suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)); + let client_tag = if suppress_client_tag { + " client_tag=omitted" } else { - log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={}", - rule_id, - reason, - input.request.method(), - ); - } + "" + }; + log::log!( + level, + "[datadome] protection decision=skipped rule={} reason={}{} method={}", + rule_id, + reason.as_str(), + client_tag, + input.request.method(), + ); } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -839,7 +836,7 @@ fn truncate_utf8(value: &str, limit: i32) -> String { mod tests { use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, @@ -855,6 +852,8 @@ mod tests { use super::*; + static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); + fn protection_integration() -> Arc { let config = DataDomeConfig { enabled: true, @@ -878,6 +877,9 @@ mod tests { services: &RuntimeServices, request: &mut Request, ) -> RequestFilterDecision { + let _guard = FASTLY_IS_STAGING_ENV_LOCK + .lock() + .expect("should lock staging environment test guard"); temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { settings, @@ -1098,6 +1100,9 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); + let _guard = FASTLY_IS_STAGING_ENV_LOCK + .lock() + .expect("should lock staging environment test guard"); let decision = temp_env::with_var( crate::constants::ENV_FASTLY_IS_STAGING, None::<&str>, diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 86b51ffa7..43934771d 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -94,8 +94,12 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots) { if ((ts.navGeneration || 0) !== 0) return; + // Slots are generation-guarded for the same reason the bids are: the + // shared-template seam sends both, and an assignment made before this call + // would overwrite a committed SPA navigation's slots. + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; var fire = function () { if ((ts.navGeneration || 0) !== 0) return; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 5bd86d19e..7a91eead4 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -114,6 +114,83 @@ impl GptDiagnosticsRequestDecision { } } +impl GptDiagnosticsRequestDecision { + /// An active decision, for tests in other modules that need one. + /// + /// The fields are private and built by `prepare_request` from a cookie or query + /// parameter; there is no other way to obtain an active decision across a module + /// boundary. + #[cfg(test)] + #[must_use] + pub(crate) fn active_for_tests() -> Self { + Self { + active: true, + clean_browser_path_and_query: None, + cookie_action: GptDiagnosticsCookieAction::None, + } + } +} + +#[cfg(test)] +mod head_seam_invariant_tests { + use super::*; + + /// Every combination of the three fields the decision carries. + fn all_decisions() -> Vec { + let mut out = Vec::new(); + for active in [false, true] { + for clean in [None, Some("/clean".to_string())] { + for cookie_action in [ + GptDiagnosticsCookieAction::None, + GptDiagnosticsCookieAction::SetSession, + GptDiagnosticsCookieAction::ClearSession, + ] { + out.push(GptDiagnosticsRequestDecision { + active, + clean_browser_path_and_query: clean.clone(), + cookie_action, + }); + } + } + } + out + } + + #[test] + fn requires_private_no_store_is_a_superset_of_injection() { + // Load-bearing relationship, not an incidental one. Whenever this decision + // injects anything into ``, the response must also be stamped + // `private, no-store` — which is what keeps request-scoped diagnostics out + // of a shared cache if the explicit assembly-mode gate in + // `create_html_stream_processor` is ever removed or bypassed. + // + // If a future change makes a script emit without also requiring the stamp, + // this fails here rather than silently in a cached template. + for decision in all_decisions() { + let injects = + decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + if injects { + assert!( + decision.requires_private_no_store(), + "decision injects into but does not require private/no-store: \ + {decision:?}" + ); + } + } + } + + #[test] + fn a_default_decision_injects_nothing() { + let decision = GptDiagnosticsRequestDecision::default(); + assert_eq!(decision.bootstrap_script(), None); + assert_eq!(decision.module_script_tag(), None); + assert!( + !decision.requires_private_no_store(), + "an inert decision should not force the response private" + ); + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum QueryDirective { Absent, diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 287f1accf..7c80f9e12 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -12,6 +12,7 @@ //! - [`PlatformBackend`] — dynamic backend registration //! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup +//! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! //! ## Platform-Agnostic Components //! @@ -36,6 +37,8 @@ mod error; mod http; mod image_optimizer; mod kv; +mod template_assembly; +mod template_cache; #[cfg(test)] pub(crate) mod test_support; mod traits; @@ -52,6 +55,16 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_assembly::{ + PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, +}; +pub use template_cache::REPLAYABLE_POLICY_HEADERS; +pub use template_cache::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TEMPLATE_SCHEMA_VERSION, + TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, + TemplateCacheReservation, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, + VaryHeaderValues, VarySpec, +}; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs new file mode 100644 index 000000000..441e7ca31 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -0,0 +1,77 @@ +//! Platform boundary for assembling a shared template with reader-specific state. +//! +//! Core owns the cache-safety ordering and the portable byte-seam fallback. An adapter +//! may provide a richer assembler for the cold response after the reader-neutral +//! template has been stored. + +use core::fmt; + +/// Why a platform assembler could not produce a document. +#[derive(Debug, derive_more::Display)] +pub enum TemplateAssemblyError { + /// The adapter has no template assembler. + #[display("this adapter cannot assemble shared templates")] + Unsupported, + /// The platform assembler rejected or could not process the document. + #[display("template assembly failed: {message}")] + Failed { + /// Human-readable failure context. + message: String, + }, +} + +impl core::error::Error for TemplateAssemblyError {} + +/// Assembles reader-specific state into a shared HTML template. +pub trait PlatformTemplateAssembler: Send + Sync { + /// Produce the complete document served to this reader. + /// + /// # Errors + /// + /// Returns [`TemplateAssemblyError`] when the adapter cannot assemble the template. + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError>; +} + +impl fmt::Debug for dyn PlatformTemplateAssembler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateAssembler") + } +} + +/// Default assembler used by adapters that do not provide platform assembly. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnavailableTemplateAssembler; + +impl PlatformTemplateAssembler for UnavailableTemplateAssembler { + fn assemble( + &self, + _template: &[u8], + _fragment: &[u8], + ) -> Result, TemplateAssemblyError> { + Err(TemplateAssemblyError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_assembler_refuses_the_document() { + let error = UnavailableTemplateAssembler + .assemble(b"", b"") + .expect_err("should refuse when platform assembly is unavailable"); + + assert!(matches!(error, TemplateAssemblyError::Unsupported)); + } + + #[test] + fn assembler_contract_is_object_safe() { + let assembler: Box = Box::new(UnavailableTemplateAssembler); + + assert!(matches!( + assembler.assemble(b"template", b"fragment"), + Err(TemplateAssemblyError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs new file mode 100644 index 000000000..ac3761b0f --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -0,0 +1,1096 @@ +//! The shared transformed-template cache (C2) for the #1009 ESI validation spike. +//! +//! Three caches are in play and conflating them is what produced the original wrong +//! conclusion in the design doc, so this module names which one it is: +//! +//! | Cache | Contents | Owner | +//! | ----- | --------------------------------- | ------------------------------ | +//! | C1 | raw origin bytes | Fastly read-through. Not this. | +//! | C2 | post-`lol_html`, pre-assembly | **This module.** | +//! | C3 | final per-user assembled response | **Must never exist.** | +//! +//! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on +//! the request. What may and may not live in it is +//! [§6.7 of the design doc](../../../../docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md), +//! and the invariant is enforced by the rendered-document byte-identity tests in +//! `publisher`. +//! +//! Spike-only. Remove with the spike. + +use core::fmt; +use std::collections::HashSet; + +use crate::creative_opportunities::AssemblyMode; + +/// Version of the transform that produced a cached template. +/// +/// Bump on **any** change to what the transform emits. Without it a deploy reads +/// yesterday's template shape and assembles against markers that moved, which fails +/// as a rendering bug far from its cause rather than as a cache miss. +/// +/// | Version | Transform | +/// | ------- | --------- | +/// | 1 | `` seam used an executable ESI include tag targeting the old fragment endpoint | +/// | 2 | Marker became the inert comment ``; the seam hands slots to `scheduleInitialAdInit` instead of assigning them | +/// | 3 | Marker became ``; canonical collision-safe key, explicit origin freshness, and complete repeated document-policy metadata | +/// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; + +/// Inputs that select one cached template. +/// +/// Every field changes the emitted bytes for the same URL. A signal that changes the +/// bytes and is **not** here produces cross-served templates; a signal that is +/// per-user does not belong here at all — it belongs out of the template entirely. +/// That distinction is the whole design: the key holds per-*variant* signals, and +/// per-*user* signals are excluded from the template rather than keyed on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateCacheKey { + /// Full request URL, stated explicitly rather than inherited from an ambient + /// request, so the key cannot silently depend on what the caller happened to + /// mutate first. + pub url: String, + /// Host and scheme. The post-processed output is host-dependent by construction: + /// both reach `IntegrationHtmlContext` and drive URL rewriting. + pub request_host: String, + /// See [`Self::request_host`]. + pub request_scheme: String, + /// Publisher origin identity, including the outbound Host override. Two virtual + /// hosts can share a connection target while producing unrelated documents. + pub origin_identity: String, + /// Inline and ESI modes emit different template bytes. Without this they poison + /// each other's entries. + pub assembly_mode: AssemblyMode, + /// Values of the request headers the **origin** declares it varies on, in the + /// order the origin listed them. Not a fixed list: the origin is authoritative, + /// and hard-coding one here would silently drift when the origin's changes. + pub vary_values: Vec, + /// Digest of every setting that can shape the transformed template plus the tsjs + /// bundle. Over-invalidating is safe; omitting a shaping input cross-serves bytes. + pub template_fingerprint: String, + /// See [`TEMPLATE_SCHEMA_VERSION`]. + pub schema_version: u32, +} + +impl TemplateCacheKey { + /// Render a fixed-size opaque key for the platform cache. + /// + /// The canonical input is length-prefixed before hashing, so neither delimiters nor + /// raw request values can collide or leak into cache diagnostics. + #[must_use] + pub fn to_cache_key(&self) -> String { + use sha2::Digest as _; + + fn push(out: &mut Vec, part: &[u8]) { + out.extend_from_slice(&(part.len() as u64).to_be_bytes()); + out.extend_from_slice(part); + } + + let mut canonical = Vec::new(); + push(&mut canonical, b"ts-c2"); + push(&mut canonical, &self.schema_version.to_be_bytes()); + push( + &mut canonical, + match self.assembly_mode { + AssemblyMode::Inline => b"inline", + AssemblyMode::Esi => b"esi", + }, + ); + push(&mut canonical, self.request_scheme.as_bytes()); + push(&mut canonical, self.request_host.as_bytes()); + push(&mut canonical, self.origin_identity.as_bytes()); + push(&mut canonical, self.url.as_bytes()); + push(&mut canonical, self.template_fingerprint.as_bytes()); + push( + &mut canonical, + &(self.vary_values.len() as u64).to_be_bytes(), + ); + for varied in &self.vary_values { + push(&mut canonical, varied.name.as_bytes()); + match &varied.values { + None => push(&mut canonical, b"absent"), + Some(values) => { + push(&mut canonical, b"present"); + push(&mut canonical, &(values.len() as u64).to_be_bytes()); + for value in values { + push(&mut canonical, value); + } + } + } + } + + let digest = sha2::Sha256::digest(canonical); + format!("ts-c2-v{}-{}", self.schema_version, hex::encode(digest)) + } + + /// Surrogate keys to attach at insert, for purge-based rollback. + /// + /// `ts-template` purges every template at once, which is the rollback lever. + /// The per-URL key allows targeted invalidation. Both are needed: the broad one + /// for an incident, the narrow one for ordinary invalidation. + #[must_use] + pub fn surrogate_keys(&self) -> Vec { + vec!["ts-template".to_string(), self.url_surrogate_key()] + } + + /// Surrogate key for every variant of this publisher URL. + /// + /// Used to evict a malformed object without flushing unrelated article templates. + #[must_use] + pub fn url_surrogate_key(&self) -> String { + format!("ts-template-url-{}", digest_hex(self.url.as_bytes())) + } +} + +fn digest_hex(bytes: &[u8]) -> String { + use sha2::Digest as _; + hex::encode(sha2::Sha256::digest(bytes)) +} + +/// One configured `Vary` input exactly as it appeared on the request. +/// +/// `None` means absent. `Some(vec![vec![]])` means present with one empty field +/// value. Repeated fields stay separate and ordered; no UTF-8 conversion is involved. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VaryHeaderValues { + /// Validated, lowercase header name. + pub name: String, + /// Every raw field value in wire order, or `None` when absent. + pub values: Option>>, +} + +/// Origin response headers safe to store with a shared template and replay on a hit. +/// +/// Every one is a per-URL policy statement, identical for every reader. Nothing +/// per-reader (`Set-Cookie`) and nothing cache-controlling (`Cache-Control`, `ETag`, +/// `Surrogate-Control`) appears here, and it is an allowlist so a new origin header is +/// excluded until someone decides otherwise. +pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ + "content-security-policy", + "content-security-policy-report-only", + "permissions-policy", + "referrer-policy", + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + "x-frame-options", + "x-content-type-options", + "content-language", + "x-robots-tag", +]; + +/// Headers the key covers by construction, whatever the operator configured. +/// +/// The shared path stores decoded identity bytes and negotiates the reader representation +/// only after assembly, so an origin declaring `Vary: Accept-Encoding` is covered without +/// reader input. This assumes those origin variants differ only by HTTP content coding; +/// operators must leave ESI disabled if an origin changes document semantics instead. +/// Without this carve-out, the ordinary declaration sent by any compressing origin reads +/// as an uncovered gap and disqualifies the response, so **C2 would never cache anything +/// against a real origin** unless the operator redundantly listed a header the transform +/// already normalizes. Found by review before it could make the spike measure a hit rate +/// of approximately zero and read that as a result. +const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; + +/// Request headers to include in the cache key, and where the list comes from. +/// +/// # The chicken-and-egg this resolves +/// +/// The key must cover everything the origin varies on, or two requests needing +/// different templates share one entry. But a **lookup happens before the fetch**, +/// so on a cold key the origin's `Vary` is not yet known. +/// +/// Three ways out, and the trade-off is real: +/// +/// 1. **Configure the list** — what this does. One lookup, no extra round trip, and +/// the operator states what the origin varies on. Cost: it drifts silently if the +/// origin's `Vary` changes and nobody updates config. +/// 2. **Two-phase lookup** — fetch a URL-keyed record holding the last-seen `Vary`, +/// then key properly. Correct, but doubles the lookups on every request. +/// 3. **Store the list alongside** and re-key on mismatch. Same cost as (2) plus +/// complexity. +/// +/// (1) is chosen for the spike because Step A already measured the origin's actual +/// `Vary`, the origin response is checked for drift before storage, and the configured +/// template-cache ceiling bounds how long a newly introduced mismatch can survive. +/// **This is a spike-grade choice, not a production one** — see the drift guard below. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VarySpec { + /// Header names, lowercased, in a fixed order. + names: Vec, +} + +impl VarySpec { + /// Build from configured header names. + /// + /// # Panics + /// + /// Panics when a name is not a valid HTTP field name. Runtime configuration is + /// validated with [`Self::try_new`] before this constructor is used. + #[must_use] + pub fn new(names: impl IntoIterator) -> Self { + Self::try_new(names).expect("VarySpec names should be validated at configuration load") + } + + /// Build from configured names, validating and deduplicating them. + /// + /// # Errors + /// + /// Returns the offending name when it is not a valid HTTP field name. + pub fn try_new(names: impl IntoIterator) -> Result { + let mut seen = HashSet::new(); + let mut normalized = Vec::new(); + for raw in names { + let name = http::header::HeaderName::from_bytes(raw.as_bytes()) + .map_err(|_| raw.clone())? + .as_str() + .to_string(); + if STRUCTURALLY_COVERED.contains(&name.as_str()) { + continue; + } + if seen.insert(name.clone()) { + normalized.push(name); + } + } + Ok(Self { names: normalized }) + } + + /// Configured names, lowercased. + #[must_use] + pub fn names(&self) -> &[String] { + &self.names + } + + /// Extract the key inputs from a request's headers. + /// + /// A header the origin varies on but the request omits still contributes an + /// entry, with an empty value — otherwise "absent" and "present but empty" + /// would collide, and those are different requests to the origin. + #[must_use] + pub fn values_from(&self, headers: &http::HeaderMap) -> Vec { + self.names + .iter() + .map(|name| { + let values = headers.contains_key(name.as_str()).then(|| { + headers + .get_all(name.as_str()) + .iter() + .map(|value| value.as_bytes().to_vec()) + .collect() + }); + VaryHeaderValues { + name: name.clone(), + values, + } + }) + .collect() + } + + /// Whether the origin's declared `Vary` contains anything this spec omits. + /// + /// The drift guard for choice (1) above. Called **after** the origin responds, + /// when its `Vary` is finally known: if the origin varies on something the key + /// did not cover, the template just built is unsafe to store, because a request + /// differing only in that header would read it. + /// + /// Returns the uncovered names, so the caller can log precisely which config is + /// stale rather than reporting a generic refusal. + #[must_use] + pub fn uncovered_by<'a>(&self, origin_vary: impl IntoIterator) -> Vec { + origin_vary + .into_iter() + .flat_map(|value| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .filter(|name| !name.is_empty() && name != "*") + .filter(|name| !STRUCTURALLY_COVERED.contains(&name.as_str())) + .filter(|name| !self.names.contains(name)) + .collect() + } +} + +/// Metadata stored alongside the template bytes. +/// +/// `cache::core` carries **no HTTP semantics** — status, headers, encoding and +/// revalidation are all the caller's. Rather than storing origin headers and +/// replaying them, store only what is needed to rebuild a response from scratch. +/// +/// That choice is deliberate and load-bearing: the publisher path forces +/// `private, no-store` and strips validators *after* the origin send, so replaying a +/// stored origin header would fight it. Rebuilding every header on a hit means no +/// origin header is ever replayed and the `Set-Cookie` privacy net stays trivially +/// safe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateMetadata { + /// Encoding of the stored bytes. C2 writes only `identity`; retaining the field in + /// metadata makes corrupt or stale representations fail validation on read. + pub content_encoding: String, + /// Content type to rebuild the response with. + pub content_type: String, + /// Schema version the bytes were produced under. Checked on read: a mismatch is + /// a miss, not an error, so a rollback to an older binary degrades to + /// re-transforming rather than misassembling. + pub schema_version: u32, + /// Length of the template bytes as written. + /// + /// Guards against a partially written entry. `Transaction::insert` consumes the + /// transaction, so a write that fails part-way cannot cancel the insert — there + /// is no handle left to cancel it with. Recording the intended length and + /// checking it on read makes a truncated entry a miss instead of a silently + /// short template that would assemble into a broken page. + pub body_len: u64, + /// Origin response headers that are policy, not per-reader state. + /// + /// Reconstructing headers from scratch on a hit keeps origin `Set-Cookie` and caching + /// directives out of a shared cache — but it also dropped `Content-Security-Policy`, + /// framing protection and `Content-Language`, weakening the page. These are + /// per-URL and identical for every reader, so they belong with the template. + /// + /// Deliberately an allowlist: anything per-reader or cache-controlling is excluded by + /// construction rather than by remembering to strip it. + pub policy_headers: Vec<(String, String)>, +} + +impl TemplateMetadata { + /// Serialize for `user_metadata`. Deliberately a tiny hand-rolled format rather + /// than JSON — one allocation, no dependency, and a parse failure is + /// unambiguous. + #[must_use] + pub fn encode(&self) -> Vec { + let mut out = format!( + "v={}\nce={}\nct={}\nlen={}", + self.schema_version, self.content_encoding, self.content_type, self.body_len + ); + for (name, value) in &self.policy_headers { + // Header values cannot contain newlines (the HTTP parser rejects them), so a + // newline-delimited encoding cannot be broken by a header value. + out.push_str(&format!("\nh={name}:{value}")); + } + out.into_bytes() + } + + /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers + /// must treat as a cache miss. + #[must_use] + pub fn decode(raw: &[u8]) -> Option { + let text = core::str::from_utf8(raw).ok()?; + let mut schema_version = None; + let mut policy_headers = Vec::new(); + let mut content_encoding = None; + let mut content_type = None; + let mut body_len = None; + for line in text.lines() { + let (key, value) = line.split_once('=')?; + match key { + "v" => { + if schema_version.replace(value.parse().ok()?).is_some() { + return None; + } + } + "ce" => { + if content_encoding.replace(value.to_string()).is_some() { + return None; + } + } + "h" => { + let (name, header_value) = value.split_once(':')?; + let name = http::header::HeaderName::from_bytes(name.as_bytes()).ok()?; + if !REPLAYABLE_POLICY_HEADERS.contains(&name.as_str()) { + return None; + } + http::HeaderValue::from_bytes(header_value.as_bytes()).ok()?; + policy_headers.push((name.as_str().to_string(), header_value.to_string())); + } + "ct" => { + if content_type.replace(value.to_string()).is_some() { + return None; + } + } + "len" => { + if body_len.replace(value.parse().ok()?).is_some() { + return None; + } + } + _ => return None, + } + } + let content_encoding = content_encoding?; + // Every template is decoded before insert. Accepting another value here would + // let corrupt metadata label plaintext bytes as gzip on a warm hit. + if content_encoding != "identity" { + return None; + } + let content_type = content_type?; + http::HeaderValue::from_bytes(content_type.as_bytes()).ok()?; + if !content_type + .split(';') + .next() + .is_some_and(|media_type| media_type.trim().eq_ignore_ascii_case("text/html")) + { + return None; + } + Some(Self { + schema_version: schema_version?, + policy_headers, + content_encoding, + content_type, + body_len: body_len?, + }) + } +} + +/// Why a template read did not produce usable bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum TemplateCacheMiss { + /// No entry for this key. + #[display("no cached template for this key")] + NotFound, + /// Found, but produced by a different transform version. + #[display("cached template has a different schema version")] + SchemaMismatch, + /// Found, but its metadata could not be parsed. + #[display("cached template metadata is unreadable")] + UnreadableMetadata, + /// Found, but shorter than the metadata says it should be — a write that failed + /// part-way. See [`TemplateMetadata::body_len`]. + #[display("cached template is truncated")] + Truncated, + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, +} + +impl core::error::Error for TemplateCacheMiss {} + +/// Errors a template cache write can produce. +#[derive(Debug, derive_more::Display)] +pub enum TemplateCacheError { + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, + /// The platform rejected the operation. + #[display("template cache backend error: {message}")] + Backend { + /// What the backend reported. + message: String, + }, +} + +impl core::error::Error for TemplateCacheError {} + +/// Result of the pre-origin cache transaction. +pub enum TemplateCacheLookup { + /// A fresh usable template. + Hit(TemplateEntry), + /// This request owns the obligation to provide or cancel the cold object. + Reserved(TemplateCacheReservation), + /// This adapter deliberately has no shared-template cache. + Unsupported, + /// A cache object existed but failed schema, metadata, or length validation. + Invalid(TemplateCacheMiss), +} + +/// Platform-owned insert obligation. Dropping it cancels, making every early-return +/// path safe without an async cleanup ladder in the publisher pipeline. +pub struct TemplateCacheReservation { + inner: Option>, +} + +impl core::fmt::Debug for TemplateCacheReservation { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TemplateCacheReservation") + .finish_non_exhaustive() + } +} + +impl TemplateCacheReservation { + /// Wrap a platform reservation. + #[must_use] + pub fn new(inner: Box) -> Self { + Self { inner: Some(inner) } + } + + /// Fulfil the reservation with a validated template. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be fulfilled. + pub fn insert( + mut self, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .insert(metadata, body, max_age) + } + + /// Explicitly give up the reservation. Drop performs the same operation as a net. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be cancelled. + pub fn cancel(mut self) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .cancel() + } +} + +impl Drop for TemplateCacheReservation { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() + && let Err(err) = inner.cancel() + { + log::warn!("c2_template_cache reservation cancellation failed: {err}"); + } + } +} + +/// Adapter-specific ownership token returned by a transactional lookup. +pub trait PlatformTemplateCacheReservation: Send { + /// Insert and discharge the obligation. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when the insert fails. + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Cancel and allow a waiting request to take ownership. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when cancellation fails. + fn cancel(self: Box) -> Result<(), TemplateCacheError>; +} + +impl fmt::Debug for dyn PlatformTemplateCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateCache") + } +} + +/// A platform's shared-template cache. +/// +/// Only the Fastly adapter implements this; every other adapter uses +/// [`UnavailableTemplateCache`], which reports [`TemplateCacheMiss::Unsupported`] so +/// the caller transforms every time rather than failing. +/// +/// `Send + Sync` on the trait, `?Send` on the futures: `RuntimeServices` is held in a +/// `LazyLock` static, so the trait object must cross threads even though the futures +/// themselves never do — the platform layer is `!Send` by construction. +#[async_trait::async_trait(?Send)] +pub trait PlatformTemplateCache: Send + Sync { + /// Transactionally look up a template before origin work begins. + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + Ok(match self.get(key).await { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(TemplateCacheMiss::Unsupported | TemplateCacheMiss::NotFound) => { + TemplateCacheLookup::Unsupported + } + Err(miss) => TemplateCacheLookup::Invalid(miss), + }) + } + + /// Read a template. `Err` is a miss, not a failure — every variant means + /// "transform it yourself". + async fn get(&self, key: &TemplateCacheKey) -> Result; + + /// Store a template. + /// + /// Callers must not call this without having consulted the C2 eligibility gate + /// first: this method stores what it is given and cannot tell a shared template + /// from a per-user one. + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Purge every cached variant for one publisher URL. + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; + + /// Purge every stored template. The rollback lever. + async fn purge_all(&self) -> Result<(), TemplateCacheError>; +} + +/// A template read from the cache. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateEntry { + /// Metadata stored at insert. + pub metadata: TemplateMetadata, + /// The transformed template bytes. + pub body: Vec, +} + +/// The null object, used by every adapter without a template cache. +/// +/// Reporting [`TemplateCacheMiss::Unsupported`] rather than erroring means the +/// ESI assembly mode degrades to transforming per request on Cloudflare, Axum and Spin +/// instead of failing — the mode stays portable, only the caching is not. +pub struct UnavailableTemplateCache; + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for UnavailableTemplateCache { + async fn lookup_or_reserve( + &self, + _key: &TemplateCacheKey, + ) -> Result { + Ok(TemplateCacheLookup::Unsupported) + } + + async fn get(&self, _key: &TemplateCacheKey) -> Result { + Err(TemplateCacheMiss::Unsupported) + } + + async fn put( + &self, + _key: &TemplateCacheKey, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_url(&self, _key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn key() -> TemplateCacheKey { + TemplateCacheKey { + url: "https://example.com/news/article".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "abc123".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + struct CountingReservation(Arc); + + impl PlatformTemplateCacheReservation for CountingReservation { + fn insert( + self: Box, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + #[test] + fn dropping_an_unfulfilled_reservation_cancels_exactly_once() { + let cancellations = Arc::new(AtomicUsize::new(0)); + drop(TemplateCacheReservation::new(Box::new( + CountingReservation(Arc::clone(&cancellations)), + ))); + assert_eq!(cancellations.load(Ordering::SeqCst), 1); + } + + /// Every field must change the key. A field that does not is a cross-serving + /// bug: two requests needing different templates would share one entry. + #[test] + fn every_field_changes_the_key() { + let base = key().to_cache_key(); + + let mut mode = key(); + mode.assembly_mode = AssemblyMode::Inline; + assert_ne!( + mode.to_cache_key(), + base, + "assembly mode must change the key" + ); + + let mut url = key(); + url.url = "https://example.com/other".to_string(); + assert_ne!(url.to_cache_key(), base, "url must change the key"); + + let mut host = key(); + host.request_host = "other.example.com".to_string(); + assert_ne!(host.to_cache_key(), base, "host must change the key"); + + let mut scheme = key(); + scheme.request_scheme = "http".to_string(); + assert_ne!(scheme.to_cache_key(), base, "scheme must change the key"); + + let mut origin = key(); + origin.origin_identity = "https://origin.example.com\0other.example.com".to_string(); + assert_ne!( + origin.to_cache_key(), + base, + "origin Host identity must change the key" + ); + + let mut fingerprint = key(); + fingerprint.template_fingerprint = "def456".to_string(); + assert_ne!( + fingerprint.to_cache_key(), + base, + "template fingerprint must change the key" + ); + + let mut schema = key(); + schema.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + assert_ne!( + schema.to_cache_key(), + base, + "schema version must change the key" + ); + + let mut vary = key(); + vary.vary_values = vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"0".to_vec()]), + }]; + assert_ne!(vary.to_cache_key(), base, "vary values must change the key"); + } + + /// The reason for length prefixes rather than a delimiter. + #[test] + fn values_containing_delimiters_cannot_collide() { + let mut a = key(); + a.request_host = "a".to_string(); + a.url = "b:c".to_string(); + + let mut b = key(); + b.request_host = "a:b".to_string(); + b.url = "c".to_string(); + + assert_ne!( + a.to_cache_key(), + b.to_cache_key(), + "field values containing the delimiter must not produce the same key; a \ + collision here serves one visitor's template to another" + ); + } + + #[test] + fn rendered_key_is_fixed_size_and_contains_no_request_material() { + let rendered = key().to_cache_key(); + assert_eq!(rendered.len(), "ts-c2-v3-".len() + 64); + for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { + assert!( + !rendered.contains(sensitive), + "key leaked `{sensitive}`: {rendered}" + ); + } + } + + #[test] + fn vary_header_names_are_matched_case_insensitively() { + let mut upper = key(); + upper.vary_values = vec![VaryHeaderValues { + name: "RSC".to_ascii_lowercase(), + values: Some(vec![b"1".to_vec()]), + }]; + assert_eq!( + upper.to_cache_key(), + key().to_cache_key(), + "header names are case-insensitive, so casing must not split the cache" + ); + } + + #[test] + fn vary_values_are_order_sensitive() { + // The origin lists them in a fixed order and the caller preserves it, so a + // differing order means differing inputs rather than the same request. + let mut a = key(); + a.vary_values = vec![ + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + ]; + let mut b = key(); + b.vary_values = vec![ + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + ]; + assert_ne!(a.to_cache_key(), b.to_cache_key()); + } + + #[test] + fn surrogate_keys_carry_a_global_and_a_per_url_lever() { + let keys = key().surrogate_keys(); + assert!( + keys.contains(&"ts-template".to_string()), + "a global purge lever is what makes rollback possible" + ); + assert_eq!(keys.len(), 2, "global plus per-URL"); + assert!( + !keys[1].contains(char::is_whitespace), + "surrogate keys are space-delimited; whitespace would purge more than \ + intended, got {:?}", + keys[1] + ); + assert!( + !keys[1].contains('/') && !keys[1].contains(':'), + "URL punctuation must be reduced, got {:?}", + keys[1] + ); + } + + #[test] + fn punctuation_distinct_urls_have_distinct_surrogate_keys() { + let mut slash = key(); + slash.url = "https://example.com/a/b".to_string(); + let mut colon = key(); + colon.url = "https://example.com/a:b".to_string(); + assert_ne!(slash.surrogate_keys()[1], colon.surrogate_keys()[1]); + } + + #[test] + fn an_absent_vary_header_is_distinct_from_an_empty_one() { + // "absent" and "present but empty" are different requests to the origin, so + // they must not share a template. + let spec = VarySpec::new(["RSC".to_string()]); + let absent_headers = http::HeaderMap::new(); + let absent = spec.values_from(&absent_headers); + let mut empty_headers = http::HeaderMap::new(); + empty_headers.insert("rsc", http::HeaderValue::from_static("")); + let empty = spec.values_from(&empty_headers); + assert_ne!(absent, empty); + + // The distinction that does matter: a present value differs from both. + let mut present_headers = http::HeaderMap::new(); + present_headers.insert("rsc", http::HeaderValue::from_static("1")); + let present = spec.values_from(&present_headers); + assert_ne!(present, absent); + } + + #[test] + fn repeated_and_non_utf8_vary_values_are_preserved() { + let spec = VarySpec::new(["x-route".to_string()]); + let mut headers = http::HeaderMap::new(); + headers.append("x-route", http::HeaderValue::from_static("first")); + headers.append( + "x-route", + http::HeaderValue::from_bytes(b"\xffsecond").expect("obs-text is valid field data"), + ); + assert_eq!( + spec.values_from(&headers), + vec![VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"first".to_vec(), b"\xffsecond".to_vec()]), + }] + ); + } + + #[test] + fn vary_spec_lowercases_configured_names() { + assert_eq!( + VarySpec::new(["RSC".to_string(), "Accept-Encoding".to_string()]).names(), + ["rsc"] + ); + } + + #[test] + fn vary_spec_rejects_invalid_names_and_deduplicates_case_insensitively() { + assert_eq!( + VarySpec::try_new(["not a header".to_string()]), + Err("not a header".to_string()) + ); + assert_eq!( + VarySpec::try_new(["RSC".to_string(), "rsc".to_string()]) + .expect("valid names") + .names(), + ["rsc"] + ); + } + + #[test] + fn drift_is_detected_when_the_origin_varies_on_something_unconfigured() { + // The failure mode configured-Vary has: the origin adds a header to its Vary, + // nobody updates config, and requests differing only in that header start + // sharing a template. + let spec = VarySpec::new(["rsc".to_string()]); + + assert!( + spec.uncovered_by(["rsc"]).is_empty(), + "a fully covered Vary is not drift" + ); + assert_eq!( + spec.uncovered_by(["rsc, next-router-prefetch, Accept-Encoding"]), + vec!["next-router-prefetch"], + "uncovered names must be reported so the stale config is identifiable; \ + accept-encoding is excluded because the key covers it structurally" + ); + } + + #[test] + fn a_key_field_counts_as_coverage_without_being_configured() { + // The failure this prevents is silent and total: every compressing origin sends + // `Vary: Accept-Encoding`, so treating it as a gap means the cache never stores + // anything, and a spike measuring hit rate would report ~0 and look like a + // finding rather than a bug. + let spec = VarySpec::new([]); + + assert!( + spec.uncovered_by(["Accept-Encoding"]).is_empty(), + "the shared path uses one upstream encoding offer and stores identity bytes" + ); + assert_eq!( + spec.uncovered_by(["accept-encoding, rsc"]), + vec!["rsc"], + "only the genuinely uncovered name should be reported" + ); + } + + #[test] + fn a_wildcard_vary_is_not_reported_as_a_named_gap() { + // `Vary: *` means uncacheable, which the eligibility gate handles. Reporting + // it here would produce a nonsense "configure a header called *". + let spec = VarySpec::new(["rsc".to_string()]); + assert!(spec.uncovered_by(["*"]).is_empty()); + } + + #[test] + fn metadata_round_trips() { + let metadata = TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: vec![ + ( + "content-security-policy".to_string(), + "default-src 'self'".to_string(), + ), + ( + "content-security-policy".to_string(), + "script-src 'self'".to_string(), + ), + ( + "link".to_string(), + "; rel=preload; as=script".to_string(), + ), + ], + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 42, + }; + let decoded = + TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); + assert_eq!(decoded, metadata); + } + + #[test] + fn unparseable_metadata_is_a_miss_not_a_panic() { + for raw in [ + &b"not-key-value"[..], + &b"v=notanumber\nce=gzip\nct=text/html\nlen=1"[..], + &b"v=1\nce=gzip\nct=text/html"[..], + &b"v=1\nce=gzip\nct=text/html\nlen=1\nunexpected=1"[..], + &b"v=1\nv=1\nce=identity\nct=text/html\nlen=1"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=cache-control:public"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=not-a-policy:value"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=malformed"[..], + &b"v=1\nce=identity\nct=application/json\nlen=1"[..], + &[0xff, 0xfe][..], + ] { + assert_eq!( + TemplateMetadata::decode(raw), + None, + "malformed metadata must be a miss, not a partial read: {raw:?}" + ); + } + } + + #[test] + fn the_policy_allowlist_covers_document_security_and_delivery_headers() { + for required in [ + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + ] { + assert!( + REPLAYABLE_POLICY_HEADERS.contains(&required), + "warm ESI hits must preserve {required}" + ); + } + } + + #[tokio::test] + async fn the_null_object_reports_unsupported_rather_than_failing() { + // Degrading to per-request transformation keeps the shared modes portable on + // adapters with no cache; erroring would make them Fastly-only outright. + let cache = UnavailableTemplateCache; + assert_eq!( + cache.get(&key()).await.err(), + Some(TemplateCacheMiss::Unsupported) + ); + assert!(matches!( + cache + .put( + &key(), + &TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: Vec::new(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + Vec::new(), + std::time::Duration::from_secs(1) + ) + .await, + Err(TemplateCacheError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a39a26430..e9e02e524 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -168,6 +168,16 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// Shared transformed-template cache (C2). Defaults to + /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming + /// per request rather than failing. Spike-only; see + /// [`crate::platform::template_cache`]. + pub(crate) template_cache: Arc, + /// Platform-specific cold-response template assembler. + /// + /// Defaults to [`super::UnavailableTemplateAssembler`]. Core retains a portable + /// byte-seam fallback when this service is unavailable or rejects a document. + pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -223,6 +233,18 @@ impl RuntimeServices { &*self.kv_store } + /// The shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { + &*self.template_cache + } + + /// Returns the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { + &*self.template_assembler + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -272,6 +294,29 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template cache replaced. + /// + /// Spike-only (#1009). + #[must_use] + pub fn with_template_cache(self, cache: Arc) -> Self { + Self { + template_cache: cache, + ..self + } + } + + /// Returns a clone of this instance with the template assembler replaced. + #[must_use] + pub fn with_template_assembler( + self, + assembler: Arc, + ) -> Self { + Self { + template_assembler: assembler, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -290,6 +335,8 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + template_cache: Option>, + template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -303,6 +350,8 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + template_cache: None, + template_assembler: None, backend: None, http_client: None, geo: None, @@ -325,6 +374,23 @@ impl RuntimeServicesBuilder { self } + /// Set the shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(mut self, cache: Arc) -> Self { + self.template_cache = Some(cache); + self + } + + /// Set the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler( + mut self, + assembler: Arc, + ) -> Self { + self.template_assembler = Some(assembler); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -387,6 +453,14 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + // Defaulted rather than required: an adapter with no template cache + // should degrade to transforming per request, not fail to build. + template_cache: self + .template_cache + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), + template_assembler: self + .template_assembler + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e2ff3d3cf..65d5dd124 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21,7 +21,7 @@ use std::borrow::Cow; use std::io::Write; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -51,15 +51,19 @@ use crate::auction::types::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; +use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; +use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; -use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{ + GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, +}; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::enforce_synthesized_html_cache_privacy; +use crate::response_privacy::{enforce_private_no_store, enforce_synthesized_html_cache_privacy}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -70,6 +74,68 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; +const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum C2ResponseState { + Hit, + MissReserved, + MissStored, + MissStoreError, + BypassRequest, + BypassResponse, + Unsupported, + Invalid, + BackendError, +} + +impl C2ResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::Hit => "hit", + Self::MissReserved => "miss-reserved", + Self::MissStored => "miss-stored", + Self::MissStoreError => "miss-store-error", + Self::BypassRequest => "bypass-request", + Self::BypassResponse => "bypass-response", + Self::Unsupported => "unsupported", + Self::Invalid => "invalid", + Self::BackendError => "backend-error", + } + } +} + +fn set_c2_response_state(response: &mut Response, state: C2ResponseState) { + response.headers_mut().insert( + HEADER_X_TS_C2_CACHE, + HeaderValue::from_static(state.as_str()), + ); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum AssemblyResponseState { + EsiParser, + ByteSeamFallback, + ByteSeam, +} + +impl AssemblyResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::EsiParser => "esi-parser", + Self::ByteSeamFallback => "byte-seam-fallback", + Self::ByteSeam => "byte-seam", + } + } +} + +fn set_assembly_response_state(response: &mut Response, state: AssemblyResponseState) { + response.headers_mut().insert( + HEADER_X_TS_ASSEMBLY, + HeaderValue::from_static(state.as_str()), + ); +} fn body_as_reader( body: EdgeBody, @@ -201,11 +267,16 @@ fn restrict_accept_encoding(req: &mut Request) { // origin responds without compression. Adding encodings here would cause the // origin to compress its response even though the client never asked for it, // and the client would then receive content it cannot decode. + if !req.headers().contains_key(header::ACCEPT_ENCODING) { + return; + } let Some(current) = req .headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) + .get_all(header::ACCEPT_ENCODING) + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .map(|values| values.join(", ")) else { return; }; @@ -273,6 +344,158 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { matched_qvalue } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderEncodingError { + Malformed, + NoAcceptableEncoding, +} + +fn parse_quality_value(value: &str) -> Option { + let value = value.trim(); + let (whole, fraction) = value + .split_once('.') + .map_or((value, None), |(whole, fraction)| (whole, Some(fraction))); + let fraction_is_valid = fraction.is_none_or(|fraction| { + fraction.len() <= 3 && fraction.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !fraction_is_valid { + return None; + } + match whole { + "0" => value.parse().ok(), + "1" if fraction.is_none_or(|fraction| fraction.bytes().all(|byte| byte == b'0')) => { + Some(1.0) + } + _ => None, + } +} + +fn negotiate_reader_compression( + headers: &edgezero_core::http::HeaderMap, +) -> Result { + if !headers.contains_key(header::ACCEPT_ENCODING) { + return Ok(Compression::None); + } + + let mut qualities = Vec::<(String, f32)>::new(); + for field in headers.get_all(header::ACCEPT_ENCODING) { + let field = field.to_str().map_err(|_| ReaderEncodingError::Malformed)?; + for item in field + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + { + let mut parts = item.split(';'); + let token = parts + .next() + .map(str::trim) + .filter(|token| !token.is_empty()) + .ok_or(ReaderEncodingError::Malformed)? + .to_ascii_lowercase(); + if token != "*" && http::HeaderName::from_bytes(token.as_bytes()).is_err() { + return Err(ReaderEncodingError::Malformed); + } + let mut quality = 1.0; + let mut saw_quality = false; + for parameter in parts { + let (name, value) = parameter + .trim() + .split_once('=') + .ok_or(ReaderEncodingError::Malformed)?; + if !name.trim().eq_ignore_ascii_case("q") || saw_quality { + return Err(ReaderEncodingError::Malformed); + } + quality = parse_quality_value(value).ok_or(ReaderEncodingError::Malformed)?; + saw_quality = true; + } + if qualities.iter().any(|(seen, _)| seen == &token) { + return Err(ReaderEncodingError::Malformed); + } + qualities.push((token, quality)); + } + } + + let explicit = |name: &str| { + qualities + .iter() + .find_map(|(candidate, quality)| (candidate == name).then_some(*quality)) + }; + let wildcard = explicit("*"); + let quality_for = |name: &str| explicit(name).or(wildcard).unwrap_or(0.0); + // Identity is implicitly acceptable at q=1 unless explicitly excluded, or a + // wildcard q=0 excludes every unlisted coding. + let identity_quality = + explicit("identity").unwrap_or_else(|| if wildcard == Some(0.0) { 0.0 } else { 1.0 }); + + let candidates = [ + (Compression::Brotli, quality_for("br")), + (Compression::Gzip, quality_for("gzip")), + (Compression::Deflate, quality_for("deflate")), + (Compression::None, identity_quality), + ]; + let mut selected = None; + for (compression, quality) in candidates { + if quality > 0.0 && selected.is_none_or(|(_, best)| quality > best) { + selected = Some((compression, quality)); + } + } + selected + .map(|(compression, _)| compression) + .ok_or(ReaderEncodingError::NoAcceptableEncoding) +} + +fn set_response_compression(response: &mut Response, compression: Compression) { + let encoding = match compression { + Compression::None => None, + Compression::Gzip => Some("gzip"), + Compression::Deflate => Some("deflate"), + Compression::Brotli => Some("br"), + }; + if let Some(encoding) = encoding { + response + .headers_mut() + .insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); + } else { + response.headers_mut().remove(header::CONTENT_ENCODING); + } + let varies_on_encoding = response + .headers() + .get_all(header::VARY) + .iter() + .any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }); + if !varies_on_encoding { + response + .headers_mut() + .append(header::VARY, HeaderValue::from_static("Accept-Encoding")); + } + response.headers_mut().remove(header::CONTENT_LENGTH); +} + +fn response_compression(response: &Response) -> Compression { + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(Compression::from_content_encoding) + .unwrap_or(Compression::None) +} + +fn encode_complete_body( + body: Vec, + compression: Compression, +) -> Result, Report> { + let mut encoder = BodyStreamEncoder::new(compression); + let mut encoded = encoder.encode_chunk(body)?; + encoded.extend_from_slice(&encoder.finish()?); + Ok(encoded) +} + /// Unified tsjs static serving: `/static/tsjs=` /// /// Serves two types of bundles: @@ -361,6 +584,8 @@ struct ProcessResponseParams<'a> { suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. + shared_template_authorized: bool, } struct PublisherBodyProcessor { @@ -384,9 +609,10 @@ impl PublisherBodyProcessor { settings, integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(¶ms.ad_bids_state), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), + shared_template_authorized: params.template_cache_key.is_some(), })?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( @@ -428,6 +654,7 @@ fn process_response_streaming( body: EdgeBody, output: &mut W, params: &ProcessResponseParams, + output_compression: Compression, ) -> Result<(), Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = @@ -443,7 +670,7 @@ fn process_response_streaming( let compression = Compression::from_content_encoding(params.content_encoding); let config = PipelineConfig { input_compression: compression, - output_compression: compression, + output_compression, chunk_size: 8192, }; // Bound how much decoded gzip output may sit in the heap at once, using the @@ -465,6 +692,7 @@ fn process_response_streaming( ad_bids_state: params.ad_bids_state.clone(), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), + shared_template_authorized: params.shared_template_authorized, })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) @@ -509,13 +737,21 @@ async fn process_response_streaming_async( params.content_encoding ); - let compression = Compression::from_content_encoding(¶ms.content_encoding); + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + // A C2 template is always identity bytes. Decode during the transform instead of + // recompressing and immediately decoding the entire buffered result afterwards. + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; let mut processor = PublisherBodyProcessor::new(params, settings, integration_registry)?; process_body_chunks_async( body, output, &mut processor, - compression, + input_compression, + output_compression, settings.publisher.max_buffered_body_bytes, ) .await @@ -557,11 +793,12 @@ async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, max_body_bytes: usize, ) -> Result<(), Report> { - let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); - let mut encoder = BodyStreamEncoder::new(compression); + let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); + let mut encoder = BodyStreamEncoder::new(output_compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); while let Some(segments) = @@ -950,6 +1187,137 @@ struct HtmlStreamProcessorParams<'a> { ad_bids_state: Arc>>, suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, + /// Whether a shared template was authorized for this response. + /// + /// Carried rather than re-derived so both seams see the same answer. See + /// [`effective_assembly_mode`]. + shared_template_authorized: bool, +} + +/// The diagnostics decision the template may carry. +/// +/// Diagnostics is request-scoped — activated by a cookie or query parameter, and +/// documented as an immutable per-request decision — so it must not reach a shared +/// template. +/// +/// It does not leak today even without this gate, but only by coincidence: +/// `requires_private_no_store()` is a strict superset of the conditions under which +/// a script is emitted, and that stamp lands before the C2 gate reads response +/// headers, so the gate refuses. Two independent conditions that happen to align, +/// with nothing enforcing the relationship. This makes the guarantee explicit; +/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a +/// backstop if this gate is ever removed. +pub(crate) fn template_gpt_diagnostics( + mode: AssemblyMode, + decision: Option, +) -> Option { + match mode { + AssemblyMode::Inline => decision, + AssemblyMode::Esi => None, + } +} + +/// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the +/// place this reader's slots and bids are spliced into. +/// +/// An inert HTML comment, deliberately. Template schema v1 used an executable ESI include +/// tag here, when the `esi` crate resolved it at the edge. That crate was removed from the +/// render path because it truncates any element larger than its 16 KB chunk size, and +/// nothing has parsed ESI since. What remained was a tag that *looked* executable, would +/// have been executed by any ESI-enabled layer in front of us, and renders as text in a +/// browser if assembly is ever skipped. A comment cannot do any of those things: an +/// unassembled template degrades to a page with no ads rather than a page with a visible +/// tag. +/// +/// Carries no URL. Every byte here is a byte every reader of the shared template +/// receives, so nothing request-scoped may appear, and keeping a URL out also removes +/// any escaping question at the seam. +pub const AD_ASSEMBLY_SEAM: &str = ""; + +/// The mode the operator asked for, before availability is taken into account. +/// +/// Spelled once, because the mode has to mean the same thing at the cache key, at the +/// seam, and at both hit finalizers. Every one of those re-derived it from the same +/// `Option` chain, and the finalizers had no way to ask at all — which is why they +/// demanded a seam marker of a mode that emits none. +fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { + settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default() +} + +/// Whether this mode's `` seam emits [`AD_ASSEMBLY_SEAM`]. +/// +/// The property that decides whether a template is *expected* to have a hole in it, and +/// therefore whether the absence of one is a defect or the design. Only `Esi` splices per +/// reader. +/// +/// Matched exhaustively rather than compared against `Esi`, so a new mode has to state +/// its answer here instead of silently inheriting one. +fn mode_emits_seam_marker(mode: AssemblyMode) -> bool { + match mode { + AssemblyMode::Inline => false, + AssemblyMode::Esi => true, + } +} + +/// The assembly mode this response will actually be delivered under. +/// +/// The configured mode says what the operator wants; the cache key says whether it is +/// available. A shared mode with no key means the gate refused this response — the +/// origin set a cookie, declared a `Vary` the key does not cover, returned a non-200, +/// and so on — so there is no shared template to build and nothing downstream will +/// assemble one. +/// +/// When that happens the request falls back to [`AssemblyMode::Inline`] **entirely**, +/// at every seam. Falling back at one seam and not another is what produced the failure +/// this function exists to prevent: the `` seam emitted a legacy ESI tag because +/// the mode was `Esi`, while assembly was skipped because there was no key, so the reader +/// received a document with unresolved executable ESI markup in it and no bids at all. +/// +/// Bypassing is the *normal* case against a real origin, not an edge case, so this path +/// runs far more often than the shared one. +fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool) -> AssemblyMode { + let configured = configured_assembly_mode(settings); + if matches!(configured, AssemblyMode::Inline) || shared_template_authorized { + return configured; + } + log::debug!( + "assembly mode {configured:?} is unavailable for this response (no shared template \ + was authorized); falling back to inline" + ); + AssemblyMode::Inline +} + +/// What the `` seam should inject, given the assembly mode. +/// +/// Explicit rather than inferred. The previous shape read +/// `ad_slots_script.is_some()` inside the element handler, which silently coupled +/// two independent decisions: once [`template_ad_slots_script`] stopped emitting a +/// head script under a shared mode, body-close injection stopped with it. +/// +/// `Esi` emits [`AD_ASSEMBLY_SEAM`], an inert HTML comment marking where this reader's +/// slots and bids are spliced in. Assembly is a byte split on that comment, performed by +/// this crate on both the miss and the hit path; no ESI layer is involved. +pub(crate) fn body_close_injection( + mode: AssemblyMode, + head_script_present: bool, +) -> BodyCloseInjection { + match mode { + // Per-navigation and never shared, so gating on slot presence is correct. + AssemblyMode::Inline => { + if head_script_present { + BodyCloseInjection::InlineBids + } else { + BodyCloseInjection::None + } + } + // Constant across every request that reaches the transform — which is what + // makes it safe in a shared template. + AssemblyMode::Esi => BodyCloseInjection::Marker(AD_ASSEMBLY_SEAM.to_string()), + } } fn create_html_stream_processor( @@ -963,10 +1331,18 @@ fn create_html_stream_processor( params.origin_host, params.request_host, params.request_scheme, - ) - .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + ); + + let assembly_mode = effective_assembly_mode(params.settings, params.shared_template_authorized); + let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); + + let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); + + let config = config + .with_ad_state(params.ad_slots_script, params.ad_bids_state) + .with_gpt_diagnostics(gpt_diagnostics) + .with_body_close(body_close) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1000,6 +1376,27 @@ pub enum PublisherResponse { /// Parameters for [`process_response_streaming`]. params: Box, }, + /// A shared template read from C2, to be assembled on the way out. + /// + /// Distinct from [`Self::Stream`] because the bytes are **already transformed** — + /// running them through `lol_html` again would inject a second tsjs `", + html_escape_for_script(slots_json), + html_escape_for_script(&bids) + ) +} + +/// The slot definitions a shared-mode seam must carry, as JSON. +/// +/// Mirrors [`template_ad_slots_script`]'s gating: same `should_run_ad_stack` condition, +/// same slot set. The difference is only *where* it is delivered — the seam, per +/// request, rather than the head, into a shared template. +pub(crate) fn seam_ad_slots_json( + mode: AssemblyMode, + should_run_ad_stack: bool, + settings: &Settings, + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + request_path: &str, +) -> Option { + if matches!(mode, AssemblyMode::Inline) || !should_run_ad_stack { + return None; + } + let co_config = settings.creative_opportunities.as_ref()?; + let section = co_config.section_for_path(request_path); + let slots: Vec = matched_slots + .iter() + .filter_map(|slot| build_slot_json(slot, co_config, §ion)) + .collect(); + Some( + serde_json::to_string(&slots) + .expect("serde_json::to_string of Vec should be infallible"), + ) +} + /// Build the empty-bids `origin" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ( + "content-security-policy", + "default-src 'self'; script-src 'nonce-reader-nonce'", + ), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "a response-bound CSP nonce and its HTML must never be reused from C2" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); + } + + #[tokio::test] + async fn a_post_is_never_answered_from_a_cached_get() { + // `handle_publisher_request` is the `*`-method fallback route, so a publisher + // path that renders a page on GET and accepts a form or webhook on POST reaches + // here for both. Serving the cached GET to the POST swallows the mutating + // request entirely: the origin never sees it, the caller gets 200 and a page, + // and nothing anywhere reports a problem. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + // Warm the cache with a GET. + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!(stub.recorded_request_uris().len(), 1); + + let post = HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::from("field=value")) + .expect("should build post request"); + let _ = run(&settings, &services, post).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "the POST must reach the origin rather than being answered from the \ + cached GET" + ); + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "and it must not store a template of its own" + ); + } + + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + authenticated, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" + ); + } + } + + mod c2_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), + ); + } + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) + } + + #[test] + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + // Deliberately not `Accept-Encoding`: the shared path normalizes supported + // content codings to one identity template, so that header is covered + // whatever the operator configured. Using it here would test the + // structural-coverage carve-out rather than the drift guard. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("rsc")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "rsc".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" + ); + } + + #[test] + fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { + // The backstop that makes `origin_is_cookie_independent` safe to offer. The + // operator asserts their origin ignores cookies; if the origin then says + // otherwise, the assertion loses. Without this, a wrong assertion would + // silently cross-serve personalized HTML. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("Cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + // The operator's assertion has already been applied here: this is + // `false` precisely because they declared independence. + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["cookie".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "the origin's declaration must override both cookie independence and an \ + accidentally configured per-cookie key" + ); + } + + #[test] + fn a_private_directive_on_a_second_cache_control_line_is_refused() { + // `HeaderMap::get` returns the first value only. An origin that sends + // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a + // separate line means exactly what one comma-joined line would mean, but the + // second line was invisible — so a response the origin marked private was + // written to a cache shared between readers. The `Vary` reads a few lines up + // already use `get_all` for the same reason. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=300"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "a directive on any Cache-Control line must disqualify the response" + ); + } + + #[test] + fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { + // Same defect, the other directive that matters — `no-store` is the one an + // origin uses for a response that must not be written down anywhere. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable) + ); + } + + #[test] + fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { + for name in crate::response_privacy::CDN_CACHE_HEADERS { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("no-store"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "C2 must fail closed on the CDN-specific policy header {name}" + ); + } + } + + #[test] + fn unsupported_vendor_freshness_does_not_authorize_c2() { + for name in crate::response_privacy::CDN_CACHE_HEADERS + .iter() + .filter(|name| **name != "surrogate-control") + { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "the Fastly exception must not authorize the vendor policy {name}" + ); + } + } + + #[test] + fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly's edge freshness should take precedence over the shorter browser \ + lifetime, while the configured safety ceiling remains authoritative" + ); + } + + #[test] + fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { + for (cache_control, surrogate_control, expected) in [ + ("public, max-age=300", "max-age=30", 30), + ("public, max-age=30", "max-age=300", 300), + ] { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, cache_control), + ( + header::HeaderName::from_static("surrogate-control"), + surrogate_control, + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(expected)) + ); + } + } + + #[test] + fn surrogate_stale_windows_do_not_extend_fresh_reuse() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=300"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=20, stale-while-revalidate=600, stale-if-error=1200", + ), + (header::AGE, "10"), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(10)), + "stale windows are validated metadata, not fresh C2 lifetime" + ); + } + + #[test] + fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { + for (policy, expected) in [ + ("max-age", C2BypassReason::MalformedCachePolicy), + ( + "max-age=30, max-age=60", + C2BypassReason::MalformedCachePolicy, + ), + ("max-age=tomorrow", C2BypassReason::MalformedCachePolicy), + ("max-age=30, public", C2BypassReason::MalformedCachePolicy), + ("stale-if-error=60", C2BypassReason::NoPositiveFreshness), + ("max-age=0", C2BypassReason::NoPositiveFreshness), + ("max-age=30,", C2BypassReason::MalformedCachePolicy), + ] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(policy).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(expected), + "`{policy}` must fail closed" + ); + } + } + + #[test] + fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` must remain authoritative" + ); + } + } + + #[test] + fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let publisher_headers = headers(&[ + ( + header::CACHE_CONTROL, + &format!("public, max-age=60, {directive}"), + ), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + ), + ]); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "standard `{directive}` must refuse C2 even with positive edge freshness" + ); + } + } + + #[test] + fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { + let publisher_headers = headers(&[( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + )]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly edge freshness should not require browser freshness" + ); + } + + #[test] + fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { + // The other direction: reading every value must not turn an ordinary + // multi-line `Cache-Control` into a bypass, or the fix would disable the + // cache instead of tightening it. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + None + ); + } + + #[test] + fn origin_freshness_is_positive_age_adjusted_and_capped() { + let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &fresh_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(60)) + ); + + let aged = headers(&[ + (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), + (header::AGE, "35"), + ]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &aged, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(15)) + ); + + let old_date_without_age = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ]); + let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") + .expect("should parse fixture time"); + assert_eq!( + origin_shared_ttl_at( + &old_date_without_age, + one_minute_later, + Duration::from_secs(60), + ), + Err(C2BypassReason::NoPositiveFreshness), + "an old Date is apparent age even when an upstream omitted Age" + ); + } + + #[test] + fn zero_exhausted_missing_and_malformed_freshness_are_refused() { + for (map, expected) in [ + ( + headers(&[(header::CACHE_CONTROL, "max-age=0")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "public")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=+60")]), + C2BypassReason::MalformedCachePolicy, + ), + ] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(expected) + ); + } + } + + #[test] + fn expires_can_authorize_but_never_extend_an_expired_response() { + let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") + .expect("should parse fixture time"); + let fresh = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), + Ok(Duration::from_secs(30)) + ); + + let expired = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), + Err(C2BypassReason::NoPositiveFreshness) + ); + } + + #[test] + fn request_semantics_bypass_c2_except_for_a_max_age_zero_reload() { + for (name, value) in [ + (header::CACHE_CONTROL, "no-cache"), + (header::CACHE_CONTROL, "max-age=30"), + (header::CACHE_CONTROL, "max-age=\"0"), + (header::CACHE_CONTROL, "min-fresh=10"), + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + (header::PRAGMA, "legacy-extension, no-cache"), + (header::RANGE, "bytes=0-99"), + (header::IF_NONE_MATCH, "\"etag\""), + (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ] { + let map = headers(&[(name.clone(), value)]); + assert!(request_bypasses_c2(&map), "{name}: {value} must bypass"); + } + assert!( + !request_bypasses_c2(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), + "a browser reload may reuse C2 because the assembled response and auction \ + are still rebuilt for this reader" + ); + assert!(!request_bypasses_c2(&headers(&[( + header::CACHE_CONTROL, + "public" + )]))); + } + + #[test] + fn a_wildcard_vary_is_refused() { + // `VarySpec::uncovered_by` filters `*` out, with a comment saying the + // eligibility gate handles it. It did not — nothing rejected the wildcard, so + // a response the origin said no key can select was shareable. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("*")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryWildcard) + ); + } + + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } + + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } + + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "a repeated Vary header must not hide names behind the first value" + ); + } + + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + None, + "ESI shareable HTML 200 should be eligible" + ); + } + + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Inline, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::InlineMode), + "inline has no shared template to write" + ); + } + + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } + + #[test] + fn a_forwarded_request_cookie_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control, + ¬hing_covered(), + ), + Some(C2BypassReason::CookieForwarded), + "cookie-personalized HTML must not become a shared template" + ); + } + + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &with_cookie, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); + } + + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); + } + } + + #[test] + fn a_datadome_block_is_refused_by_the_status_check() { + // DataDome replaces the document with a 403 + // (`integrations/datadome/protection.rs:778`). There is no separate + // marker to detect, and none is needed. + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } + + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + content_type, + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); + } + } + + #[test] + fn unsupported_content_encoding_is_refused_before_representation_headers_change() { + let map = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::CONTENT_ENCODING, "zstd"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::UnsupportedContentEncoding) + ); + + let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); + repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + repeated.append( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &repeated, + ¬hing_covered(), + ), + Some(C2BypassReason::MalformedRepresentationHeaders) + ); + } + + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::FORBIDDEN, + "application/json", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" + ); + } + } + + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. + + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + + pub(super) fn slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { id: "atf".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], + formats: vec![CreativeOpportunityFormat { width: 300, height: 250, + media_type: MediaType::Banner, }], floor_price: None, targeting: Default::default(), - bidders: Default::default(), - }], - publisher: PublisherInfo { - domain: "test-publisher.com".to_string(), - page_url: Some("https://test-publisher.com/article".to_string()), - }, - user: UserInfo { - id: None, - consent: None, - eids: None, - }, - device: None, - site: None, - context: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } } - } - - fn build_request(method: Method, uri: &str) -> HttpRequest { - HttpRequest::builder() - .method(method) - .uri(uri) - .body(EdgeBody::empty()) - .expect("should build test request") - } - - #[test] - fn stream_publisher_body_injects_active_diagnostics_for_materialized_html() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let mut request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/article?ts_console=1") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build activation request"); - let decision = - crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) - .expect("should prepare diagnostics request"); - let mut params = make_stream_params(&settings, ""); - params.content_type = "text/html".to_owned(); - params.gpt_diagnostics = Some(decision); - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from("Example"), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should process materialized HTML"); - - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - assert!( - html.contains("__tsjs_gpt_diagnostics_active"), - "should inject the activation flag" - ); - assert!( - html.contains("tsjs-gpt_diagnostics.min.js"), - "should inject the standalone diagnostics module" - ); - } - - #[test] - fn stream_publisher_body_round_trips_gzip() { - let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.js\"}"; - let compressed = gzip_encode(input); - let params = make_stream_params(&settings, "gzip"); - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from(compressed), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should stream gzip response through rewrite pipeline"); - let decoded = gzip_decode(&output); - let decoded = String::from_utf8(decoded).expect("should decode rewritten gzip payload"); - assert!( - decoded.contains("https://test-publisher.com/path/file.js"), - "should rewrite origin URLs to the request host" - ); - assert!( - !decoded.contains("origin.test-publisher.com"), - "should remove the origin hostname from the rewritten payload" - ); - } + pub(super) fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, + section_segment: None, + slot: vec![slot()], + }); + settings + } - #[test] - fn stream_publisher_body_round_trips_brotli() { - let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.css\"}"; - let compressed = brotli_encode(input); - let params = make_stream_params(&settings, "br"); - let mut output = Vec::new(); + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + let mode = AssemblyMode::Esi; + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); - stream_publisher_body( - EdgeBody::from(compressed), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should stream brotli response through rewrite pipeline"); + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request seam, not the template" + ); + } - let decoded = brotli_decode(&output); - let decoded = String::from_utf8(decoded).expect("should decode rewritten brotli payload"); - assert!( - decoded.contains("https://test-publisher.com/path/file.css"), - "should rewrite origin URLs to the request host" - ); - assert!( - !decoded.contains("origin.test-publisher.com"), - "should remove the origin hostname from the rewritten payload" - ); - } + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; - #[test] - fn request_ec_uses_cookie_not_header() { - let settings = create_test_settings(); - let header_ec = format!("{}.HdrId1", "a".repeat(64)); - let cookie_ec = format!("{}.CkId01", "b".repeat(64)); - let req = Request::builder() - .method(Method::GET) - .uri("https://test.example.com/page") - .header("x-ts-ec", &header_ec) - .header("cookie", format!("ts-ec={cookie_ec}; other=value")) - .body(EdgeBody::empty()) - .expect("should build test request"); + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); + } - let ec_context = EcContext::read_from_request(&settings, &req, &noop_services()) - .expect("should read EC context"); + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); - assert_eq!( - ec_context.ec_value(), - Some(cookie_ec.as_str()), - "should resolve request EC ID from cookie" - ); - assert!( - ec_context.cookie_was_present(), - "should detect cookie was present" - ); - assert_eq!( - ec_context.existing_cookie_ec_id(), - Some(cookie_ec.as_str()), - "should return cookie EC value for revocation" - ); - } + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); - /// Drive `handle_publisher_request` with no creative opportunities — a plain - /// proxy with no server-side auction. Hides the auction/EC wiring so callers - /// read like a simple `(settings, services, req)` proxy. - async fn run_publisher_proxy( - settings: &Settings, - services: &RuntimeServices, - req: Request, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut ec_context = - EcContext::read_from_request(settings, &req, services).expect("should read EC context"); - handle_publisher_request( - settings, - services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - req, - ) - .await - .expect("should proxy publisher request") + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); + } } mod ssat_cache_policy_tests { @@ -5024,6 +11633,7 @@ mod tests { match response { PublisherResponse::Buffered(response) | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, } } @@ -5353,7 +11963,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("unexpected origin 304 should return a buffered response") } }; @@ -5436,7 +12048,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("noneligible origin 304 should remain buffered") } }; @@ -5510,7 +12124,8 @@ mod tests { *response.body_mut() = body; response } - PublisherResponse::Stream { response, .. } => response, + PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } => response, }; assert_eq!(response.status(), StatusCode::OK); @@ -6222,7 +12837,7 @@ mod tests { read_count: Arc::clone(&read_count), body_close_processed_at: Arc::clone(&body_close_processed_at), }; - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let ctx = AuctionCollectCtx { dispatched, telemetry: AuctionTelemetryCarry { @@ -6267,7 +12882,7 @@ mod tests { let settings = create_test_settings(); let services = noop_services(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let mut state = AuctionHoldState::new( DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( test_auction_request(), @@ -6316,6 +12931,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_none(), @@ -6333,6 +12949,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_some(), @@ -6764,6 +13381,59 @@ mod tests { ); } + #[test] + fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { + let headers = |values: &[&str]| { + let mut headers = edgezero_core::http::HeaderMap::new(); + for value in values { + headers.append( + header::ACCEPT_ENCODING, + HeaderValue::from_str(value).expect("should build accept-encoding"), + ); + } + headers + }; + + assert_eq!( + negotiate_reader_compression(&headers(&[])), + Ok(Compression::None) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), + Ok(Compression::Gzip) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip, br"])), + Ok(Compression::Brotli), + "server preference breaks an equal-quality tie" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.5"])), + Ok(Compression::None), + "implicit identity has q=1" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), + Err(ReaderEncodingError::NoAcceptableEncoding) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=invalid"])), + Err(ReaderEncodingError::Malformed) + ); + for malformed in [ + "gzip;q=1e-1", + "gzip;q=0.1234", + "gzip;q=1.001", + "not a coding;q=1", + ] { + assert_eq!( + negotiate_reader_compression(&headers(&[malformed])), + Err(ReaderEncodingError::Malformed), + "{malformed} is not valid Accept-Encoding syntax" + ); + } + } + #[test] fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); @@ -6982,6 +13652,9 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -6989,7 +13662,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7031,6 +13704,9 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7038,7 +13714,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7069,6 +13745,9 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7076,7 +13755,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7185,6 +13864,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7192,7 +13874,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7239,6 +13921,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7246,7 +13931,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7296,6 +13981,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "deflate".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7303,7 +13991,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7353,6 +14041,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7360,7 +14051,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7410,6 +14101,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7417,7 +14111,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7455,6 +14149,9 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7462,7 +14159,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7642,8 +14339,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7707,8 +14407,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7776,6 +14479,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7783,7 +14489,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -7836,6 +14542,9 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7843,7 +14552,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7970,6 +14679,9 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7980,7 +14692,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, @@ -8316,6 +15028,9 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8323,7 +15038,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( AuctionSource::SpaNavigation, "proxy.example.com", @@ -8499,6 +15214,9 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8509,7 +15227,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -8568,8 +15286,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let bids_script = r#""#; - let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8624,6 +15345,9 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8631,7 +15355,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8733,6 +15457,9 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8740,7 +15467,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8791,6 +15518,9 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8798,7 +15528,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8835,8 +15565,9 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, diagnostics_auction_id, html_escape_for_script, write_bids_to_state, + AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, + build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + write_bids_to_state, }; use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; @@ -8862,6 +15593,10 @@ mod tests { auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), } @@ -9147,7 +15882,7 @@ mod tests { ), ); - let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + let state = AdBidsState::default(); write_bids_to_state( &winning_bids, PriceGranularity::Dense, @@ -9158,6 +15893,7 @@ mod tests { Some(&auction_request.id), ); let script = state + .script_cell() .lock() .expect("should lock initial bid state") .clone() @@ -9192,6 +15928,7 @@ mod tests { Some(&auction_request.id), ); let empty_script = state + .script_cell() .lock() .expect("should lock empty initial bid state") .clone() diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 21ba9f20b..0fe7650dd 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -9,7 +9,7 @@ //! cache such as Cloudflare would otherwise serve an operator/origin //! `Cache-Control: public` on a cookie-bearing response as-is. -use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::settings::Settings; @@ -30,21 +30,67 @@ fn strip_cdn_cache_headers(response: &mut Response) { } } -/// Forces synthesized HTML to be private and non-storable. +/// Whether `Cache-Control` already forbids shared caching. /// -/// Use this exact policy whenever Trusted Server changes an origin HTML -/// representation with request-specific content: force `private, no-store`, -/// remove origin validators, and remove all CDN-targeted cache directives. -pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { +/// Extracted because both arms of the cookie-privacy net below need it. +/// +/// `publisher::c2_bypass_reason` deliberately does **not** call this and keeps its own +/// copy: it additionally treats `no-cache` as non-shareable, because "revalidate before +/// reuse" is correct for an HTTP cache and too permissive for a spike-owned one. The +/// duplicate is the stricter of the two, so consolidating them would loosen the shared- +/// template gate rather than tidy it. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub fn is_private_or_no_store(headers: &HeaderMap) -> bool { + headers.get_all(header::CACHE_CONTROL).iter().any(|value| { + value.to_str().is_ok_and(|value| { + value.split(',').any(|directive| { + let name = directive + .split_once('=') + .map_or(directive, |(name, _)| name); + matches!( + name.trim().to_ascii_lowercase().as_str(), + "private" | "no-store" + ) + }) + }) + }) +} + +/// Reassert the terminal privacy invariant for a synthesized per-reader response. +/// +/// Call this after every configurable response mutation. It deliberately overwrites +/// `Cache-Control` and strips validators, expiry metadata, and CDN-specific cache +/// directives so a later integration cannot turn an assembled document into C3. +pub fn enforce_private_no_store(response: &mut Response) { response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store"), ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + ] { + response.headers_mut().remove(name); + } strip_cdn_cache_headers(response); } +/// Forces synthesized HTML to be private and non-storable. +/// +/// Use this exact policy whenever Trusted Server changes an origin HTML +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. +pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { + enforce_private_no_store(response); +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -63,14 +109,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. strip_cdn_cache_headers(response); - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let already_uncacheable = is_private_or_no_store(response.headers()); if !already_uncacheable { response.headers_mut().insert( header::CACHE_CONTROL, @@ -96,12 +135,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let response_is_uncacheable = is_private_or_no_store(response.headers()); for (key, value) in &settings.response_headers { if response_is_uncacheable @@ -296,6 +330,40 @@ mod tests { } } + #[test] + fn terminal_private_stamp_removes_every_cache_and_validator_header() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "public, s-maxage=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") + .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") + .header(header::AGE, "30") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "public, max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_private_no_store(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + "surrogate-control", + "cdn-cache-control", + ] { + assert!( + !response.headers().contains_key(name), + "terminal private stamp must strip {name}" + ); + } + } + #[test] fn applies_operator_headers_on_cookieless_response() { let settings = settings_with_response_headers(&[("x-operator", "value")]); diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..624b4ef92 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -452,8 +452,18 @@ export interface TsjsApi { * Lives in the bundle so the lifecycle is executable under test and shares * [`navGeneration`] with the SPA auction hook; `gpt_bootstrap.js` installs * a minimal fallback for pages where the bundle fails to load. + * + * `initialSlots` exists for the shared-template `` seam, which is the + * only place slot definitions arrive with the bids rather than from the head + * script. Passing them here rather than assigning `tsjs.adSlots` before the + * call puts them behind the same generation guard: an assignment made ahead + * of the guard would clobber a committed SPA navigation's slots with the SSR + * document's, and then be read by that route's `adInit()`. */ - scheduleInitialAdInit?: (initialBids?: Record) => void; + scheduleInitialAdInit?: ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) => void; /** Read-only GPT lifecycle diagnostics API, present only in an activated tab. */ gptDiagnostics?: GptDiagnosticsApi; /** diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ebf7e225e..bfafd6e6a 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -665,7 +665,15 @@ function installInitialLoadDetector(ts: TsjsApi): void { * SSR bootstrap as current. For the same reason the initial bids payload is * passed in and applied here, generation-guarded — assigning it * unconditionally at body end would clobber the live bids a faster SPA - * navigation already applied. When a navigation has committed since — or + * navigation already applied. + * + * `initialSlots` is passed in for exactly the same reason and was missing it. + * Only the shared-template `` seam sends slots — under `inline` they + * come from the head script, which runs before any navigation can commit — and + * that seam assigned `tsjs.adSlots` on the line *before* calling this. The + * guard protected the bids and `adInit()` while the assignment it was meant to + * protect had already happened, so a committed SPA navigation kept its bids and + * lost its slots. When a navigation has committed since — or * commits while the deferred callback is pending — the SSR payload is * dropped and `adInit()` is not run: running anyway would re-run the newer * route's live slots/bids, destroying and redefining that route's TS slots @@ -685,8 +693,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { - ts.scheduleInitialAdInit = function (initialBids?: Record) { + ts.scheduleInitialAdInit = function ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) { if ((ts.navGeneration ?? 0) !== 0) return; + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index d3e1d7099..69cb65bfc 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -159,6 +159,36 @@ describe('gpt_bootstrap.js fallback', () => { expect(adInit).not.toHaveBeenCalled(); }); + it('fallback scheduler guards the SSR slot definitions with the same generation check', () => { + // The shared-template seam hands slots to the scheduler rather than assigning + // them itself, so the fallback has to honour the same guard as the bundle. If it + // applied them unconditionally, a page whose bundle failed to load would take the + // stale SSR slots over a committed navigation's. + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([ssrSlot]); + + ts.adSlots = [liveSlot]; + ts.navGeneration = 1; + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([liveSlot]); + }); + it('fallback adInit defines, targets, and displays a TS slot through the command queue', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 999c60c55..830bac217 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -310,6 +310,69 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('applies the SSR slot definitions on the initial document', async () => { + // Under a shared-template mode the head script emits no `tsjs.adSlots`, so the + // `` seam is the only source of slot definitions. They must arrive, or + // `adInit()` iterates an empty list and the page defines no TS slots at all. + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + + expect(ts.adSlots).toEqual([ssrSlot]); + expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); + }); + + it('drops the SSR slot definitions when a navigation has already committed', async () => { + // The guard covered the bids and the adInit call, but the shared-template seam + // assigned `tsjs.adSlots` on the line *before* calling the scheduler — outside the + // guard entirely. A navigation that committed while the SSR document was still + // streaming therefore kept its own bids and silently lost its slots to the stale + // SSR payload, and the next `adInit()` for that route defined the wrong slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.navGeneration).toBe(1); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [liveSlot]; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ + { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]], + }, + ]); + + expect(ts.adSlots).toEqual([liveSlot]); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).not.toHaveBeenCalled(); + }); + it('cancels queued GPT work when a navigation commits before the command queue drains', async () => { // adInit() only queues its slot work on googletag.cmd, which drains when // GPT itself loads — possibly long after the generation check that diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b763cac24..0b94587c4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1382,6 +1382,122 @@ loader: TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false ``` +### Shared template assembly (`assembly_mode = "esi"`) + +`assembly_mode` controls how initial-page slot and bid state is delivered: + +- `inline` (default) transforms every origin response and injects the current + reader's slots and bids directly. +- `esi` opts into a reader-neutral transformed-template cache on Fastly. The + cache stores identity bytes containing one inert, versioned comment. On an + authorized cold miss, Fastly replaces that comment in a private working copy + with one synthetic ESI include and resolves it from the already-built reader + state using the pinned `stackpop/esi` parser. No HTTP fragment request occurs. + Warm hits use an exact byte split instead, preserving the fast article-prefix + stream while the auction finishes. + +This is deliberately not general publisher-controlled ESI. A transformed origin +document containing any ` **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn off the redundant origin cache bypass that the spec identifies as the +actual TTFB cost, behind an operator flag, and establish the measurement baseline that +later work is compared against. + +> **This plan does not close #1009.** It contains no ESI arm and no client-fill arm, so +> completing it cannot answer whether ESI separates cacheable content from per-user +> state. It is a **supporting optimisation and the experimental control** for +> [the ESI validation spike](./2026-08-10-1009-esi-validation-spike.md), which is where +> #1009 is actually decided. Scoped and framed this way after external review on +> 2026-08-10. + +**Architecture:** Two investigation tasks that produce recorded findings and no code; one +code task that adds a config-gated timing log and makes the cache bypass operator- +controlled; and one config change that flips it, gated on the first investigation. +Nothing here touches the auction, the `` hold, or bid delivery — those are +Stages 1–2 in the spec and are explicitly out of scope. + +**Tech Stack:** Rust 2024 edition, `wasm32-wasip1`, Fastly Compute, `web_time::Instant` +for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +(§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. + +**Before pushing, run both documentation gates:** + +```bash +cd docs && npm run format && npm run build && cd .. +``` + +`npm run build` is not optional — `format` passes on documents with dead links, and that +shipped a broken docs build on this branch once already. + +**Two prettier gotchas, both hit while writing this plan.** CI gate 7 +(`cd docs && npm run format`) runs `prettier --check` over all of `docs/`, so both bite. + +1. **Not idempotent on embedded markdown fences.** The first `--write` reformats the + outer document and the embedded ` ```markdown ` block only settles on a second pass. + If `--check` still warns immediately after a `--write`, run `--write` again before + concluding anything is wrong. +2. **It mangles bare `snake_case` identifiers inside fences**, reading the underscores as + emphasis and rewriting `origin_fetch_ms` to `origin*fetch_ms`. **Always wrap + identifiers in backticks**, including inside fenced blocks and table cells. + +--- + +## Background an implementer needs + +Trusted Server proxies a publisher's origin, rewrites the HTML at the edge to inject ad +slot definitions and a JS bundle, and runs a server-side ad auction. For requests that +are eligible for that ad stack, `publisher.rs` currently does three things to the origin +request and response that together make the page uncacheable: + +1. strips conditional and range headers so the origin must return a full body, +2. sets a **cache bypass** so the Fastly read-through cache is skipped entirely, and +3. strips every cacheability header from the response. + +The spec establishes that (2) is redundant given (1) — by the time the request reaches +the cache it is already unconditional, so a cache HIT returns a full body anyway — and +that (2) is the dominant cost. This plan makes (2) operator-controlled and then turns it +off, after first confirming that is safe. + +**Why it might not be safe:** RSC (React Server Component) requests and ordinary HTML +navigations share the same URL and are distinguished only by request headers. RSC +requests are not classified as navigations, so they already flow through the cache while +HTML navigations bypass it. Removing the bypass puts both under one cache key. If the +origin does not declare `Vary` for those headers, the cache could serve one +representation in response to a request for the other. Task 1 checks this. + +**Terms:** _POP_ = Fastly edge point of presence. _shield_ = a designated POP that +backs other POPs. _read-through cache_ = Fastly's cache on the backend request path. +_bypass / `Pass`_ = skip that cache. + +--- + +## File structure + +| File | Responsibility in this plan | +| ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` | **Create.** Recorded output of Tasks 1–2. Gates Task 5. | +| `crates/trusted-server-core/src/publisher.rs` | **Modify.** Timing log and the bypass flag (Task 3); tests (Task 5). | +| `crates/trusted-server-core/src/settings.rs` | **Modify.** `publisher.bypass_origin_cache` and `debug.publisher_timing` (Task 3). | +| `trusted-server.example.toml` | **Modify.** Document the new key (Task 5). | + +No new modules. No adapter changes: the `bypass_cache` platform capability and its +per-adapter mappings stay in place and keep their tests — the publisher-path call site +becomes operator-controlled rather than unconditional. + +## Task order and dependencies + +Only one edge is real. Do not serialize the rest. + +``` +Task 1 (origin Vary check) ──────┬──> Task 2 (appends to the findings file Task 1 creates) + │ + ├──> Task 5 (flip the flag) +Task 3 (instrumentation + flag) ─┘ +``` + +**Task 1 is externally blocked.** It needs the publisher origin hostname, which lives in +the operator's gitignored `trusted-server.toml`. Arrange access before starting, or the +plan stalls on its first step. + +Task 3 is independent and can start immediately. Task 2 only needs Task 1 far enough to +have created the findings document. Task 5 needs Task 1's verdict **and** Task 3's config +flag to exist. + +--- + +## Task 1: Step A — origin `Vary` check + +**Files:** + +- Create: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` + +This task is an investigation. It writes no code and gates Task 5. + +- [ ] **Step 1: Get the origin URL** + +The publisher origin is operator config, not in the repo. Read it from the deployed +service config or ask the operator. Do **not** hardcode it into any committed file — the +findings document records the _result_, not the hostname. + +```bash +# The key is `publisher.origin_url` in the operator's trusted-server.toml +# (gitignored). Confirm the value before proceeding. +``` + +- [ ] **Step 2: Request the HTML representation and capture `Vary`** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept: text/html' +``` + +Expected: response headers. Record whether a `Vary` header is present and its value. + +- [ ] **Step 3: Request the RSC representation at the same URL** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' \ + -H 'Accept: text/x-component' +``` + +Expected: a different `Content-Type` (`text/x-component`) than Step 2, proving the two +representations share a URL. Record `Vary` again. + +- [ ] **Step 4: Probe the `Next-Router-*` headers** + +Do not skip this. The PASS criterion below names these headers, and an implementer who +tests only HTML and `RSC` can record a PASS that is wrong — which routes to Task 5a, the +one outcome this plan calls dangerous. + +```bash +for H in 'Next-Router-Prefetch: 1' 'Next-Router-State-Tree: %5B%22%22%5D'; do + echo "--- $H" + curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' -H "$H" \ + | grep -iE '^(vary|content-type|content-length|cache-control|set-cookie):' +done +``` + +Compare `Content-Type` and `Content-Length` against the plain `RSC: 1` request from +Step 3. If either differs, the origin varies on that header and `Vary` must name it. + +Capture `Cache-Control` and `Set-Cookie` on every request in this task, not just this +one — see Step 5. + +- [ ] **Step 5: Probe cookie personalization — the bigger hole** + +The representation check above covers RSC-vs-HTML. It does **not** cover the larger +class: TS forwards client cookies to origin unchanged, so any cookie-personalized HTML +(logged-in state, paywall meter, publisher-side A/B assignment) becomes cross-servable +once the cache is on. + +**Do not compare body hashes.** Verified on the live origin: this page regenerates +~170 ad-slot container IDs as fresh 32-hex UUIDs on every request, so three requests give +three different hashes with byte-identical lengths, cookie or not. A hash comparison +reports a false FAIL every time. + +Normalize per-request identifiers, establish the no-cookie baseline drift first, then ask +whether the cookie arm differs by _more_ than that baseline: + +```bash +ORIGIN="https://"; HOSTH="Host: " +norm() { sed -E 's/[0-9a-f]{32}/UUID/g' "$1"; } + +for n in a b; do + curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' > "nc_$n.html" +done +curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' \ + -H 'Cookie: ' > ck.html + +echo "baseline drift: $(diff <(norm nc_a.html) <(norm nc_b.html) | grep -c '^[<>]')" +echo "with cookie: $(diff <(norm nc_a.html) <(norm ck.html) | grep -c '^[<>]')" +diff <(norm nc_a.html) <(norm ck.html) | head -20 +``` + +Send the `Host` override — the origin is a shared vhost and will not return the right +document without it. Read it from `publisher.origin_host_header_override`. + +**Step A has been run once and returned a PROVISIONAL PASS**, which is **not** sufficient +to flip the flag. See [the findings](./2026-08-08-1009-measurement-findings.md) for the +five untested conditions. Complete them and record a `FINAL PASS` before Task 5. + +Three failure shapes, any of which blocks Stage 0 independently of the `Vary` verdict: + +- Bodies differ by cookie **and** `Vary` does not name `Cookie` → cross-serving of + personalized HTML. +- Origin emits `Set-Cookie` alongside a shared-cacheable `Cache-Control` → the cache can + replay one visitor's cookie to the next. TS's privacy net does not help; it downgrades + **TS's** response, after the cache has already stored the origin's. +- The deployment is `Authorization`-gated (as #1009 describes) and authorized responses + are cacheable → same problem, different header. + +- [ ] **Step 6: Request with the experiment header, if the operator uses one** + +Repeat Step 2 with the publisher's experiment header set to two different values. +Record whether the bodies differ and whether `Vary` names that header. + +- [ ] **Step 7: Record the finding** + +Create `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md`: + +```markdown +# #1009 measurement findings + +## Step A — origin `Vary` declaration + +**Date:** · **Checked by:** + +| Representation | `Content-Type` returned | `Content-Length` | `Vary` present? | `Vary` value | +| --------------------- | ----------------------- | ---------------- | --------------- | ------------ | +| HTML navigation | | | | | +| RSC | | | | | +| RSC + `Next-Router-*` | | | | | +| Experiment variant | | | | | + +**Cookie / auth exposure:** bodies differ by cookie? `Vary: Cookie` present? origin +`Set-Cookie` on a shared-cacheable response? `Authorization`-gated responses cacheable? + +**Verdict:** FINAL PASS / PROVISIONAL PASS / FAIL + +`FINAL PASS` = `Vary` names every request header the origin varies on (`RSC`, any +`Next-Router-*` or experiment header whose value changed the body, **and `Cookie` if +bodies differ by cookie**), no `Set-Cookie` rides a shared-cacheable response, **and** all +five conditions in Task 5's gate are recorded — a real authenticated session cookie, Basic +Auth through TS, the experiment variant, representative routes, and cached-hit +slot/render attribution. + +`PROVISIONAL PASS` = the `Vary` and cookie checks hold, but one or more of those five is +untested. **Not a release gate.** A first pass lands here. + +`FAIL` = any `Vary` or `Set-Cookie` criterion is unmet. + +**Consequence:** `FINAL PASS` → Task 5a (flip the flag). `PROVISIONAL PASS` → close the +gaps before Task 5 starts. `FAIL` → Task 5b (cache-key discriminator). See spec §4. + +**A FAIL is also a live production defect, not only a Stage 0 blocker.** RSC fetches are +not navigations, so they never set the bypass and **already transit the read-through +cache today**. If the origin varies undeclared on `Next-Router-*`, TS is cross-serving RSC +variants in production right now. File it immediately rather than deferring with Task 5b. +``` + +- [ ] **Step 8: Commit** + +CI gate 7 runs `prettier --check` across all of `docs/`, so format the findings file +before staging it — a filled-in markdown table will not be prettier-clean by hand. + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record origin Vary findings for #1009 Stage 0 gate" +``` + +--- + +## Task 2: Step B — what consumes TS's own response headers + +**Files:** + +- Modify: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` — **created by + Task 1 Step 6.** If Task 1 has not reached that step, create the file with just its + `# #1009 measurement findings` heading rather than blocking. + +Investigation. Determines whether the spec's Stage 3b has a consumer. Does not gate +Task 5, but it appends to Task 1's findings document — do not run the two concurrently +against that file. + +- [ ] **Step 1: Pick a path that already emits shared-cache headers** + +`serve_static_with_etag` emits `public, max-age=300, s-maxage=300` plus +`Surrogate-Control` — see `crates/trusted-server-core/src/http_util.rs:294-311`. It backs +the `/static/tsjs=` bundle route (`publisher.rs:303`, `:322`). Use that URL against +the deployed service. + +- [ ] **Step 2: Request it twice and inspect for cache markers** + +```bash +URL="https:///static/tsjs=" +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +sleep 2 +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +``` + +Expected on the second request: if a cache sits in front of the Compute service, an `age` +greater than zero or an `x-cache` containing `HIT`. + +**The probe above is weak evidence** — absence of `age` is equally consistent with "no +cache" and "cold cache". **The topology check below is the actual answer; run it first and +skip the probe if it is conclusive.** + +```bash +fastly service list +fastly service-version list --service-id +# Look for a Delivery service fronting the Compute service, and for shielding +# configured on the service rather than only on the origin backend. +``` + +A Compute service with no Delivery service in front and no fronting shield does not have +its own output cached — that is the configuration the spec assumes, and this step exists +to confirm or refute it rather than to leave it assumed. + +**While you have the service open, answer a second question that matters more than this +task does:** is the _publisher backend_ shielded on the TS service? + +```bash +fastly backend list --service-id --version active +# Look for a shield on the publisher origin backend. +``` + +#1009's entire off-TS advantage came from a **shield** HIT, not a POP HIT. Whether +Stage 0 recovers a shield HIT or only a single-POP HIT changes the size of the win +materially, and nothing else in this plan establishes it. + +- [ ] **Step 3: Record the finding** + +Append to the findings document: + +```markdown +## Step B — consumers of TS's own response headers + +**Verdict:** SHARED CACHE PRESENT / NO SHARED CACHE + +**Evidence:** + +**Consequence:** NO SHARED CACHE → spec Stage 3b is inert until a topology change; +deprioritize it and ship only Stage 3a (browser caching). SHARED CACHE PRESENT → +Stage 3b gains a consumer AND the per-user `x-geo-*` header leak in spec §7 becomes an +active privacy exposure rather than a theoretical one. Escalate immediately in that case. +``` + +- [ ] **Step 4: Commit** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record response-header cache consumer findings for #1009" +``` + +--- + +## Task 3: Step C — origin fetch timing, and the bypass flag + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` — `publisher.bypass_origin_cache`, + `default_bypass_origin_cache`, `debug.publisher_timing`, the `Publisher` `Default` impl, + eight test literals, and the `origin_host` doctest +- Modify: `crates/trusted-server-core/src/test_support.rs` (log-capture helper) +- Test: inside `mod ssat_cache_policy_tests` at `crates/trusted-server-core/src/publisher.rs:4541` + +**Use `web_time::Instant`, not `std::time::Instant`** — the workspace targets +`wasm32-wasip1` and `web_time` is the wasm-safe clock already used at +`crates/trusted-server-core/src/auction/orchestrator.rs:7`. + +### Two timings, and why these two + +Measure **`hold_wait_ms`** and **`origin_fetch_ms`**. Not the rewrite. + +`hold_wait_ms` is the decision. The hold's cost is literally the duration of one +`.await` — `collect_stream_auction` at `publisher.rs:793`, plus the two EOF variants in +`hold_finish_ready_segments` (`:869`) and `hold_finish_tail_segments` (`:896`). Two +`Instant`s around those calls answer "does the hold block?" directly, instead of +inferring it by comparing origin fetch against auction duration. + +`origin_fetch_ms` is attribution — how much of any win Stage 0 can claim. + +`rewrite_ms` decides nothing. Step C's verdict compares origin fetch against auction +collect, and the ceiling argument in spec §6.4 is structural — it needs no number. +Measuring the rewrite would mean instrumenting two finalizers +(`buffer_publisher_response_async` at `publisher.rs:1114`, and the +`async_stream::try_stream!` block at `publisher.rs:1286`), working around moves out of +`params` inside that block, and finding a correlation key that does not exist — +`OwnedProcessResponseParams` (`publisher.rs:1065-1087`) has no `request_path`, and adding +one means touching all 26 construction sites. + +None of that buys a decision. Skip it. If a rewrite figure is later wanted to set a +target, add it as a separate follow-on once the verdict is known. + +**Why a log line and not `Server-Timing`:** for `origin_fetch_ms` alone a response header +would in fact work — the value is known before headers commit. A log line is still +preferred because it is server-side (no dependence on a browser harness to collect it), +`log` is this project's instrumentation crate per `CLAUDE.md`, and the auction path +already measures itself the same way. The spec previously claimed `Server-Timing` cannot +work at all; that overbroad claim has already been corrected there. + +### Log volume — gate it + +The line sits after the origin send, so it fires for every publisher request that reaches +origin — tagged `ad_stack=false` for ineligible ones, not only for eligible navigations. +That is more useful for comparison and more log spend, and the instrumentation is +temporary either way. Gate it behind the existing debug surface rather than +emitting unconditionally: add a `#[serde(default)] pub publisher_timing: bool` to +`DebugConfig` (`crates/trusted-server-core/src/settings.rs:1872`), following +`ja4_endpoint_enabled` and `auction_html_comment` alongside it. Default `false`; enable +via `ts config push` for the measurement window, then disable. + +This also means the Step 1 test must set that flag in its settings fixture. + +The split is also what makes the Step 1 test achievable — `run_with_slots` +(`publisher.rs:4769`) invokes only `handle_publisher_request` and never drives either +finalizer, so a test asserting on a combined line could never pass. + +**What `origin_fetch_ms` actually measures.** `publisher.rs:2863-2865` sets +`.with_stream_response()` when the adapter supports it, so on Fastly `send()` returns at +response _headers_, not after the body downloads. `origin_fetch_ms` is therefore **origin +TTFB**, not full download time. Name it that way in the findings document. It is still +the correct before/after signal for Stage 0 — the bypass affects whether the request hits +a cache at all — but when comparing against auction `total_time_ms` in Step 9, compare +like with like and say which quantity each column holds. + +- [ ] **Step 1: Write the failing test** + +**Placement matters.** Add the test **inside `mod ssat_cache_policy_tests`** +(`publisher.rs:4541`), not the outer `mod tests` (`:4035`). Every helper it uses is +private to that nested module: `settings_with_enabled_auction_and_creative_opportunities` +(`:4684`), `article_slot` (`:4721`), `conditional_navigation_request` (`:4740`), +`queue_cacheable_html_response` (`:4752`), `run_with_slots` (`:4769`). Placed in the outer +module it will not resolve — and because two _other_ `article_slot` functions exist +(`:9593`, `:10276`) returning a different type, the failure surfaces as a confusing type +error rather than a missing-name error. + +**First, add the log-capture helper.** `crates/trusted-server-core/src/test_support.rs` +has none. Note its shape: the whole file is `#[cfg(test)] pub mod tests { … }`, so the +path is `crate::test_support::tests::capture_logs`, not `crate::test_support::capture_logs` +— see existing consumers at `auth.rs:103` and `config_payload.rs:48`. + +Two constraints the helper must respect or the test fails for unrelated reasons: + +- `log::set_boxed_logger` succeeds **once per process**. Install via a `OnceLock`/`Once` + and have `capture_logs()` return a guard that clears and then reads a shared buffer. +- Call `log::set_max_level(log::LevelFilter::Info)` or higher, or `log::info!` is filtered + out before it reaches the logger. +- **Do not have the guard hold the buffer's own `Mutex`.** The test body runs code that + calls `log::info!` on the same thread, and the logger must lock that same mutex to + append — `std::sync::Mutex` is not reentrant, so this **hangs** rather than failing. + Use two locks: a separate process-wide serialization mutex held by the guard, and the + buffer's own mutex taken and released per line by the logger. +- The buffer is process-global and every other concurrently-running `trusted-server-core` + test logs into it, so a `got: {captured}` diagnostic will be large. Assert with + `contains`, not equality. +- `log::set_max_level` is global for the test binary. Setting it to `Info` is fine, but it + affects every test in the process. + +```rust +#[tokio::test] +async fn eligible_navigation_logs_origin_fetch_duration() { + // Arrange + let logs = crate::test_support::tests::capture_logs(); + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + // The log line is gated; without this the assertions below can never pass. + settings.debug.publisher_timing = true; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let _ = run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + + // Assert + let captured = logs.contents(); + assert!( + captured.contains("publisher_timing"), + "eligible navigation should emit a publisher_timing log line, got: {captured}" + ); + assert!( + captured.contains("origin_fetch_ms="), + "publisher_timing should record origin_fetch_ms, got: {captured}" + ); +} +``` + +This test deliberately asserts only on the `publisher_timing` line. `run_with_slots` never +drives a finalizer, so `publisher_rewrite` is out of its reach — cover that separately if +at all, rather than contorting this test. + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: FAIL — no `publisher_timing` in the captured logs. (Substitute your host +triple; core tests run natively for fast iteration. The Viceroy run comes in Step 6.) + +- [ ] **Step 3: Time the origin fetch** + +In `publisher.rs`, at the top with the other imports, add: + +```rust +use web_time::Instant; +``` + +Then wrap the origin send. The current code is at `publisher.rs:2870`: + +```rust +let mut response = match services.http_client().send(platform_request).await { +``` + +Change it to: + +```rust +let origin_fetch_start = Instant::now(); +let mut response = match services.http_client().send(platform_request).await { +``` + +and immediately after the `match` completes (after the existing `};` that closes it, +before the existing `log::debug!("Publisher origin response received: ...")` at `:2888`): + +```rust +let origin_fetch_ms = u64::try_from(origin_fetch_start.elapsed().as_millis()).unwrap_or(u64::MAX); +``` + +**Make the bypass config-driven in the same change.** This is what lets Stage 0 ship as a +config flip rather than a second deploy — see Task 5. Replace the block at +`publisher.rs:2866-2868`: + +```rust +// Single source of truth for the request and the log line below. Operator- +// controlled so the read-through cache can be re-enabled without a release; +// see docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md §4. +let cache_bypass = should_run_ad_stack && settings.publisher.bypass_origin_cache; +if cache_bypass { + platform_request = platform_request.with_cache_bypass(); +} +``` + +Add the setting to `Publisher` in `crates/trusted-server-core/src/settings.rs:29`, +**defaulting to today's behaviour** so this change is a no-op until deliberately flipped: + +```rust +/// Bypass the platform read-through cache on ad-eligible publisher navigations. +/// +/// `true` preserves the historical behaviour introduced by the SSAT 304-prevention +/// design. `false` lets those navigations use the read-through cache; the +/// conditional-header strip already guarantees a complete body on a cache HIT. +/// Temporary operator control for the Stage 0 rollout — remove once settled. +#[serde(default = "default_bypass_origin_cache")] +pub bypass_origin_cache: bool, +``` + +```rust +fn default_bypass_origin_cache() -> bool { + true +} +``` + +**Adding this field breaks nine sites. Update them in the same commit or Step 2 fails to +compile before it can produce the intended RED failure:** + +- The hand-written `Default` impl at `settings.rs:81-97`. +- Eight exhaustive test literals. The line numbers below anchor each + `let publisher = Publisher {` **opening**, not a field — add the new field inside each + brace: `settings.rs:3553`, `:3564`, `:3575`, `:3586`, `:3597`, `:3608`, `:3621`, + `:3635`. `clippy-fastly` runs `--all-targets`, so these gate lint too. +- The rustdoc example for `origin_host`, whose literal opens at `settings.rs:130`. + **This is a live doctest** and the host-triple test command below does not skip + doctests. + +While there, mirror the existing default-agreement test +`publisher_default_max_buffered_body_bytes_matches_config_default` (`settings.rs:3648`) — +it exists to catch a hand-written `Default` diverging from a serde default, which is +exactly the shape this field re-introduces. One assertion. + +Then emit the line, immediately after computing `origin_fetch_ms`, gated on the debug +flag from the section above: + +```rust +if settings.debug.publisher_timing { + log::info!( + "publisher_timing origin_fetch_ms={origin_fetch_ms} \ + cache_bypass={cache_bypass} ad_stack={should_run_ad_stack}" + ); +} +``` + +- [ ] **Step 4: Instrument `hold_wait_ms` — the decision metric** + +This is the number the whole effort turns on, and it needs **one edit in one function**. + +`collect_stream_auction` (`publisher.rs:2431`) is the only function that awaits the +auction collect, and all three call sites reach it: + +| Call site | Path | +| ------------------- | ------------------------------------------------------------ | +| `publisher.rs:793` | `hold_collect_close_tail` — Fastly lazy stream | +| `publisher.rs:2257` | `body_close_hold_loop`, EOF arm — Axum, Cloudflare, Spin | +| `publisher.rs:2311` | `body_close_hold_loop`, mid-stream arm — same three adapters | + +Instrument the callee, not the callers. It already destructures `settings` out of +`AuctionCollectDeps` (`:2436`), so the debug flag is in scope with no new plumbing, and +one edit covers every adapter. + +Wrap the `collect_dispatched_auction` await at `:2447-2449`: + +```rust + let hold_wait_start = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &collect_ctx) + .await; + if settings.debug.publisher_timing { + let hold_wait_ms = + u64::try_from(hold_wait_start.elapsed().as_millis()).unwrap_or(u64::MAX); + log::info!("publisher_hold hold_wait_ms={hold_wait_ms}"); + } +``` + +`settings` here is `&&Settings` from the destructure — deref as needed; the compiler will +say so. + +**Do not instrument `hold_finish_ready_segments` (`:869`) or `hold_finish_tail_segments` +(`:896`).** Neither awaits the collect. The first returns `close_found` for its caller to +act on; the second delegates to `hold_collect_close_tail` at `:909`. Instrumenting them +would double-count. + +**Do not instrument the auction itself.** `OrchestrationResult::total_time_ms` +(`orchestrator.rs:285`, struct at `:1449`, per-provider at `:365`) already flows to +`auction_events_raw`. `hold_wait_ms` measures something different and more useful: how +long the _response_ waited, which is near zero when the auction finished during transfer +even though `total_time_ms` is large. + +- [ ] **Step 5: Run the test to verify it passes** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: PASS. + +- [ ] **Step 6: Run the full publisher test module under the real target** + +A format-changing edit to this file can break tests far from the one you added, and the +Viceroy runner aborts on the first panic — so run the whole suite, not a filtered subset. + +```bash +cargo test-fastly +``` + +Expected: PASS. `app::tests` DNS `Error` lines in the output are pre-existing noise. + +- [ ] **Step 7: Verify format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +``` + +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/settings.rs \ + crates/trusted-server-core/src/test_support.rs +git commit -m "Add an operator switch for the origin cache bypass and log origin fetch time" +``` + +Staging without `settings.rs` leaves a tree that does not compile. + +- [ ] **Step 9: Deploy and collect** + +Deploy first. Then enable the log — it is gated and off by default: + +```bash +# In the operator's trusted-server.toml, under [debug]: +# publisher_timing = true +ts config push +``` + +**Deploy before pushing, not after.** `Settings`, `Publisher`, and `DebugConfig` all carry +`#[serde(deny_unknown_fields)]`, and `ts config push` validates against the typed schema +(`crates/trusted-server-cli` → `run_config_push_typed::`). So the +`ts` binary must be rebuilt from this commit (`cargo install-cli`), and pushing the new +keys before the new WASM is live would break config load on the deployed build. +`trusted-server.example.toml:121-125` records this same hazard for +`auction.rewrite_creatives`. + +Then capture the **bypass-on baseline only**. Do not try to collect an off arm here — +turning the bypass off _is_ Task 5, which is gated on Task 1's verdict and forbidden on a +FAIL. The off arm is collected in Task 5 Step 8. + +Capture enough navigations to separate the medians with confidence, across both a homepage and an article path, with the bypass +both on and off. Record the N alongside the result. + +Append to the findings document: + +```markdown +## Step C — server-side latency breakdown + +**N per arm:** · **Paths:** · **Date:** + +| Arm | `origin_fetch_ms` = origin TTFB (median) | auction `total_time_ms` (median) | `rewrite_ms` (median) | +| ---------- | ---------------------------------------- | -------------------------------- | --------------------- | +| bypass on | | | | +| bypass off | | | | + +Read the asymmetry carefully. `origin_fetch_ms` is origin **TTFB** — the send returns at +response headers because `.with_stream_response()` is set — whereas `total_time_ms` is +the auction's full duration. The comparison below is still the right one, but it is not +comparing two like quantities. + +**Verdict:** HOLD IS FREE / HOLD IS COSTING + +Read it off `hold_wait_ms` directly — no model, no comparison against auction duration. + +HOLD IS FREE = `hold_wait_ms` median near zero. The auction finishes during body +transfer. Proceed as staged in spec §7: Stage 0 primary, Stage 2 protects its win. + +HOLD IS COSTING = `hold_wait_ms` median materially non-zero. **Staging inverts** — +Stage 2 becomes primary and Stage 0 secondary. The work does not change, only its order. +Spec §6.2 argues for the first outcome but explicitly does not prove it, so treat the +second as a live possibility. +``` + +- [ ] **Step 10: Commit the findings** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record server-side latency breakdown for #1009" +``` + +--- + +> **Task 4 (spec correction) was completed while this plan was being written.** §3's +> mechanism bullet, §4's operator-flag framing, and the `unexpected_origin_304` watch are +> all already in the spec. Nothing to do; the task is removed rather than left as a +> no-op an implementer would stall on. + +--- + +## Task 5: Stage 0 — turn the origin cache bypass off + +**Gate:** do not flip the flag until Task 1 has recorded a **`FINAL PASS`**. There are +three verdicts, not two. + +- **`FINAL PASS`** → Task 5a (config flip). +- **`PROVISIONAL PASS`** → **stop.** Not a release gate. This is the current state. It + means the representation split is declared correctly under the conditions tested, and + that those conditions were too narrow to flip production on. +- **`FAIL`** → Task 5b. Do **not** flip; it can serve an RSC payload to an HTML + navigation. + +**`FINAL PASS` requires all five, each recorded in the findings document:** + +| Condition | Why the provisional run is insufficient | +| -------------------------------------------------------- | ---------------------------------------------------- | +| A real authenticated or state-bearing session cookie | `sessionid=abc123` is synthetic and proves nothing | +| Basic Auth exercised **through TS**, not just the origin | #1009 describes a gated deployment | +| The experiment variant named in #1009 | Absent from the origin's `Vary`; unexplained | +| Representative routes — article, section, search | Only the homepage was probed | +| Cached-hit slot and render attribution | The randomized div IDs are an unverified interaction | + +Any one of these unrecorded means the verdict stays `PROVISIONAL PASS` and Task 5 does +not start. + +Because Task 3 made the bypass config-driven, Stage 0 ships as a **config change on an +already-deployed build** — no second release, and the read path reverts with another +config push rather than a revert. That matters here: the failure mode this gates on is +cache poisoning, where minutes of exposure are worse than a slow rollout. + +**But a config push is not a full rollback.** It stops HTML navigations reading from +cache; it evicts nothing already stored. See Step 4's rollback sequence — flip, then purge +or roll a versioned namespace, then observe past the origin TTL. Until a C1 purge path +exists, the tail is "wait out the origin TTL," and that must be an accepted, recorded +risk before the flip. + +### Task 5a: flip the flag (Task 1 verdict = `FINAL PASS`) + +**Files:** + +- Modify: the operator's `trusted-server.toml` (gitignored) +- Modify: `crates/trusted-server-core/src/publisher.rs` — the test, and later the default +- Modify: `trusted-server.example.toml` — document the key + +- [ ] **Step 1: Add a test covering the flag in both positions** + +The existing test at `publisher.rs:4824` +(`eligible_navigation_bypasses_cache_and_returns_non_storable_html`) asserts `vec![true]` +and must **keep passing** while the default is `true` — it now documents the default +rather than the only behaviour. Leave it, and add a sibling next to it: + +```rust +#[tokio::test] +async fn eligible_navigation_uses_read_through_cache_when_bypass_disabled() { + // Arrange + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + settings.publisher.bypass_origin_cache = false; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = + run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabling bypass_origin_cache should let the navigation use the read-through \ + cache; the conditional-header strip already guarantees a full body on a HIT" + ); + assert_eq!( + recorded_header( + stub.recorded_request_headers().first().expect("should record request"), + header::IF_NONE_MATCH.as_str() + ), + None, + "conditional headers must still be stripped with the bypass disabled" + ); + assert!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("no-store")), + "the synthesized document must stay non-storable regardless of the bypass flag" + ); +} +``` + +Those last two assertions are the point of the test: the flag must change **only** the +cache mode, leaving the conditional-header strip and the response non-storability intact. + +**Leave `publisher.rs:4941` and `:5160` unchanged** — they already assert `vec![false]` +for non-eligible requests and must keep doing so. `Range`/`If-Range` stripping is covered +by `eligible_range_navigation_fetches_complete_html` (`publisher.rs:4883`), unaffected. + +- [ ] **Step 2: Run both tests** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation -- --nocapture +``` + +Expected: both the existing default-behaviour test and the new flag-disabled test PASS. +If Task 3's `bypass_origin_cache` field is not yet in place, the new test will not +compile — land Task 3 first. + +- [ ] **Step 3: Document both keys in the example config** + +Add to `trusted-server.example.toml` under `[debug]` (line 149, alongside +`ja4_endpoint_enabled` and `auction_html_comment`): + +```toml +# Emit a `publisher_timing` log line per publisher origin fetch. Temporary +# instrumentation for the #1009 latency measurement; leave false in production. +publisher_timing = false +``` + +And under `[publisher]`: + +```toml +# Bypass the platform read-through cache on ad-eligible navigations. +# `true` is the historical default. Set `false` to let those navigations use the +# read-through cache — only after confirming the origin declares `Vary` for every +# header it varies on (see the Stage 0 precondition). +bypass_origin_cache = true +``` + +- [ ] **Step 4: Flip it in the operator config and push** + +```bash +# In the operator's trusted-server.toml, under [publisher]: +# bypass_origin_cache = false +ts config push +``` + +Note from prior operational experience in this repo: the environment-variable overlay is +scalar-only **and** only overrides keys that already exist in the TOML. Adding the key to +the operator's file is required; setting only an env var will be silently dropped. + +**Rollback is a config push plus an eviction — not a config push alone.** Pushing `true` +again stops HTML navigations reading from cache, but evicts nothing: objects already +cached, including those RSC and other request classes keep reading, persist until they +expire. The origin's `max-age=60` bounds that, but does not remove it. + +Full rollback: + +1. Push `bypass_origin_cache = true`. +2. Purge — **and note this is C1, not C2.** `InsertBuilder::surrogate_keys` belongs to + the Core Cache API and applies to the transformed-template cache the ESI spike builds. + It has no effect on the HTTP read-through cache that Stage 0 turns on. Purging C1 + requires either surrogate keys the **origin** supplies on its responses, or the HTTP + cache's own request/candidate surrogate-key surface. Confirm which is available before + relying on it. + + **Neither is wired today.** If the flip ships before one exists, the rollback story is + "wait out the origin TTL" — roughly a minute, per the Step A findings. That is + survivable, but it must be an accepted risk recorded before the flip rather than a + discovery during an incident. + +3. Observe past the origin TTL before declaring the incident closed. + +- [ ] **Step 5: Run the full suite across every adapter** + +```bash +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +``` + +Expected: all PASS. If `platform/test_support.rs:797` or `:888` fail, they are testing +the stub's own recording behaviour rather than publisher behaviour — read them before +changing anything. + +- [ ] **Step 6: Format and lint every target** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare \ + && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm +``` + +Expected: all clean. + +- [ ] **Step 7: Commit the code and config-template changes** + +```bash +git add crates/trusted-server-core/src/publisher.rs trusted-server.example.toml +git commit -m "Add an operator switch for the publisher origin cache bypass" +``` + +- [ ] **Step 8: Watch for the failure modes, not just the win** + +After the flip, check three things before declaring success. The first two are regression +signals, not confirmations. + +1. **`unexpected_origin_304` abandonment telemetry.** This reason + (`publisher.rs:2896`, emitted via `emit_abandoned_auction` at `:2360`) exists because + the ad-stack path refuses cached and conditional origin responses. Re-enabling the + cache is precisely what could revive it. **Any non-zero rate is a rollback signal** — + it means a 304 is reaching TS, which the conditional-header strip was supposed to make + impossible. Push `true` and investigate before continuing. +2. **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch is the Task 1 risk having + materialized despite a PASS verdict — roll back immediately, this is cache poisoning. +3. **`origin_fetch_ms` and `cache_bypass=false`** in the `publisher_timing` logs. This is + the win, and it is the _last_ thing to check, not the first. + +- [ ] **Step 9: Record and commit** + +Append the before/after medians and the three checks above to the findings document, +format it, and commit. + +- [ ] **Step 10: Retire the flag (follow-up, not now)** + +Once the flip has held for a sustained period, flip the default to `false` in +`default_bypass_origin_cache`, then remove the setting and the branch entirely. Track it; +a temporary flag left in place becomes permanent configuration surface. + +### Task 5b: cache-key discriminator (Task 1 verdict = FAIL) + +**Do not implement from this plan.** A FAIL means the origin serves multiple +representations at one URL without declaring `Vary`, so removing the bypass requires TS +to add its own cache-key discriminator — a feature, not a deletion, and materially larger +than Stage 0 as scoped here. + +Escalate with the Task 1 findings and write a separate plan. Two things that plan must +address, both from spec §4: + +1. The discriminator must key on the request headers that actually distinguish the + representations (`RSC`, `Next-Router-*`, the experiment header), **not** on the + navigation classification. `is_navigation_request` + (`crates/trusted-server-core/src/http_util.rs:73-98`) falls back to the `Accept` + header when Fetch Metadata is absent, and its own comment warns that `fetch()` can set + `Accept: text/html` — so a fetch-based request can be misclassified as a navigation. +2. Whether the origin should simply be asked to declare `Vary`, which is cheaper than + building the discriminator and fixes the problem for every consumer rather than only + for TS. + +--- + +## Out of scope + +Named so nobody widens this plan mid-flight. All are specified in the spec. + +- **Stages 1–2** — moving bid delivery off the response body and deleting the `` + hold. Spec §7 and §8 put these behind the correctness defects. Spec §5 explains why + starting them casually produces a silent revenue loss. +- **Stages 3a/3b** — response cacheability. 3b is additionally gated on Task 2. +- **Stages 4–5** — purge capability, TS-owned template cache, ESI. +- **Removing the `bypass_cache` platform capability.** Task 5a removes one call site only. + +--- + +## Definition of done + +- [ ] Findings document records verdicts for Steps A, B, and C, each with its date, its + N where applicable, and the consequence spelled out. +- [ ] `publisher_timing` and `publisher_hold` lines are emitted in production and + readable, and `hold_wait_ms` has a recorded median. +- [ ] Task 1 recorded a **`FINAL PASS`** — all five conditions in Task 5's gate closed, + not merely the provisional run. +- [ ] Either Task 5a is shipped, or Task 1 returned FAIL and both a production defect and + a follow-up plan for 5b exist. +- [ ] A purge path or versioned cache-key namespace exists **before** the flip, or the + "wait out the TTL" rollback is explicitly accepted and recorded as a risk. +- [ ] **The win is measured client-side, not from `origin_fetch_ms`.** That figure is + origin TTFB and excludes body download, rewrite, and post-processing — it is + attribution, not the outcome. #1009 already has a working tester-cookie browser A/B + measuring the TTFB the publisher actually complained about; use it for before/after. +- [ ] `unexpected_origin_304` rate is zero and representations are not mixed (Task 5a + Step 8) — both checked **before** the win is claimed. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md new file mode 100644 index 000000000..fa3e90544 --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -0,0 +1,503 @@ +# #1009 measurement findings + +Recorded output of the checks in +[the plan](./2026-08-08-1009-measurement-and-stage-0.md). Results only — the origin +hostname is operator config and is deliberately not reproduced here. + +> **Final implementation note, 2026-08-12.** The opt-in `esi` mode now uses Fastly +> Core Cache plus an exact inert byte seam. The parser and client-fill experiments were +> removed. Real-origin numbers in this file remain evidence about the observed path, not +> a clean before/after benchmark. Current operational semantics are documented in +> [the configuration guide](../../guide/configuration.md). + +## Step A — origin `Vary` declaration and cookie exposure + +**Date:** 2026-08-08 · **Method:** direct `curl` against the publisher origin with the +configured `origin_host_header_override`, homepage path. + +### Representation split + +| Representation | `Content-Type` | `Cache-Control` | `Set-Cookie` | +| ------------------------------------ | ------------------ | --------------- | ------------ | +| HTML navigation | `text/html` | `max-age=60` | none | +| `RSC: 1` | `text/x-component` | `max-age=60` | none | +| `RSC: 1` + `Next-Router-Prefetch: 1` | `text/x-component` | `max-age=60` | none | + +`Vary`, identical on every response: + +``` +vary: rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch, Accept-Encoding +``` + +The origin declares **every** header that distinguishes the representations, including +`next-router-segment-prefetch`, which the plan's probe list did not think to check. The +HTML/RSC split at one URL is real and correctly declared. + +### Cookie personalization + +Hash comparison was useless here and the plan's probe as written would have produced a +false FAIL — see the method note below. After normalizing per-request identifiers: + +| Comparison | Differing lines | +| ------------------------------ | --------------- | +| no-cookie A vs no-cookie B | 2 | +| no-cookie A vs **with cookie** | 2 | + +Both diffs are the same single `generationTimestamp` field in the RSC payload. **The +cookie changes nothing.** Byte lengths were identical across all three responses +(1,432,944). + +Cookie sent: `ts-tester=true; sessionid=abc123; ts-ec=probe`. + +### Verdict: **PROVISIONAL PASS** — not sufficient to gate a production flip + +Downgraded 2026-08-10 after external review. Everything below held under the conditions +tested; the conditions tested are narrower than the gate requires. + +What passed: + +- `Vary` names every request header the origin varies on. ✅ +- Bodies did not differ by the cookie sent, so `Vary: Cookie` was not required **for + that cookie**. ✅ +- No `Set-Cookie` on a shared-cacheable response. ✅ +- Origin returns 200 without credentials at this layer. ✅ + +**What was not tested, and each of these can flip the verdict:** + +| Gap | Why it matters | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `sessionid=abc123` is not a real session | A synthetic value proves nothing about a state-bearing publisher session. An authenticated or paywall-metered session is exactly the case that would personalize. | +| One route (homepage) only | Article, section, and search routes may personalize differently. | +| Experiment variant never exercised | #1009 says the origin varies on one. It is absent from `Vary` — see Residual uncertainty below. | +| Basic Auth through TS untested | #1009 describes a gated deployment. Only the origin was probed directly. | +| Cached-hit slot resolution untested | The randomized div IDs below are an unverified interaction, not a cleared one. | + +**Consequence:** Stage 0 still takes the operator-flag path rather than the cache-key +discriminator, and no live cross-serving defect is indicated. But this is **not** a +release gate. Close the table above before flipping the flag in production. + +## Two findings the checks were not looking for + +### 1. The origin already intends this page to be shared-cached + +`cache-control: max-age=60` with a correct `Vary` and no `Set-Cookie`. The origin has +been cacheable all along; Trusted Server opted out of it. That is the spec's §4 framing +confirmed from the other side, and it strengthens the case that the bypass was +belt-and-braces rather than load-bearing. + +It also bounds the win: a 60-second TTL means Stage 0 buys a cache hit only within that +window. Whether that translates into a meaningful hit rate depends on request volume per +URL, which is not measured here. + +### 2. Ad-slot div IDs are randomized per request — and this interacts with Stage 0 + +The only per-request variance in the document is ~170 lines of ad-slot container IDs, +each a fresh 32-hex UUID: + +``` +ad-in_content-f75fa7fba54a4fc2a2d787f51c1837dd-in_content-0 +ad-in_content-a968b27e3ee2424f8bb1c19560abf2b1-in_content-0 ← same slot, next request +``` + +Under the bypass, Trusted Server sees fresh IDs on every request. **Once the cache is on, +every visitor within a 60-second window receives the same IDs.** + +This is very likely fine — `tsjs.adSlots` is built from configured slot definitions, not +scraped from origin markup, and injection is a prefix match on the configured `div_id`. +But it is an untested interaction between Stage 0 and the slot-matching path, and it was +not in anyone's risk list. **This is a release gate, not a note.** Verify slot matching resolves against a cached +document before flipping the flag, and watch TS-attributed renders across the flip +rather than only `origin_fetch_ms`. + +## Method note — a defect in the plan's Step A probe + +The plan's cookie check compares `shasum` of the response bodies. On this origin that +test always fails, cookie or not, because of the randomized div IDs above. Three requests +produced three different hashes with byte-identical lengths. + +**Correct method:** normalize per-request identifiers before comparing, e.g. +`sed -E 's/[0-9a-f]{32}/UUID/g'`, and diff the normalized bodies rather than hashing +them. Establish the no-cookie baseline drift first, then compare the cookie arm against +that baseline — a cookie arm is only interesting if it differs by _more_ than the +baseline does. Fix the plan before anyone re-runs this. + +## Residual uncertainty + +#1009 states the origin varies on an experiment header as well as `rsc` and +`next-router-*`. **No experiment header appears in the origin's `Vary` list**, and the +RSC payload's `experiments` key did not differ across any of the requests made here. + +Three readings, unresolved: the issue was imprecise; experiments are assigned +client-side; or they key on a cookie value this probe did not supply. The `Vary` +declaration is authoritative for cache correctness and it is thorough enough to name four +Next-specific headers, so this is unlikely to be a cache-safety gap. Worth one question +to whoever wrote that line in #1009 rather than further probing. + +## Rollback caveat, added 2026-08-10 + +The plan described flipping the flag back as a seconds-long rollback. That is +incomplete. Re-enabling the bypass stops **HTML navigations** reading from cache; it +evicts nothing. Objects already cached — including those RSC and other request classes +continue to read — persist until they expire. + +Two mitigations, both real: + +- The origin's `max-age=60` bounds read-through exposure to roughly a minute. +- Purge exists in-process — `fastly::http::purge::purge_surrogate_key`. An earlier claim + that TS had no purge capability was wrong; it has no _wiring_, which is buildable. + +**But note which cache.** `InsertBuilder::surrogate_keys` belongs to the **Core Cache** +API and applies to the transformed-template cache the ESI spike would build (C2). It has +**no effect on the HTTP read-through cache** that Stage 0 turns on (C1). Purging C1 needs +surrogate keys the _origin_ supplies on its responses, or the HTTP cache's own +request/candidate surrogate-key surface. Confirm which is available before relying on it — +an earlier revision of this document conflated the two. + +**C2's purge is locally testable; C1's is not.** Verified 2026-08-10: Viceroy 0.17 +implements `purge_surrogate_key` against the same in-process cache it serves reads from +(`viceroy-lib-0.17.0/src/wiggle_abi/fastly_purge_impl.rs:10-32`), soft purge included. So +the purge-based rollback for the C2 template cache the spike builds can be exercised end +to end without a Fastly service. That does nothing for Stage 0, whose exposure is C1. + +Rollback is therefore: flip the flag, **then** purge C1 by whichever mechanism is actually +available (or roll a versioned key namespace), **then** observe past the origin TTL before +declaring the incident closed. With no C1 purge path wired, the tail is the TTL itself — +roughly a minute here, and a recorded risk rather than a surprise. + +## ESI spike Task 1 — does `esi` 0.7 build on this toolchain? + +**Date:** 2026-08-10 · **Verdict: PASS.** The cheapest falsifier for the ESI question +clears. #1009 is not closed by a toolchain limit. + +| Check | Result | +| ------------------------------------------------------------------------------------------ | -------------------- | +| `cargo add esi@0.7 --package trusted-server-adapter-fastly` | resolved `esi 0.7.1` | +| `cargo check-fastly` (Rust 1.95.0 / `wasm32-wasip1`) | clean | +| `cargo fmt --all -- --check` | clean | +| All six clippy targets (fastly, axum, cloudflare, cloudflare-wasm, spin-native, spin-wasm) | clean | +| `cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests` | clean | + +**Nine new transitive dependencies:** `esi 0.7.1`, `nom 8.0.0`, `rand 0.10.2`, +`rand_core 0.10.1`, `chacha20 0.10.1`, `cpufeatures 0.3.0`, `atoi 2.0.0`, +`html-escape 0.2.15`, `md5 0.8.1`. + +**No existing shared dependency moved.** `regex` stays 1.12.4, `bytes` 1.12.0, `log` +0.4.33. `nom` and `rand` gain new majors that coexist with the existing 7.1.3 / 0.8.6 / +0.9.4 rather than replacing them — the best available outcome, since a forced bump on a +shared dep is what would have made this expensive. + +### A claim in the spike plan was wrong + +Task 1 Step 3 told the implementer to check for a desync between the root `Cargo.lock` and +`crates/trusted-server-integration-tests/Cargo.lock`. **That second lockfile does not +exist.** The integration-tests crate is a workspace member (root `Cargo.toml:10`) and +shares the root lockfile, so the desync hazard cannot arise in that form. The plan has +been corrected. The dual-lockfile constraint was real at some earlier point; it is not the +current layout. + +### Viceroy 0.17 supports the whole Core Cache surface this spike needs + +**Date:** 2026-08-10 · **Verdict: PASS.** Probed directly under +`cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1`, then removed: + +| API | Result | +| ------------------------------------------------------------------------ | ------ | +| `cache::core::insert(key, ttl).execute()` → write → `finish()` | works | +| `cache::core::lookup(key).execute()` → `Found::to_stream()` | works | +| `Transaction::lookup(key).execute()` → `must_insert_or_update()` | works | +| `Transaction::insert(ttl).surrogate_keys([…]).execute_and_stream_back()` | works | +| Second transactional lookup reports a hit, no obligation | works | + +That is the entire API surface the spike's Task 3 Step 4 specifies, including the +transaction and stream-back shapes. + +**Consequence: provisioning a Fastly service is not a prerequisite.** An earlier revision +of the spike plan made it Task 2 and a blocker on everything downstream. Almost all of the +correctness and safety work — the C2 cache logic, the transform, template byte-identity, +ESI assembly, DCA and dispatcher refusal, fragment-failure degradation, header ordering, +and the leakage gates — runs locally. The plan is re-sequenced accordingly. + +**What still needs a real service:** shielding behaviour, POP-level cache tiering, +request collapsing under genuine concurrency (Viceroy is a single instance, so a passing +`Transaction` test proves the API works and not that collapsing is correct under load), +stale revalidation timing, and **every performance number in the decision rule**. Local +timings are meaningless for the decision. + +### Not yet verified + +Compiling and a cache round-trip are not an implementation. Nothing yet exercises the +`lol_html` transform into C2, ESI assembly, or any runtime behaviour on the publisher path, +and the `esi` dependency is added but unused. + +## ESI spike Task 3 — implementation progress + +**Date:** 2026-08-10. All of it behaviour-neutral under the default +`AssemblyMode::Inline`; nothing here changes a shipped code path. + +| Step | State | +| -------------------------------- | ------------------------------------------------------------------- | +| 1 — `AssemblyMode` setting | **Done.** `Option` on `CreativeOpportunitiesConfig`. | +| 2 — head-seam neutrality gate | **Done.** `template_ad_slots_script`, three byte-identity tests. | +| 2b — body-close decoupling | **Done.** `BodyCloseInjection`, `body_close_injection`. | +| 2c — emit the marker under `Esi` | **Not done.** Blocked on the fragment endpoint; see below. | +| 3 — C2 eligibility gate | **Done.** `c2_bypass_reason`, eight tests. Logs only, no cache I/O. | +| 4 — C2 cache read/write | **Not started.** Design choice open; see below. | + +### What is deliberately absent + +**No marker is emitted under `Esi`.** The marker must point at a fragment endpoint +returning an **executable script**. `/_ts/page-bids` returns JSON +(`publisher.rs`, `handle_page_bids`) and ESI splices fragment bytes verbatim, so aiming +at it would put raw JSON where a script belongs. That endpoint does not exist, and a +marker with nothing behind it is worse than no marker. A test pins the current answer so +it changes deliberately. + +**No cache read or write.** `c2_bypass_reason` has a real call site that logs its verdict, +which makes the decision observable during the spike without mutating anything. Task 3 +Step 4 is blocked on choosing between read-through-with-body-transform and explicit +`cache::core` — the plan names that as a decision to make before writing code, and it is +under investigation rather than assumed. + +### A defect this work introduced and then caught + +Gating the head seam on neutrality made `ad_slots_script` `None` under the shared modes. +The body-close element handler read exactly that value to decide whether to inject at all, +so shared modes silently stopped injecting anything at `` — a side effect of a +`` change. Safe, since emitting nothing cannot leak, but wrong in the way the spec +warns about: the gate has to be "did this response carry bids", not "does this page have +slots". + +Found by reading the handler while starting the next step, not by a failing test. Fixed by +replacing the inference with a named decision. The test that now guards it asserts +body-close is identical whether or not the head script is present — a decision that read +the head script would be _accidentally_ correct today, because that script is always +absent under shared modes, and wrong the moment that changes. + +Worth recording because it is the same shape as the bug the whole task exists to prevent: +something that looks correct and quietly does nothing. + +### Coverage and its limits + +Fourteen new tests. `fmt`, all six clippy targets, and all four adapter suites pass, with +1850 core tests under Viceroy. `clippy --all-targets` caught a benchmark construction site +that all four test suites missed — the suites are not the whole gate. + +**The neutrality guarantee is narrower than it looks.** The tests prove `tsjs.adSlots` is +neutral. They say nothing about the other things injected at the same seam — integration +`head_inserts`, the gpt-diagnostics bootstrap, the RSC placeholder rewriter — which the +spec flags as needing an audit and which that audit has not yet covered. Until it does, +treat request-neutrality as asserted for one element rather than established for the +template. + +## Code review of the Task 3 commits — three HIGH findings, all closed + +**Date:** 2026-08-11. An independent review of the four implementation commits found +three HIGH issues. The default `Inline` path was verified unchanged byte-for-byte, so +none was a live regression — but all three were invariants this branch exists to +establish and none was enforced or tested. + +### 1. The auction dispatched under shared modes with nothing to consume it + +`assembly_mode` was computed _after_ the dispatch decision, so flipping to `client_fill` +or `esi` would still have sent real SSP bid requests, held the response for the full +auction budget, and discarded the result — because both injection seams now return +nothing — with no error, no warning and no log. + +Exactly the silent-waste signature §5 of the design doc is about, reached by an +incomplete feature flag rather than by removing the hold. Fixed by hoisting +`assembly_mode` above the dispatch and gating on `root_auction_is_useful`. + +The test derives the invariant rather than asserting per-variant: a root auction is +useful exactly when a seam will consume its result. A new mode cannot make the dispatch +gate and the injection decisions disagree without failing it. + +### 2. The C2 gate ignored the forwarded client `Cookie` + +TS forwards client cookies to origin unchanged — there is no `Cookie` strip on the +publisher path. So a response can be cookie-personalized while carrying no `Set-Cookie` +itself (session established earlier), no `Cache-Control` at all, status 200, HTML — and +every condition in the gate reported it cacheable. + +§4 of the design doc names this. The plan's own Task 3 Step 3 checklist missed it, so +the implementation matching the checklist exactly still had the hole. Now disqualifying +until an origin `Vary` covering `Cookie` is verified. + +### 3. Request-neutrality was asserted for one element, not the seam + +The head seam still injected integration `head_inserts` and the GPT-diagnostics +bootstrap unconditionally. + +Audited both. **`head_inserts` is clean** — all three implementations (datadome, didomi, +gpt) take the context parameter unused, so output depends on configuration, not the +request. **GPT diagnostics is not** — cookie- or query-activated, and documented as an +immutable request-scoped decision. + +It did not leak, but only by coincidence: `requires_private_no_store()` is a strict +superset of the conditions under which either script is emitted, and that stamp lands +before the C2 gate reads response headers, so the gate refused. Two independent +conditions that happened to align, with nothing enforcing the relationship. + +Fixed on both sides — the processor receives no diagnostics decision under shared modes, +**and** a test enumerates every combination of the decision's three fields asserting that +anything which injects also requires the stamp. The gate is the guarantee; the invariant +test is the backstop if the gate is ever removed. + +### What this says about the tests that existed + +All three findings were in code the existing tests covered — and passed. The tests +exercised the pure decision functions with hand-built inputs and never the rendered +``/`` bytes. That is still true: **no test renders a full document through +`create_html_processor` and compares two requests byte-for-byte.** The plan's Task 3 +Step 2 requires exactly that, and it remains the most valuable missing test. + +### Reviewer's gate, adopted + +Do not proceed to Task 3 Step 4 (actual C2 read/write) or expose `AssemblyMode` to any +test or staging traffic until the full-document byte-identity test exists. The three +fixes above close the known holes; that test is what would catch the next one. + +## Task 3 complete — the C2 cache engages end to end + +`2db10639` (store), `2a2e6c6a` (lookup), plus `b688d667`/`577eb85a` for the `Vary` +handling. A second request for the same URL is now served without touching the origin, +byte-identical to what was stored. + +**Three problems only appeared once the code had to run**, none of them visible in the +plan or in review: + +1. **The key needed the origin's `Vary`, but a lookup precedes the fetch.** Resolved with + an operator-stated list plus a post-response drift guard that refuses to store under a + key that missed something. Spike-grade: a two-phase lookup is the correct answer and + doubles the lookups. +2. **The key carried the encoding the _origin_ chose**, which also does not exist at + lookup time — storing under `br`, looking up under `gzip, br`, a cache that never hits. + Now keyed on what was sent to the origin. +3. **Storing needs every transformed byte; streaming does not collect them.** Shared modes + take the buffered finalizer, branching on the store authorization rather than the + assembly mode, so `Inline` cannot reach it. + +Each was a case where the design read as complete and the implementation had a hole in +it. That is the same pattern as the three review findings above, arriving one layer down. + +**Verified by mutation, not just by green tests.** Disabling the lookup fails the hit +test, so the hit is the cache answering rather than the fixture answering twice; dropping +the `Authorization` re-check fails the authenticated test; reading only the first `Vary` +header value, and disabling the drift guard, each fail their own tests. The reviewer's +gate above was satisfied first: the byte-identity tests it demanded exist and were +themselves mutation-checked. + +**Still not deployable.** `ClientFill` and `Esi` render a template with a hole and +nothing filling it — Task 4 and Task 5. A cache that works is necessary, not sufficient. + +## Local end-to-end run — the Esi arm renders + +`viceroy serve` against a stub origin, config pushed into a scratchpad `fastly.toml` so +nothing tracked was modified. Served document: + +```html +

Stub article

+
+

Body copy.

+ +``` + +No executable ESI tag. One origin fetch for two requests. `private, no-store` on the hit. +Cached template 353 bytes against 467 served, so the cache holds the pre-assembly +template. All three fragment formats behave: script, JSON, and `400` on a typo. `Inline` +unaffected — two fetches for two requests, no C2 activity, no markers. + +### The bug only a running server could find + +With the auction **enabled**, C2 never engaged: two origin fetches, marker unresolved. + +TS stamps its own `private, no-store` when `should_run_ad_stack` is true. The C2 gate ran +after that stamp, read it as the origin's declaration, concluded `OriginNotShareable`, and +refused — **on every page that serves ads**, which is every page that matters. + +The more important half is why no test caught it. The fixture left the auction disabled +and passed `slots: &[]`, so `should_run_ad_stack` was false in every test, the stamp never +fired, and the ordering was unobservable. Every C2 assertion had been made against the one +configuration where C2's hardest condition does not apply. + +Demonstrated both ways: with the old fixture, reintroducing the bug passes all seven +tests; with the corrected fixture it fails six. + +### Pattern across this branch + +Five bugs now share one shape — compiled, passed every existing test, and were wrong: + +1. The head-seam gate silently disabled body-close injection (`d9e05973`). +2. The key held the encoding the origin _chose_, so the cache could never hit (`2a2e6c6a`). +3. A C2 hit served with no `Cache-Control` at all (`0adb578e`). +4. A C2 hit dropped its in-flight auction, billing SSPs for nothing (`b3ac59a6`). +5. The gate read TS's own header as the origin's (`4c557347`). + +Three were found by writing the test the plan asked for. One needed a running server. None +were found by review — including my own, twice over on the same gate. + +The stale-cache test is the same failure in miniature: it passed while never reaching +`is_stale()`, and only mutation testing exposed that. A test that passes for the wrong +reason is worse than no test, because it is counted as coverage. + +## Independent review — two blockers, and two reasons the cache would have measured nothing + +An independent reviewer read `main...HEAD` and, importantly, **demonstrated** findings by +running code rather than inferring them. Four things it found that review-by-reading had +not. + +**A POST was answered from a cached GET.** `handle_publisher_request` is the `*`-method +fallback route, so a publisher path that renders on GET and accepts a form or webhook on +POST reaches it for both. The origin never saw the mutating request; the caller got `200` +and a page. Fixed at key construction, since the key governs lookup and store alike. + +**`Vary: Accept-Encoding` disqualified everything.** The key has a dedicated +`accept_encoding` field, so such an origin is already keyed correctly — but the coverage +check consulted only the operator-configured list and reported a gap. Every compressing +origin sends that header, so **C2 would have stored nothing against any real origin**. +This is worse than a plain bug: the spike would have measured a hit rate near zero and +reported it as a result. + +**Cookies excluded essentially every repeat visitor.** Any cookie disqualified in both +directions, and TS sets its own identity cookie. The population that could ever see a warm +hit was roughly first-ever page views and cookie-less clients. The design notes called +this the "first-nav exception"; it is the common case, not the exception. Now opt-in via +`origin_is_cookie_independent`, with the `Vary: Cookie` drift guard overriding a wrong +assertion. + +**`ClientFill` had no end-to-end coverage.** The reviewer reintroduced a diagnostics leak +scoped to that mode and all 1889 tests passed. Investigation showed that specific mutation +is unreachable — `requires_private_no_store()` is a strict superset of the injection +condition and stamps before the gate reads headers — but only by a coincidence between two +independent conditions. Both the coverage gap and the coincidence are now pinned by tests. + +### What the review says about the review process + +The reviewer's confirmed findings all came from **running** something. Its clean bills — +no leak in the template itself, no `Inline` regression — came with positive evidence: +tracing that integration context types carry no per-reader field at all, and separately +proving the leakage test has teeth by breaking the store/assemble order and watching it +fail. + +Two of my own comments were wrong, and it caught both by checking rather than reading: +one claimed three call sites where there are two, the other described a dispatcher +mechanism that stopped existing when assembly moved to `CompletedRequest`. + +Verified afterwards against a running server with the origin advertising +`Vary: Accept-Encoding`: a cookie-bearing repeat visitor now costs one origin fetch across +two requests, and a POST still reaches the origin. + +## Step B — consumers of TS's own response headers + +Not yet run. + +## Step C — hold and origin fetch timings + +Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..5b9a455e7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,967 @@ +# #1009 ESI Validation Spike + +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See +> [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) +``` + +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check no shared dependency was forced to move** + +```bash +git diff --stat Cargo.lock +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" +``` + +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. + +- [ ] **Step 4: Record and commit, or stop** + +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. + +Propagate a **lineage ID plus the experiment arm** through the whole chain: + +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture C1 and C2 status separately** + +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. + +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** + +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. + +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. + +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. + +- [ ] **Step 4: Write and read C2 — with the real API** + +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: + +```rust +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; +``` + +Correct shape, using a transaction so a cold cache under load transforms once: + +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + found.to_stream()? // C2 HIT — skip origin fetch and transform +} else { + unreachable!("a transaction is either obliged to insert or has found an item") +}; +``` + +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` — measured, see the Stage 0 findings): + +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. + +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. + +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own ESI include tag comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own nested ESI include is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); +``` + +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** + +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal +rather than an address. + +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** + +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. + +- [ ] **Step 6: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 7: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own nested ESI include is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md new file mode 100644 index 000000000..a2b388594 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md @@ -0,0 +1,294 @@ +# #1009 ESI Merge and Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement +> this plan task-by-task. This plan is intentionally executed inline because the operator +> explicitly prohibited subagents. + +**Goal:** Merge current `main` and make the opt-in ESI byte-seam/shared-template path correct, +private, cache-semantic, compressed, observable, and operationally reversible. + +**Architecture:** Fastly Core Cache holds identity-encoded reader-neutral templates behind a +transaction acquired before origin work. Every request assembles its own slots and structured bid +map at an exact inert seam, encodes the result for that client, and receives a final immutable +private/no-store policy. + +**Tech Stack:** Rust 1.95, Fastly Compute/Core Cache, `edgezero_core` HTTP types, `lol_html`, +TypeScript/Vitest, Viceroy, shell harness. + +> **Implementation status, 2026-08-12:** Tasks 1–12 are complete on the branch. The Viceroy +> harness passed in both modes after running outside the filesystem sandbox so it could read the +> macOS native-certificate keychain. + +--- + +### Task 1: Merge live main and preserve auction contracts + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: adjacent Rust and Vitest modules + +- [x] Merge `origin/main` with `git merge --no-ff origin/main`. +- [x] Resolve the `AdBidsState`/`write_bids_to_state` conflict by building one structured map with + `auction_id`, storing both map and script, and returning its delivered slot IDs. +- [x] Add/adjust tests proving ESI and inline retain `hb_auction_id`, APS renderer metadata, and + delivered-winner attribution. +- [x] Run the focused Rust and GPT tests. +- [x] Complete the merge commit. + +### Task 2: Remove mechanisms outside the approved ESI byte-seam design + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Delete: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Delete: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` + +- [x] Update mode tests to specify only `inline` and `esi`; watch the old client-fill expectations + fail or stop compiling. +- [x] Remove `ClientFill`, executable fragment serialization, assembler traits/registration, and + the `esi` crate. +- [x] Update comments to call the production path byte-seam assembly. +- [x] Run focused configuration, publisher, and Fastly adapter tests. +- [x] Commit the scope cleanup. + +### Task 3: Canonicalize and bound the template key + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing tests for absent versus empty `Vary`, repeated raw values, invalid configured + names, punctuation-colliding purge URLs, changed origin host override, and changed creative + configuration. +- [x] Replace string pairs with a typed canonical `Vary` value preserving presence and all bytes. +- [x] Hash a length-prefixed canonical key and hash the URL-specific surrogate key. +- [x] Include publisher origin identity and the complete template-shaping fingerprint. +- [x] Run focused key/configuration tests and commit. + +### Task 4: Enforce request and origin cache semantics + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/Cargo.toml` if HTTP-date parsing needs a direct dependency + +- [x] Write failing tests for response `max-age=0`, positive max age, repeated/malformed cache + directives, `Age` exhaustion, expired/malformed `Expires`, missing freshness, invalid `Vary`, + and request no-cache/no-store/range/conditional bypasses. +- [x] Add a typed cache eligibility result carrying the positive remaining TTL. +- [x] Parse relevant response directives fail-closed and cap, never extend, origin freshness. +- [x] Add request-side bypass classification before lookup. +- [x] Make unsupported/backend-failed cache lookups fall back to inline processing on non-Fastly + adapters rather than buffering a cacheless ESI path. +- [x] Run focused eligibility tests and commit. + +### Task 5: Move request collapse before the origin fetch + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` + +- [x] Verify the reservation is acquired before origin work and the Fastly transaction contract + blocks same-key waiters. Viceroy is single-threaded, so it cannot directly reproduce two + truly concurrent cold requests. +- [x] Introduce a lookup outcome with an opaque insert reservation and explicit cancellation. +- [x] Implement Fastly `Transaction::lookup` before origin work and consume/cancel its obligation + on every exit path. +- [x] Ensure invalid fresh entries become replaceable rather than causing repeated refetches. +- [x] Run focused Core Cache/Viceroy tests and commit. + +### Task 6: Make privacy and policy-header parity final + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` tests + +- [x] Write failing tests for repeated CSP/CSP-Report-Only, omitted COOP/COEP/CORP/HSTS/Link, + unknown cached header metadata, duplicate required metadata fields, and a late integration + changing `Cache-Control` to public. +- [x] Capture all ordered values, expand the safe allowlist, and decode metadata strictly. +- [x] Replay with `append`, then apply the assembled-response privacy policy last. +- [x] Preserve and reassert private/no-store after request-filter effects in Fastly's final send. +- [x] Run focused header/privacy tests and commit. + +### Task 7: Bypass shared templates for request-private diagnostics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write a failing warm-cache test activated by diagnostics query and another by diagnostics + cookie. +- [x] Make `requires_private_no_store()` a lookup/store disqualifier. +- [x] Verify ordinary diagnostics-disabled requests still hit C2. +- [x] Run focused diagnostics/C2 tests and commit. + +### Task 8: Re-encode assembled responses for the reader + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing cold/warm tests requiring gzip/br clients to receive a matching encoded body + and proving reader encoding no longer partitions the stored template. +- [x] Keep the origin offer within the reader's supported codings so a response-gate bypass remains + lossless, while decoding every stored template to identity. +- [x] Carry the selected response encoding separately from identity template metadata. +- [x] Encode buffered assembly after splicing and stream hit prefix/seam/suffix through one encoder. +- [x] Handle `identity;q=0` without serving an unacceptable representation. +- [x] Emit the correct `Vary: Accept-Encoding` response semantics after final encoding. +- [x] Run focused compression tests and commit. + +### Task 9: Make marker failures safe + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Write failing tests for HTML with no explicit ``, a publisher-authored marker + collision, and a corrupt cached marker. +- [x] Record/validate a schema-bound seam location or use a collision-resistant marker contract. +- [x] Cancel storage and fall back safely when the optimization cannot produce one seam. +- [x] Run focused miss/hit assembly tests and commit. + +### Task 10: Add operational observability and harden the harness + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `scripts/c2-local-test.sh` +- Modify: `.github/workflows/test.yml` + +- [x] Write failing tests for distinct backend-error versus not-found status and C2 response-state + reporting. +- [x] Preserve backend errors and emit bounded C2 status without exposing key material. +- [x] Change the harness to operate on a temporary manifest and fail on missing/non-numeric probe + output or empty response bodies. +- [x] Test both cold and warm integrity and execute the generated scheduler payload contract. +- [x] Add the ESI harness to CI where Viceroy prerequisites are available. +- [x] Run shell syntax/static checks and commit. + +### Task 11: Document configuration, semantics, and rollback + +**Files:** + +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: relevant #1009 findings documents + +- [x] Document that `esi` means Fastly C2 plus byte-seam assembly, not parser execution or final + HTTP shared caching. +- [x] Document `template_cache_vary`, cookie independence, freshness, metrics, purge, rollback + ordering, and limitations on non-Fastly adapters. +- [x] Close or supersede stale spike checkboxes and remove claims contradicted by the final code. +- [x] Run docs format/build and commit. + +### Task 12: Full verification + +**Files:** none expected beyond fixes discovered by verification + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run all four adapter test aliases and the parity suite. +- [x] Run all six clippy aliases. +- [x] Build the Fastly release WASM. +- [x] Run JS tests, build, and format under pinned Node 24.12.0. +- [x] Run docs format/build. +- [x] Run `scripts/c2-local-test.sh esi` and `inline` if the environment exposes the required + local certificate store; otherwise report the exact environment blocker. +- [x] Run `git diff --check`, inspect the merge graph, and confirm the worktree contains only + intended changes. + +### Task 13: Interpret Fastly Surrogate-Control conservatively + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing gate test using `Cache-Control: max-age=60` plus the observed publisher + `Surrogate-Control` policy (`max-age=1200`, `stale-while-revalidate=21600`, and + `stale-if-error=604800`). +- [x] Write failing tests proving the shorter standard/surrogate freshness wins, stale windows do + not extend fresh reuse, restrictive directives are refused, and unknown, duplicate, or + malformed directives fail closed. +- [x] Parse only Fastly's supported `max-age`, `stale-while-revalidate`, and `stale-if-error` + directives; continue refusing every other vendor CDN policy field. +- [x] Keep request `Cache-Control: max-age=0` as an intentional C2 bypass so reload preserves its + revalidation semantics. +- [x] Run focused tests, `cargo test-fastly`, target-matched formatting/clippy, both local harness + modes, and verify the observed publisher policy progresses from `miss-stored` to `hit` in + the local Fastly runtime on an ordinary navigation. + +### Task 14: Allow browser reloads to reuse a fresh ESI template + +Task 14 supersedes Task 13's conservative request `max-age=0` bypass after end-to-end testing +proved that C2 reuses only the neutral template and still creates a new private response and +auction. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing end-to-end test proving `Cache-Control: max-age=0` reruns the auction but + does not refetch the reader-neutral publisher template. +- [x] Treat only a valid zero request max age as compatible with C2; continue bypassing positive + or malformed constraints and every explicit revalidation directive. +- [x] Verify the focused tests, formatting, and Fastly clippy, then commit independently. + +### Task 15: Make the ESI template-cache ceiling configurable + +Task 15 supersedes Task 13's shorter-of-standard-and-surrogate rule. The final behavior follows +Fastly edge precedence while retaining restrictive directives as hard refusals. + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Add failing configuration tests for the 60-second default, an explicit 1,200-second ceiling, + zero, values above one day, and omission from serialized rollback-compatible config. +- [x] Add failing freshness tests proving Fastly precedence, age deduction, and the configured + ceiling for the observed `Cache-Control: max-age=60` plus + `Surrogate-Control: max-age=1200` response. +- [x] Implement `template_cache_max_age_seconds` under `[creative_opportunities]` and thread its + resolved duration into C2 eligibility. +- [x] Remove the Fastly adapter's second hard-coded 60-second cap; the already-authorized + per-entry max age becomes the sole insertion lifetime. +- [x] Update the example and operator guide, without editing the tracked deployment + `fastly.toml`. +- [x] Run focused red/green tests, full adapter tests and clippy gates, documentation checks, and + inspect the final diff with `fastly.toml` excluded. diff --git a/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md new file mode 100644 index 000000000..a4c04b9f8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md @@ -0,0 +1,104 @@ +# #1009 ESI Parser Assembly Implementation Plan + +> **Execution note:** Implemented inline in the current checkout, without a worktree or +> subagents, as requested. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Use the repaired ESI parser on authorized cold C2 misses without changing the existing warm-hit streaming behavior. + +**Architecture:** C2 retains the inert schema-v4 seam. Core delegates cold assembly through a platform trait; Fastly converts the seam to one synthetic ESI include and resolves it from the already-collected per-reader script. Parser failure falls back to core's validated byte split, while warm hits continue to stream by byte seam. + +**Tech Stack:** Rust 1.95, `wasm32-wasip1`, Fastly Compute/Viceroy, `stackpop/esi` pinned by Git revision, `error-stack`. + +--- + +### Task 1: Restore a platform assembly boundary + +**Files:** + +- Create: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Test: `crates/trusted-server-core/src/platform/template_assembly.rs` + +- [x] Add a failing object-safety/default-behavior test for `PlatformTemplateAssembler`. +- [x] Run the focused core test and confirm it fails because the boundary is absent. +- [x] Add the trait, error type, unavailable default, runtime service field, builder method, + accessor, and test support. +- [x] Run the focused tests and confirm they pass. + +### Task 2: Delegate only cold-miss assembly + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [x] Add a recording assembler to the C2 end-to-end tests. +- [x] Add a test asserting one platform call on a cold miss and no additional call on the + subsequent warm hit. +- [x] Add a test asserting platform failure returns a complete byte-seam response. +- [x] Add tests for `x-ts-assembly` values on parser, fallback, and warm paths. +- [x] Run each test first and confirm the expected failure. +- [x] Change `assemble_if_shared` to call the platform assembler after storage, fall back + to the validated byte split on error, and return the assembly method. +- [x] Set `x-ts-assembly` without changing `x-ts-c2-cache` or privacy headers. +- [x] Re-run the C2 end-to-end test module. + +### Task 3: Add the repaired Fastly ESI adapter + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` +- Create: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Test: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [x] Add failing adapter tests for a large Next.js script followed by the seam, an + unexpected publisher ESI directive, an unexpected dispatcher URL, and verbatim + fragment content. +- [x] Run the focused Fastly test filter and confirm the missing module/implementation + fails. +- [x] Pin `https://github.com/stackpop/esi.git` at + `4c53feab4d22ad9a84641b4c46f3f63bc6d197e2`. +- [x] Implement the explicit no-cache/no-DCA ESI configuration and synthetic completed + fragment dispatcher. +- [x] Register `FastlyTemplateAssembler` in per-request runtime services. +- [x] Run the focused Fastly tests and confirm they pass. + +### Task 4: Preserve cache schema and documentation truth + +**Files:** + +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: `docs/guide/configuration.md` +- Modify: `scripts/c2-local-test.sh` +- Test: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Add/adjust tests proving schema version 4 and the inert stored marker remain + unchanged. +- [x] Extend the local harness to require `esi-parser` on the miss and `byte-seam` on the + hit. +- [x] Update architecture and operator documentation to describe the hybrid path and + pinned fork accurately. +- [x] Run formatting and the harness's static checks. + +### Task 5: Full verification and signed commit + +**Files:** + +- Review every modified file. + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run every target-matched Clippy alias from `CLAUDE.md`. +- [x] Run `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and + `cargo test-spin`. +- [x] Run the integration parity test. +- [x] Run JS tests/build/format and docs format. +- [x] Run the C2 local harness when Viceroy and its certificate environment are + available; otherwise report that environmental gap explicitly. +- [x] Run `git diff --check`, inspect staged scope, and confirm no operator configuration + or secrets are staged. +- [x] Create one SSH-signed commit only after every required gate is green. +- [x] Verify the commit signature locally and report the exact commit ID and test counts. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md new file mode 100644 index 000000000..031ecc50c --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -0,0 +1,864 @@ +# The Cacheable Root: Latency Diagnosis and Stage 0 Design + +_Filename retains its original `esi-` prefix; the commit history and every +cross-reference point at it. The subject moved, the path did not._ + +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · + +> **HISTORICAL RECORD — NOT THE CURRENT IMPLEMENTATION.** This document preserves the +> measurement and feasibility investigation. Every executable ESI tag, parser, and +> subrequest described below belongs to a rejected spike; do not use those sections to +> infer current runtime behavior. The final branch retains `assembly_mode = "esi"` only as +> the operator spelling for Fastly C2 plus exact byte-seam assembly. See +> [the merge-hardening design](./2026-08-12-1009-esi-merge-hardening-design.md). + +**Revised:** 2026-08-10 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3`. + +> ## ⚠️ Correction, 2026-08-10 — this document's original ESI verdict was wrong +> +> The first revision concluded that ESI was **structurally blocked**: that it +> presupposed a TS-owned template cache which did not exist, and that such a cache was +> in turn blocked on purge capability the platform did not offer. **Both claims are +> false**, and an external review was right to reject them. +> +> Verified against the pinned `fastly` 0.12.1: +> +> - **The cache boundary is native.** `fastly::cache::core` provides +> `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / +> `found()` to read them back, and `Transaction` with `must_insert()` for request +> collapsing. The two-stage design needs no separate KV or template service. +> - **Purge exists in-process.** `fastly::http::purge::purge_surrogate_key` purges from +> inside Compute; the management-API token scope cited in the original is irrelevant to +> it. Note which cache, though: `InsertBuilder::surrogate_keys([...])` is the **Core +> Cache** API and keys the transformed-template cache (C2). It does **not** key the HTTP +> read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied +> keys or the HTTP cache's own surrogate-key surface. +> - **The original pipeline ordering was backwards.** It said "order esi → lol*html, +> never the reverse." `lol_html` \_emits* the ESI include tags, so ESI must run after it. +> Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> +> The error was inspecting what this repository does and reporting it as what the +> platform permits — the same mistake this document criticises #1009 for making in the +> other direction. +> +> **ESI is therefore feasible and unvalidated, not rejected.** Validating it is +> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing +> and are **not** an answer to #1009. + +## Document map — read this first + +#1009 is answered across three documents, not one. This is the only place that says +which owns what. + +| Document | Owns | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | + +**If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, +[§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it +gets validated. Everything else here is Stage 0 and the latency analysis behind it. + +**Decision requested:** approve the four items in §1. + +> **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments +> so that cacheable publisher HTML is separated from per-user ad state, recovering a +> TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router +> publisher running on Fastly Compute. ESI can do this; whether it should is not settled +> here. Separately, the regression has a cheaper cause than the issue assumes. +> +> **This document deliberately carries no performance measurements.** Every conclusion +> below is derived from code at the pinned baseline, so it can be checked by reading the +> repository rather than by trusting a benchmark. Where a quantity is needed and unknown, +> it is named as unknown and [§3](#3-monday-morning) says how to obtain it. +> +> Terms used throughout: **the hold** = TS holding the HTTP response open at `` +> until the server-side auction (SSAT) resolves. **React #418** = the React +> hydration-mismatch error raised when `adInit()` mutates ad-slot subtrees during +> hydration; it is why bid application is deferred to `window.load`. It is a React error +> number, **not** a repository issue — the tracker is +> [#938](https://github.com/IABTechLab/trusted-server/issues/938). **The SSAT price +> defect** = a live mispricing bug named in #1009 (prices reading 100× high) — cited +> from #1009 and prior investigation, not re-verified here. + +--- + +## 1. Decision requested + +| # | Decision | Owner needed | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | + +Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ +doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower +detail than the work it recommends. + +--- + +## 2. Why — the three findings + +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits executable ESI +include tags into a shared template; `fastly::cache::core` stores that template; the +`esi` crate assembles per request on the way out. Everything that requires is +already a dependency. The real open questions are empirical, not architectural: does it +beat a plain client fetch by enough to justify a Fastly-only rendering path, and can +per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment +failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. + +Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it +is a per-platform accelerator rather than the architecture, and its maintenance cost +belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** +— bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP +could embed an ESI include targeting an arbitrary URL and make the edge fetch it. Details in +[Appendix E](#appendix-e--esi-notes-condensed). + +**The auction is already out of band; the hold is ~free.** It is dispatched _before_ +the origin fetch and does not block — dispatched at `publisher.rs:2751-2755`, sent at `:2870` — +with a 500 ms budget. The actual cost is `with_cache_bypass` +(`publisher.rs:2867`), +which forces every ad-eligible navigation to miss the Fastly readthrough cache. + +**The two fixes are multiplicative.** Removing the bypass alone lets the previously +hidden auction surface as the new bottleneck. Removing the hold alone changes nothing, +because the auction was never the bottleneck. **Shipping the hold removal without the +bypass removal will measure no improvement and will read as the effort having failed** — +the most likely way this work gets judged unfairly. + +**Ordering is established; magnitude is not.** The ordering above follows from code and +needs no measurement. The _size_ of the win does — and the one quantity it depends on, +the origin build time under `Pass`, has never been measured. #1009's timings do not +supply it: they compare cached fetches against each other, not against an origin build. +**Quote no figure to a publisher until §3 Step C runs.** Full reasoning in +[§6](#6-the-analysis). + +--- + +## 3. Monday morning + +Three checks, ordered cheapest-first. Each needs a named owner before starting. + +**Step A — origin `Vary` and cookie check (minutes for the first pass).** `curl` the +origin with and without `RSC`, `Next-Router-*`, and the experiment header; inspect `Vary`, +`Cache-Control`, and `Set-Cookie`. **This first pass yields a `PROVISIONAL PASS` only** — +it is not what gates the flip. A `FINAL PASS` additionally requires a real authenticated +session, Basic Auth through TS, the experiment variant, representative routes, and +cached-hit render attribution. Do the cheap pass first because it is the +cheapest thing that unblocks anything. + +**Step B — what consumes TS's own response headers (under a day).** Request a TS-served +path that already emits `public, s-maxage` +(`http_util.rs:294-311`) +twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b +split** — see [§7](#7-deferred-work-specified-not-scheduled). + +**Step C — measure the hold directly (1 day + a measurement window).** + +The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at +`publisher.rs:793`, plus the +two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two +`Instant`s around it yield **`hold_wait_ms`** — the number this entire document is +arguing about, measured rather than modelled. + +Emit two timings per ad-eligible navigation: + +| Metric | Why | +| ----------------- | ---------------------------------------------------------------------- | +| `hold_wait_ms` | **The decision.** How long the response was actually held for bids. | +| `origin_fetch_ms` | Attribution — how much of the win Stage 0 can claim. Origin TTFB only. | + +`hold_wait_ms` replaces the proxy comparison an earlier draft proposed. Comparing `O` +against `A` was an indirect way of asking "does the hold block?"; this asks it directly, +costs less to build, and removes the modelling error corrected in +[§6.2](#62-what-the-hold-actually-costs). + +Deliberately not measured: auction collect duration is already instrumented +(`OrchestrationResult::total_time_ms`, `auction/orchestrator.rs:285`, flowing to +`auction_events_raw`) — read it, don't rebuild it. Rewrite duration decides nothing and +would mean touching two finalizers. + +- **Mechanism: a `log::info!` line behind a debug flag, not `Server-Timing`.** A response + header would in fact work for the origin-fetch figure — that value is known before + headers commit — but a server-side log needs no browser harness to collect it, `log` is + this project's instrumentation crate, and the auction path already measures itself with + `web_time::Instant`. Gate it behind config: one line per eligible navigation is real log + spend and the instrumentation is temporary. +- **Sample: enough navigations per arm to separate the medians with confidence**, across + both page types, and state the N alongside any result. #1009's sample was small enough + that its conclusion did not survive contact with the code; replacing it with another + underpowered sample would repeat the error. + +**Step C has two outcomes, both actionable:** + +| `hold_wait_ms` median | Meaning | Effect on staging | +| --------------------- | ----------------------- | ------------------------------------------------------------ | +| Near zero | The hold is free | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| Materially non-zero | The hold **is** costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | + +The work does not change; its order and justification do. **The staging in §7 is +conditional on this measurement**, and the second outcome is a live possibility rather +than a formality — §6.2's argument for the first is weaker than an earlier draft claimed. + +Stage 1's bids-fetch timeout still needs a measured client-side figure rather than an +invented constant, but Step C is server-side and does not supply it. Capture it from the +browser harness when Stage 1 is actually scheduled. + +--- + +## 4. Stage 0 — the only build item recommended now + +Stop bypassing the read-through cache on ad-eligible navigations +(`publisher.rs:2867`). + +**Ship it as an operator flag, not a deletion.** Add +`publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as +the Step C instrumentation. Then turn it off with `ts config push`. + +The diff is slightly larger than deleting a line, and that is the point. The risk being +gated here is **cache poisoning** — serving one representation in response to a request +for another. For that class of failure, rollback speed dominates diff size: a config push +reverts the read path in seconds where a release does not — but a config push **evicts +nothing**, so full rollback is flip, then purge or roll a versioned key namespace, then +observe past the origin TTL. The flag also buys an A/B on a byte-identical +build, removing build difference as a confound in the very measurement this depends on, +and allows flipping for a tester-cookie population before all traffic. + +Retire the flag once the change has held: flip the default, then delete the setting and +its branch. A temporary flag left in place becomes permanent configuration surface. + +### What to watch after the flip + +Two regression signals, both checked before the win is: + +- **`unexpected_origin_304` abandonment rate.** That reason + (`publisher.rs:2894-2916`, + emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack + path refuses cached and conditional origin responses. Re-enabling the cache is what + could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching + TS that the conditional-header strip was supposed to make impossible. +- **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch means the `Vary` risk materialized + despite a PASS verdict. Roll back immediately; this is cache poisoning, not a + performance regression. + +**Why it is safe in principle.** The conditional-header strip runs 34 lines earlier +under the same gate (`publisher.rs:2832-2836`, +which also strips `Range`/`If-Range`), so the request already reaches the cache +unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) +added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The +strip alone satisfies its invariant. + +**But it carries a risk that design never considered — and this is the blocking +precondition.** RSC fetches are not navigations +(`is_navigation_request` +requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow +through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass +puts both representations under one cache key. #1009 states the origin varies on +`rsc`, `next-router-*`, and a publisher-specific experiment header — if that variance is +not declared via `Vary`, the +cache can serve a flight payload to an HTML navigation. + +The classification is also not airtight: `is_navigation_request` falls back to the +`Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is +weaker — `fetch()` can set Accept: text/html"_ +(`http_util.rs:84-88`). + +**A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches +already transit the read-through cache today, because they never set the bypass. If the +origin varies on `Next-Router-*` without declaring it, TS is cross-serving RSC variants +right now. On a FAIL, file that immediately and treat "ask the origin to declare `Vary`" +as urgent rather than as the cheaper of two options. + +**The `Vary` check is necessary but not sufficient.** Turning the read-through cache on +for HTML navigations exposes three things a representation check does not cover, and all +three are a larger class than the RSC split: + +- **Client `Cookie`.** TS forwards client cookies to origin unchanged — there is no + `COOKIE` strip on the publisher path. Any cookie-personalized HTML (logged-in state, + paywall meter, publisher-side A/B assignment) becomes cross-servable unless the origin + declares `Vary: Cookie` or marks those responses private. +- **Origin `Set-Cookie`.** If the origin emits `Set-Cookie` alongside a shared-cacheable + `Cache-Control`, the read-through cache can replay one visitor's cookie to the next. + TS's own privacy net downgrades **TS's** response — it runs after the cache has already + stored the origin's. +- **`Authorization`.** #1009 describes a basic-auth-gated deployment. Responses to + authorized requests entering a shared cache needs its own check. + +So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with +and without a session cookie. Same minutes of work; closes the bigger hole. + +**Two effort branches, and Step A's `Vary` result decides which** — note this selects the +_shape_ of Stage 0, while the `FINAL PASS` conditions decide _whether it ships at all_: + +| Step A result | Stage 0 is… | Effort | +| ---------------------- | --------------------------------------------- | ------ | +| Origin declares `Vary` | the flag, its tests, then a config push | 1–2 d | +| Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | + +The discriminator is the safer design either way, because it keys on the headers that +actually distinguish the representations rather than on the navigation classification. + +**Two benefits beyond TTFB, worth stating to a publisher:** + +- **Origin load drops.** The 304-prevention design explicitly accepted _"increasing + origin load"_ as a cost. This reverses it. +- **`stale-if-error` becomes reachable.** Under `Pass` an origin outage is a hard + failure. This needs a decision rather than a default: stale HTML carries stale slot + markup, and whether that beats an error is a product call. + +--- + +## 5. The trap in the deferred work — read this before scheduling Stages 1–2 + +The hold is load-bearing for something other than latency. The invariant is: + +> `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. + +The end-tag handler (`html_processor.rs:381-395`) +locks that mutex once and falls back to `build_empty_bids_script()` on `None`. + +**Removing the hold without relocating collection renders a normal page with +`tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On +Axum, Cloudflare, and Spin the loss is fully silent: +`publisher.rs:2248` holds a +bare `Option` with no guard, so not even a drop warning fires. **The +SSPs are billed regardless.** + +This is why Stage 2 is gated on three companions and a production soak, and why slot +fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled). + +--- + +## 6. The analysis + +### 6.1 Corrections to #1009's premises + +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | Partly. `tsjs.adSlots` **content** is per-URL — `build_slot_json` emits config- and path-derived fields only. But its **presence** is gated on `should_run_ad_stack` (consent, bot, prefetch, kill switch), so it is request-dependent and **must not live in a shared template**. See §6.7. | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | + +Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its +two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). + +**Credit where due.** #1009 names the hold as blocker 1 and states it correctly. What +changes here is its _causal weight_. Likewise, #1009's own observation that TS _"shifts +the auction cost from client-side to server-side rather than adding new work"_ is the +argument for client-fill, which the issue then declines in favour of ESI. + +### 6.2 What the hold actually costs + +**An earlier draft of this section claimed a stronger argument than the code supports. +It was wrong, and the correction matters.** + +The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` +(`publisher.rs:2190-2202`) +scans the **decoded origin input** for ` Dispatch precedes the origin fetch, so the hold costs `max(0, A − T)`, where `A` is the +> auction collect duration and `T` is origin TTFB plus body transfer up to the `` +> byte. Since `` sits at the end of a document, `T` is close to the full download. + +`A` is bounded by `auction_timeout_ms`, resolved as +`creative_opportunities.auction_timeout_ms` falling back to `auction.timeout_ms` +(`publisher.rs:2680-2684`) +— check the resolution order against your own config rather than trusting a number; the +shipped example sets different values at each level. + +**This is a claim requiring measurement, not a proof.** §3 Step C measures the hold's +cost directly rather than inferring it. + +A finding that does survive, and belongs with [the ceiling](#64-the-ceiling): because +`HtmlWithPostProcessing` withholds all output until the final chunk, the streaming-prefix +design at `publisher.rs:1343-1348` +— whose comment promises "the client receives the document up to `` while the +auction rides alongside transfer" — is **inert on a Next.js publisher**. Every +`step.ready` yields empty bytes. That comment is misleading on exactly the publisher +under discussion. + +### 6.3 The quantity nobody has measured + +Write the fetch time under `Pass` as `O`. Recovery depends on it, and it has never been +captured. #1009's timings cannot supply it: they compare a POP hit against a +shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` +bypasses TS's read-through cache and its shield. + +Note `Pass` bypasses **TS's** caches only. It has no authority over any CDN the publisher +runs in front of their own origin — and #1009's `x-cache: MISS, MISS` on the TS-on arm +hints one may exist. So `O` may not be origin build time at all. Since `O` is the single +quantity this model depends on, that ambiguity is worth resolving in Step C rather than +assuming. + +What follows from code alone, without any number: + +| Configuration | Long pole after the change | Recovery | +| ------------------- | -------------------------- | ------------------------------ | +| Hold removal only | origin (still `PASS`) | **none** | +| Bypass removal only | the auction budget | partial — the auction surfaces | +| **Both** | the rewrite | **the full available win** | + +That ordering is what the staging rests on, and it is measurement-independent. The +magnitude of each row is not, and §3 Step C supplies it. + +### 6.4 The ceiling + +#1009 targets "approach the TS-off warm numbers." **Unreachable, structurally.** Those +numbers are TS-off _streaming_ a POP HIT. TS buffers the whole document before emitting +a byte (16 MB cap), so its floor is `full origin body download + full rewrite` — above a +streamed hit by construction, whatever the timings turn out to be. Set the target from +Step C's measured rewrite cost rather than from the TS-off baseline. Going below the +floor requires true origin streaming (#849), out of scope. A non-Next.js publisher with +no post-processor takes the streaming path and would see a lower floor. + +### 6.5 Confidence + +**High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the +the silent-empty-bids failure mode, the geo and `Vary` blockers, +and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone +can check them without running anything. + +**None on magnitude.** `O` is unmeasured and the rewrite cost is unmeasured. This +document does not estimate them, and no figure in it should be quoted as one. + +Worth stating plainly: #1009 reached the opposite causal conclusion from a small sample. +That is a caution about small samples generally, not only about that one — which is why +§3 Step C specifies the measurement rather than this document supplying a substitute +for it. + +### 6.6 The ESI pipeline, corrected + +An earlier revision of this document said "order esi → lol*html, never the reverse." +That is backwards. `lol_html` is what \_emits* the ESI include tags; ESI cannot process +tags that do not exist yet. The correct order: + +``` +origin → lol_html transform → fastly::cache::core → finalize headers → stream esi assembly → client + (one unconditional marker (shared template, (EC cookie, geo, (per request, + at the body-close seam; surrogate-keyed, unconditional fetch the + the head seam is NOT a TS-chosen TTL) private/no-store) fragment) + hole — adSlots presence + is request-gated, §6.7) nothing may change + after this point +``` + +The push/pull mismatch that the earlier revision treated as a blocker is real but +irrelevant: `lol_html` pushes, `esi` pulls, and **the cache is the buffer between them**. +That is not an obstacle to the two-stage design — it _is_ the two-stage design, which is +what #1009 proposed in the first place. + +Mechanism, all present in the pinned `fastly` 0.12.1: + +| Need | API | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate **C2 only** | `InsertBuilder::surrogate_keys([...])` (Core Cache) + `fastly::http::purge::purge_surrogate_key`. Does **not** key C1 — see the row below. | +| Invalidate C1 | Origin-supplied surrogate keys, or the HTTP cache's own surrogate-key surface. Not the Core Cache API. | + +Purge runs **inside Compute**. The management-API token scope cited under +[Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does +not gate this. + +**Three caches, kept distinct.** Conflating them is what produced the original error: + +1. **Origin read-through** — raw origin bytes. What Stage 0 turns back on. +2. **Shared transformed template** — post-`lol_html`, pre-ESI, no per-user data. The ESI + target, and new. +3. **Assembled-response delivery cache** — the final per-user output. **Must never + exist.** Nothing in this document or the spike proposes one. + +**Validation constraint.** Viceroy 0.17 cannot exercise the customized read-through hooks +end to end. Unit tests can cover the transform and the security properties; MISS / HIT / +stale / shielding behaviour must run against a real Fastly test service. + +--- + +### 6.7 What may and may not live in a shared template + +A correction to §6.1 row 1, and the constraint that governs any shared-template design. + +The original framing — "`adSlots` is per-URL, so there is one per-user hole, not two" — +is half right and dangerously so. `build_slot_json` really does emit only config- and +path-derived fields. But whether the script is emitted **at all** is gated on +`should_run_ad_stack` (`publisher.rs:2920-2927`), which is +`is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the _content_ is per-URL and the _presence_ is per-request. A shared object filled by +the first request would freeze that request's consent decision, bot classification, +prefetch status, and kill-switch state for every later reader. A consent-denied fill +serves a no-ads template to consenting users; a consenting fill serves ad markup to +someone who refused. + +**The rule for anything cached and shared:** + +| May live in the template | Must live in the per-request fragment | +| --------------------------------------- | ---------------------------------------------- | +| tsjs bundle script tag (content-hashed) | `tsjs.adSlots` — presence is request-gated | +| URL rewrites (per-host, in the key) | `tsjs.bids` | +| | GPT diagnostics bootstrap (cookie/query-gated) | +| | Integration head-inserts (request-scoped) | + +The test that catches this class is **byte-identity of the template across requests +differing in consent, bot classification, and prefetch status** — not an absence-of- +per-user-values scan, which the broken design would have passed. + +This applies to any shared-template work, ESI or client-fill alike. The +[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. + +--- + +## 7. Deferred work, specified not scheduled + +**The full sequence, in one place.** Stage 0 is specified in [§4](#4-stage-0--the-only-build-item-recommended-now) +rather than repeated here; everything below it is deferred. + +| Stage | What | Status | +| ----- | ----------------------------------------------- | ---------------------------------------------- | +| **0** | Operator flag disabling the origin cache bypass | Recommended now. Gated on a `FINAL PASS`. §4. | +| 1 | Bid delivery off the response body | Deferred behind the correctness defects | +| 2 | Delete the `` hold | Deferred; one-way, needs a Stage 1 soak | +| 3a | Browser caching (`private, max-age` + `ETag`) | Specified, low risk, unscheduled | +| 3b | Shared cacheability | Blocked on geo suppression, `Vary`, and Step B | +| 4 | Purge wiring | Prerequisite for any TS-owned cache | + +**ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the +rest. It no longer queues: it is feasible on the pinned SDK and is decided by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` +([§6.6](#66-the-esi-pipeline-corrected)), not a new service. + +Lower detail below is deliberate. Full specifications are in the appendices. + +**Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at +navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer +already exist. Three decisions must be made before planning: the `slots: []` precedence +rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event +emission point; and whether the dispatch/collect split survives at all. Plumbing detail +in [Appendix B](#appendix-b--stage-1-plumbing-condensed). Estimated 8–13 d, low-to-medium +confidence, uncertainty concentrated client-side. + +Three companions are mandatory, not optional: **suppress the server bids script +entirely** (not an empty one), **fail loud** (the end-tag handler takes bids by value so +a missing auction is a compile error), and **relocate telemetry** (navigation +`Completed` rows are emitted only from the collect functions, and the `ts-debug` dump +rides the same string). Behaviour change to accept: under client-fill the auction runs +only if the browser executes the fetch, so bots and JS-disabled clients stop triggering +server-side auctions — revenue-relevant, sign unknown. + +**Stage 2 — delete the hold.** 5–8 d. **Rollback is one-way**: it deletes the hold, the +dispatch/collect split, and twelve tests, so the only revert is a release. Ships only +after Stage 1 has run flag-on in production for a window defined _before_ Stage 1 +starts, with TS-attributed renders flat and `auction_events_raw` navigation rows intact. +Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six compression +imports, and the non-parser-context `` runs _all_ attempts and concatenates every non-failed output** — not + first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in + the crate. +- Single include, not per-slot: the auction is one operation producing all slots' bids. + +--- + +## Appendix F — deferred open items (condensed) + +Implementation-level, for unscheduled work only. Decisions needing a human are in +[§9](#9-decisions-needed-from-this-review). + +Should `collect_non_html_auction` (`publisher.rs:2388`) go with the hold or stay? Is +`body_close_hold_loop_stream` (`:2109`, no production caller) safe to delete, or is the +buffered-adapter streaming cutover (#495) still live? Does hidden-tab rAF behaviour +interact badly with a bids timeout? What are Fastly's pending-request semantics when a +`DispatchedAuction` drops mid-flight? Does `stale-if-error` on a cached root serve +acceptable content given stale slot markup? And the googletag shim discards listeners +queued before it loads (#1009 Part 1) — not filed, should be. + +--- + +## Appendix G — code-grounded seams + +All pinned to `cfb98f4`. + +| Concern | Location | +| --------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (content per-URL, presence request-gated) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md new file mode 100644 index 000000000..5b593a15d --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -0,0 +1,260 @@ +# Streaming assembly: the architecture #1009 actually needs + +**Date:** 2026-08-11 +**Status:** Decision record. Supersedes the delivery half of the +[ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md); the cache half +stands. +**Issue:** IABTechLab/trusted-server#1009 + +> **Implementation update, 2026-08-14.** Warm C2 hits still use Design C exactly as +> specified here. Authorized cold misses now also validate the repaired +> `stackpop/esi` parser, pinned by commit: the inert C2 marker becomes one synthetic +> include only in a private working copy, resolved from the already-built reader +> fragment without an HTTP request. Parser failure falls back to the byte seam. See +> [the hybrid implementation design](./2026-08-14-1009-esi-parser-assembly-design.md). +> Sections 2–2c and Design B remain investigation history; the native self-subrequest +> design is still not implemented. + +--- + +## 1. The correction this document exists for + +An earlier reading of the latency, recorded in +[the measurement findings](../plans/2026-08-08-1009-measurement-findings.md), said the +`` hold costs approximately nothing and the whole cost is the origin-cache bypass. + +That is true **today, and only today.** It is true for a reason that stops holding the +moment the rest of this work lands: + +| | Origin fetch | Auction | Reader waits | +| ----------------------------------- | ------------ | -------------------- | ------------ | +| Today | ~650 ms | hidden inside it | ~650–800 ms | +| Cached root, **buffered** assembly | 0 | fully exposed | ~auction cap | +| Cached root, **streaming** assembly | 0 | overlapped with send | ~ms | + +The auction is dispatched before the origin fetch and both run concurrently, so the hold +costs `max(0, auction − origin)` — zero while the origin is slow. Make the root cacheable +and the origin fetch disappears; the auction then has nothing left to hide behind and +becomes the _entire_ remaining cost. + +**So the two problems are coupled, and neither fix shows a win alone.** That is why the +issue is right to treat both as prerequisites, and why measuring one at a time misleads. + +Two distinct problems get bundled in the issue as one blocker. They need different fixes: + +1. **Bids live in the response body** → the page is _uncacheable_. Fixed by templatizing. + **Done.** +2. **The response is held for the auction** → the page is _slow_. Fixed by streaming the + shell and filling the seam late. **Not done** — this document. + +## 2. What the current implementation gets wrong + +On a C2 hit, `collect_and_assemble_cached_template` awaits the auction, then assembles, +then returns a fully buffered `PublisherResponse::Buffered`. The reader receives nothing +until bids resolve. + +That relocates the hold rather than removing it, and on a hit it is _worse than today_ in +one respect: there is no origin fetch left to hide it behind, so the full auction latency +lands on first byte. + +The routing decision that caused it — shared modes take the buffered finalizer — was made +because **a store needs complete transformed bytes.** True on a miss. Irrelevant on a hit, +where the template is already materialized. + +## 2b. Demonstrated, not argued + +Run locally under `viceroy serve` against a stub origin with a **self-imposed 1.5 s bid +endpoint**. These are synthetic numbers from a delay chosen to be observable — not a +measurement of any real deployment, and not comparable to publisher data. + +| Request | Cache | TTFB | Total | Origin fetched | +| ------- | ----- | --------- | --------- | -------------- | +| 1 | miss | ~injected | ~injected | yes | +| 2 | hit | ~injected | ~injected | **no** | +| 3 | hit | ~injected | ~injected | **no** | + +Two things are visible, and both matter more than the absolute values: + +1. **The cache works.** One origin fetch across three requests; the C2 log shows one + miss, one store, two hits. +2. **The reader waits exactly as long anyway.** Time-to-first-byte equals total on every + request, so nothing streams — the entire response lands at once, after the auction. + On the hits the origin fetch is gone and first byte still tracks the injected bid + delay. + +That is the claim in §1 and §2 reproduced on demand: a cached root delivers **no latency +benefit to the reader** while the response is held for the auction. It also gives the +harness a pass/fail shape for the change this document proposes — under streaming +assembly, TTFB must fall away from total by approximately the injected delay. + +## 2c. Measured against the shipped path — a ~100x TTFB regression + +`scripts/c2-local-test.sh` runs both modes against the same stub, with a self-imposed +1.5 s bid endpoint. Synthetic numbers, not a measurement of any deployment. + +| Mode | TTFB | Total | +| ------------------------- | ----------------- | ------- | +| `inline` (shipped) | **0.010–0.019 s** | ~1.51 s | +| `esi` (buffered assembly) | **1.524–1.532 s** | ~1.53 s | + +**The shipped path already streams correctly.** First byte in ~10 ms; the article paints +while the auction runs; only `` waits. Buffered assembly turns that into a wait +for the whole auction before the first byte — roughly **100x worse TTFB than doing +nothing**. + +This corrects §1 and §2, which framed buffered assembly as capturing the origin-fetch +saving and merely failing to add the streaming benefit. It is worse than that: it +**removes** a benefit today's code already delivers. The origin-fetch saving is +irrelevant beside losing the stream. + +It also sharpens where production's latency actually goes. Locally the stub origin +answers in ~2 ms, so `inline` TTFB is ~10 ms. In production the origin fetch is slow and +uncached, and TS cannot send a first byte until the origin sends one — so production TTFB +is the **origin fetch**, with the auction hidden behind the remainder of the body plus the +`` hold. The fix is therefore a fast origin _while keeping the stream_: exactly +Design C, and exactly what buffered assembly gives up. + +**Consequence for the plan:** `esi` mode must not be exposed to any traffic in its current +form. It is not a smaller win than hoped, it is a regression. + +### A harness bug worth recording + +The first version of this comparison reported `inline` fetching the origin zero times — +nonsense that still printed four passes. Viceroy was launched inside a subshell, so `$!` +was the subshell rather than the server; cleanup killed the wrapper and orphaned viceroy. +The next run then failed to bind and **silently answered from the previous run's process**, +carrying that run's config and warm cache. + +A harness that answers from the wrong server is worse than one that crashes, because its +output looks like data. Fixed with no subshell, a pre-flight port check, and a startup +wait that fails loudly. The regression above was invisible until the control worked. + +## 3. The decisive facts + +Three, all verified in the codebase rather than assumed: + +1. **The existing streaming path already implements stream-then-stall-at-the-seam.** + `publisher.rs` builds an `async_stream::try_stream!` that streams body chunks and holds + **only** at `` for the auction (`hold_auction`, `AuctionHoldState`). This is + shipping behaviour, not new work. +2. **`EdgeBody::Stream` is an async stream** — consumers call `stream.next().await` — so an + `await` may sit between chunks. Nothing needs a nested executor. +3. **`BodyCloseInjection::Marker(String)` already exists**, and the streaming finalizers + already strip `Content-Length`. + +## 4. Three designs + +| | Streams | Auctions | Requires | Adapters | +| ----------------------------------- | ------- | -------------- | ------------------------ | --------- | +| **A** — buffered assembly (current) | No | 1 | nothing | Fastly | +| **B** — native ESI subrequest | Yes | 1, in fragment | self-referencing backend | Fastly | +| **C** — cached shell + seam split | Yes | 1 | nothing | **All 4** | + +### Design B, for the record + +`PendingFragmentContent::PendingRequest` is what the `esi` crate is built for: the +dispatcher fires a real subrequest and the processor blocks on the handle. Fastly's +`send_async`/`wait` is **synchronous**, so this sidesteps the sync-dispatcher problem +without any executor. + +It also vindicates the _original_ dispatch gate. Under B the root must **not** dispatch, +because the fragment request runs the auction. The later reversal to +`root_auction_is_useful(Esi) = true` is correct for buffered assembly and wrong for +streaming. **Dispatch-usefulness is a function of the delivery mechanism**, which is the +non-obvious coupling in this design space. + +### Design C — the recommendation + +The template carries an **inert HTML comment sentinel** where the reader's ad slots and +bids go, emitted by the existing `Marker` variant: + +``` + +``` + +On a C2 hit: + +``` +commit headers (private, no-store; no Content-Length) ← must precede any byte on Fastly +stream template[..sentinel] ← the article paints here +await the auction ← the only stall, at the very end +write the bids script +stream template[sentinel+len..] +``` + +Since a hit has the whole template in hand, this is a `split_once`, not a streaming +search. Three yields from a `try_stream!`. + +**Why a comment sentinel rather than a byte offset in metadata.** An offset is O(1), but +capturing it means plumbing the writer position into a `lol_html` end-tag handler, and it +does not survive re-encoding. A `find` over a ~100 KB buffered template is free by +comparison. + +**Why a comment rather than executable ESI markup.** An HTML comment is inert. If +assembly ever fails to substitute, the reader sees nothing; an unresolved ESI include +tag renders as visible text. Failure degrades to "no ads" instead of "broken page". + +**Why not re-run `lol_html` over the cached template.** It would inject a second tsjs +`" + ), + format!("", GPT_BOOTSTRAP_JS), + ]; + ``` + + Verify the false string remains exactly: + + ```text + + ``` + +- [ ] **Step 5: Override the metadata hook from the same `GptConfig`.** + + ```rust + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.config.gam_attribution_enabled { + vec![("data-ts-gam-attribution", "true")] + } else { + Vec::new() + } + } + ``` + + Do not store this state in `HtmlProcessorConfig` or + `IntegrationDocumentState`. + +- [ ] **Step 6: Run focused and neighboring GPT tests.** + + ```bash + cargo test-fastly gam_attribution + cargo test-fastly head_injector + ``` + + Expected: PASS; false preserves two current inserts, true adds the flag and + metadata while still emitting two inserts when `slim_prebid_url` is absent. + +- [ ] **Step 7: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt.rs crates/trusted-server-core/src/integrations/registry.rs + git commit -m "Add GPT GAM attribution option" + ``` + +## Task 2: Put activation metadata on only the publisher bundle tag + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs:1043-1054` +- Modify: `crates/trusted-server-core/src/tsjs.rs:11-39` +- Modify: `crates/trusted-server-core/src/html_processor.rs:324-360` +- Test: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: `crates/trusted-server-core/src/tsjs.rs:161-187` +- Test: `crates/trusted-server-core/src/html_processor.rs:768-820` +- Test: `crates/trusted-server-core/src/html_processor.rs:1637-1671` + +- [ ] **Step 1: Write failing registry and tag-rendering tests.** + + Add a test head injector whose metadata method returns the attribution pair. + Assert registry aggregation is deterministic and preserves the default-empty + behavior of injectors that implement only `head_inserts`. + + Add exact tag tests: + + ```rust + #[test] + fn publisher_script_tag_renders_static_attributes() { + let ids = ["gpt"]; + let src = tsjs_script_src(&ids); + + assert_eq!( + tsjs_script_tag_with_attributes( + &ids, + &[("data-ts-gam-attribution", "true")] + ), + format!( + "" + ) + ); + assert_eq!( + tsjs_script_tag(&ids), + format!("") + ); + } + ``` + + The final string must contain no formatting whitespace introduced only by the + multiline example. + +- [ ] **Step 2: Write a failing HTML matrix test.** + + Process `` with: + 1. a real enabled GPT registry with attribution true; + 2. enabled GPT with attribution false; and + 3. no GPT integration. + + Assert exactly one `#trustedserver-js` tag in every case, the attribute only + in case 1, and integration head inserts remain before the external bundle. + Also retain the generic `tsjs_unified_script_tag()` exact-output test so + creative/all-modules callers stay unmarked. + +- [ ] **Step 3: Run the tests and confirm RED.** + + ```bash + cargo test-fastly tsjs_script_tag + cargo test-fastly integration_head_injector + ``` + + Expected: FAIL because the registry aggregator and attributed publisher + helper are not implemented. + +- [ ] **Step 4: Aggregate integration-owned static attributes.** + + Add beside `IntegrationRegistry::head_inserts`: + + ```rust + #[must_use] + pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.tsjs_script_tag_attributes()) + .collect() + } + ``` + + Keep the hook default-empty so existing integration injectors and test doubles + compile without changes. + +- [ ] **Step 5: Add the publisher-only tag helper.** + + Render only trusted, compile-time static attribute pairs: + + ```rust + #[must_use] + pub fn tsjs_script_tag_with_attributes( + module_ids: &[&str], + attributes: &[(&'static str, &'static str)], + ) -> String { + let attributes = attributes + .iter() + .map(|(name, value)| format!(" {name}=\"{value}\"")) + .collect::(); + format!( + "", + tsjs_script_src(module_ids) + ) + } + ``` + + Have `tsjs_script_tag(module_ids)` retain its exact output, either directly or + by delegating with an empty slice. Do not change + `tsjs_unified_script_tag()` or either creative call site. + +- [ ] **Step 6: Wire only the publisher HTML path.** + + Replace the single `html_processor.rs` call with: + + ```rust + let immediate_ids = integrations.js_module_ids_immediate(); + let script_attributes = integrations.tsjs_script_tag_attributes(); + snippet.push_str(&tsjs::tsjs_script_tag_with_attributes( + &immediate_ids, + &script_attributes, + )); + ``` + + Preserve source order: ad slots, integration head inserts, diagnostics + bootstrap, one synchronous bundle, diagnostics module, deferred bundles. + +- [ ] **Step 7: Run focused tests.** + + ```bash + cargo test-fastly tsjs_script_tag + cargo test-fastly integration_head_injector + cargo test-fastly golden_script_tag + ``` + + Expected: PASS; false/non-GPT/generic output is unmarked and true output has + one attributed publisher tag. + +- [ ] **Step 8: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/registry.rs crates/trusted-server-core/src/tsjs.rs crates/trusted-server-core/src/html_processor.rs + git commit -m "Authorize GAM attribution bundle" + ``` + +## Task 3: Queue the primary marker before the bootstrap guard + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js:17-45` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts:8-220` +- Test: `crates/trusted-server-core/src/integrations/gpt.rs:1131-1454` + +- [ ] **Step 1: Extend the raw-source test harness.** + + Add the optional page flag and `setConfig` surface: + + ```typescript + interface MockGoogleTag { + cmd: MockCommandQueue + setConfig?: (config: Record) => void + // retain the existing members + } + + type TestWindow = Omit & { + googletag?: MockGoogleTag + tsjs?: Partial + __tsjs_gam_attribution_enabled?: boolean + } + + function makeGoogleTag( + overrides: Partial = {} + ): MockGoogleTag { + return { + cmd: [], + defineSlot: vi.fn(), + pubads: vi.fn(() => ({})), + enableServices: vi.fn(), + display: vi.fn(), + ...overrides, + } + } + ``` + + Delete the flag in both `beforeEach` and `afterEach`. + +- [ ] **Step 2: Write failing behavioral tests.** + + Cover all of these independently: + - default/false plus a preinstalled `ts.adInit` returns without creating + `window.googletag`; + - true queues the exact string-valued targeting callback before a publisher + callback appended after `runBootstrap()`; + - true plus preinstalled `ts.adInit` still queues and applies targeting but + does not replace `adInit` or install the fallback scheduler; + - missing `setConfig` is a no-op and the initial-load detector and `adInit` + still install; + - throwing `setConfig` is caught inside the marker callback, and a later + publisher callback still executes; + - the wrapped `disableInitialLoad` path still records + `ts.gptInitialLoadDisabled`. + + Use a real array queue, append a publisher spy after bootstrap execution, and + drain a snapshot in order: + + ```typescript + const queue: Array<() => void> = [] + const setConfig = vi.fn() + ;(window as TestWindow).googletag = makeGoogleTag({ cmd: queue, setConfig }) + ;(window as TestWindow).__tsjs_gam_attribution_enabled = true + + runBootstrap() + const publisherCommand = vi.fn() + queue.push(publisherCommand) + ;[...queue].forEach((command) => command()) + + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }) + expect(setConfig.mock.invocationCallOrder[0]).toBeLessThan( + publisherCommand.mock.invocationCallOrder[0] + ) + ``` + +- [ ] **Step 3: Run the raw bootstrap tests and confirm RED.** + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts + ``` + + Expected: targeting assertions fail because the raw bootstrap returns before + any marker enqueue. + +- [ ] **Step 4: Implement one flag-gated queue initialization before the guard.** + + Preserve the local `ts` namespace and reuse `tag` in the existing detector: + + ```javascript + var ts = (window.tsjs = window.tsjs || {}) + var tag + + if (window.__tsjs_gam_attribution_enabled === true) { + tag = window.googletag = window.googletag || { cmd: [] } + tag.cmd = tag.cmd || [] + tag.cmd.push(function () { + try { + var gpt = window.googletag + if (gpt && typeof gpt.setConfig === 'function') { + // "ts" is the fixed GAM key, not the local window.tsjs alias. + gpt.setConfig({ targeting: { ts: 'true' } }) + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap queue. + } + }) + } + + if (ts.adInit) return + + tag = tag || (window.googletag = window.googletag || { cmd: [] }) + tag.cmd = tag.cmd || [] + tag.cmd.push(function () { + // existing initial-load detector body, unchanged + }) + ``` + + Do not add a global deduplication state machine, network call, beacon, cookie + read, slot-level key, or third head insert. + +- [ ] **Step 5: Add/retain Rust source-order assertions.** + + In `gpt.rs`, assert the embedded bootstrap's attribution enqueue occurs before + `if (ts.adInit) return;` and before the executable + `googletag.display(` and `googletag.pubads().refresh(` tokens. Do not compare + against comment-only `display()`/`refresh()` text. Retain `ts_initial` + assertions and the two-insert count. + +- [ ] **Step 6: Run focused JS and Rust tests.** + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/gpt_bootstrap.test.ts + cd ../../.. + cargo test-fastly head_inserts + ``` + + Expected: PASS; default behavior is unchanged and every failure mode is + isolated from the existing bootstrap. + +- [ ] **Step 7: Commit.** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt_bootstrap.js crates/trusted-server-core/src/integrations/gpt.rs crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts + git commit -m "Queue page-level GAM attribution" + ``` + +## Task 4: Add the exact-executing-tag bundle fallback + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:259-307` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:1878-1898` +- Modify: `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts:350-421` + +- [ ] **Step 1: Add a test helper that controls `document.currentScript`.** + + In the runtime-gating suite, install a configurable getter before a fresh + dynamic import and restore it afterward: + + ```typescript + let executingScript: HTMLScriptElement | null + + Object.defineProperty(document, 'currentScript', { + configurable: true, + get: () => executingScript, + }) + ``` + + Use actual `" - .to_string(), + format!( + "" + ), format!("", GPT_BOOTSTRAP_JS), ]; @@ -508,6 +519,14 @@ impl IntegrationHeadInjector for GptIntegration { scripts } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.config.gam_attribution_enabled { + vec![("data-ts-gam-attribution", "true")] + } else { + Vec::new() + } + } } /// Inline `window.tsjs.adInit` bootstrap injected at `` so the bids @@ -549,6 +568,7 @@ mod tests { fn test_config() -> GptConfig { GptConfig { enabled: true, + gam_attribution_enabled: false, script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, @@ -573,6 +593,29 @@ mod tests { .expect("should build HTTP request") } + #[test] + fn gam_attribution_defaults_to_disabled() { + let config: GptConfig = + serde_json::from_value(serde_json::json!({})).expect("should parse defaults"); + + assert!(!config.gam_attribution_enabled); + } + + #[test] + fn gam_attribution_deserializes_explicit_values() { + let disabled: GptConfig = serde_json::from_value(serde_json::json!({ + "gam_attribution_enabled": false + })) + .expect("should parse explicit false"); + let enabled: GptConfig = serde_json::from_value(serde_json::json!({ + "gam_attribution_enabled": true + })) + .expect("should parse explicit true"); + + assert!(!disabled.gam_attribution_enabled); + assert!(enabled.gam_attribution_enabled); + } + // -- URL detection -- #[test] @@ -1146,6 +1189,38 @@ mod tests { "", "should set the enable flag and call the GPT shim activation function" ); + assert!( + integration.tsjs_script_tag_attributes().is_empty(), + "should not authorize GAM attribution metadata by default" + ); + } + + #[test] + fn gam_attribution_true_adds_both_activation_signals_without_a_new_insert() { + let integration = GptIntegration::new(GptConfig { + gam_attribution_enabled: true, + ..test_config() + }); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&context); + + assert_eq!(inserts.len(), 2, "should not add another head insert"); + assert!( + inserts[0].contains("window.__tsjs_gam_attribution_enabled=true;"), + "should activate the early bootstrap marker" + ); + assert_eq!( + integration.tsjs_script_tag_attributes(), + vec![("data-ts-gam-attribution", "true")], + "should authorize the bundle fallback on the publisher tag" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 16cbac868..66d56e4dc 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -575,6 +575,11 @@ pub trait IntegrationHeadInjector: Send + Sync { fn integration_id(&self) -> &'static str; /// Return HTML snippets to insert at the start of ``. fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; + + /// Return attributes to add to the publisher TSJS bundle tag. + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + Vec::new() + } } /// Registration payload returned by integration builders. From 990b6bfdb866770d98732b0fab945ea848c9f67e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 20:22:34 +0530 Subject: [PATCH 170/195] Authorize GAM attribution bundle --- .../trusted-server-core/src/html_processor.rs | 71 ++++++++++++++++++- .../src/integrations/registry.rs | 62 ++++++++++++++++ crates/trusted-server-core/src/tsjs.rs | 44 ++++++++++-- 3 files changed, 171 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3bff588fe..12d5b3636 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -356,7 +356,11 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } // Main bundle: core + non-deferred integrations (synchronous). let immediate_ids = integrations.js_module_ids_immediate(); - snippet.push_str(&tsjs::tsjs_script_tag(&immediate_ids)); + let script_attributes = integrations.tsjs_script_tag_attributes(); + snippet.push_str(&tsjs::tsjs_script_tag_with_attributes( + &immediate_ids, + &script_attributes, + )); // Active diagnostics loads synchronously after core so its // GPT listeners precede publisher scripts in the origin head. if let Some(module_tag) = gpt_diagnostics @@ -835,6 +839,71 @@ mod tests { ); } + #[test] + fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { + fn process(gam_attribution_enabled: Option) -> String { + let integrations = if let Some(gam_attribution_enabled) = gam_attribution_enabled { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "gpt", + &json!({ + "enabled": true, + "gam_attribution_enabled": gam_attribution_enabled + }), + ) + .expect("should insert GPT config"); + IntegrationRegistry::new(&settings).expect("should build GPT registry") + } else { + IntegrationRegistry::empty_for_tests() + }; + let mut config = create_test_config(); + config.integrations = integrations; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"", true) + .expect("should process HTML"); + + String::from_utf8(output).expect("should produce valid UTF-8") + } + + let attributed = process(Some(true)); + let unattributed = process(Some(false)); + let without_gpt = process(None); + + for html in [&attributed, &unattributed, &without_gpt] { + assert_eq!( + html.matches("id=\"trustedserver-js\"").count(), + 1, + "should emit exactly one publisher bundle tag: {html}" + ); + } + assert!( + attributed.contains("data-ts-gam-attribution=\"true\""), + "should mark only an attribution-enabled GPT publisher bundle" + ); + assert!( + !unattributed.contains("data-ts-gam-attribution"), + "should leave an attribution-disabled GPT publisher bundle unmarked" + ); + assert!( + !without_gpt.contains("data-ts-gam-attribution"), + "should leave a non-GPT publisher bundle unmarked" + ); + + let head_insert_index = attributed + .find("window.__tsjs_installGptShim") + .expect("should include the GPT head insert"); + let publisher_bundle_index = attributed + .find("id=\"trustedserver-js\"") + .expect("should include the publisher bundle"); + assert!( + head_insert_index < publisher_bundle_index, + "should keep integration head inserts before the publisher bundle" + ); + } + #[test] fn active_gpt_diagnostics_loads_standalone_after_unified_bundle_once() { let html = "Test"; diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 66d56e4dc..fb93ac5b3 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1058,6 +1058,16 @@ impl IntegrationRegistry { inserts } + /// Collect static attributes for the publisher TSJS bundle tag. + #[must_use] + pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.tsjs_script_tag_attributes()) + .collect() + } + /// Provide a snapshot of registered integrations and their hooks. #[must_use] pub fn registered_integrations(&self) -> Vec { @@ -1326,6 +1336,58 @@ mod tests { use crate::platform::test_support::noop_services; use http::{HeaderValue, StatusCode, header}; + struct DefaultMetadataHeadInjector; + + impl IntegrationHeadInjector for DefaultMetadataHeadInjector { + fn integration_id(&self) -> &'static str { + "default-metadata" + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + } + + struct StaticMetadataHeadInjector; + + impl IntegrationHeadInjector for StaticMetadataHeadInjector { + fn integration_id(&self) -> &'static str { + "static-metadata" + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + vec![ + ("data-ts-gam-attribution", "true"), + ("data-test-order", "second"), + ] + } + } + + #[test] + fn tsjs_script_tag_attributes_preserve_registration_order_and_default_empty() { + let registry = IntegrationRegistry::from_rewriters_with_head_injectors( + Vec::new(), + Vec::new(), + vec![ + Arc::new(DefaultMetadataHeadInjector), + Arc::new(StaticMetadataHeadInjector), + ], + ); + + assert_eq!( + registry.tsjs_script_tag_attributes(), + vec![ + ("data-ts-gam-attribution", "true"), + ("data-test-order", "second"), + ], + "should omit default-empty metadata and preserve registered attribute order" + ); + } + // Mock integration proxy for testing struct MockProxy; diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index 133e6d011..f4c9a13a5 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -11,9 +11,23 @@ pub fn tsjs_script_src(module_ids: &[&str]) -> String { /// `", - tsjs_script_src(module_ids) + "", + tsjs_script_src(module_ids), ) } @@ -170,19 +184,39 @@ mod tests { ); } + #[test] + fn publisher_tsjs_script_tag_renders_static_attributes() { + let module_ids = ["gpt"]; + let src = tsjs_script_src(&module_ids); + + assert_eq!( + tsjs_script_tag_with_attributes(&module_ids, &[("data-ts-gam-attribution", "true")]), + format!( + "" + ), + "should render trusted static attributes on the publisher bundle tag" + ); + assert_eq!( + tsjs_script_tag(&module_ids), + format!(""), + "should keep the generic tag byte-for-byte unmarked" + ); + } + #[test] fn tsjs_unified_helpers_use_all_module_ids() { let ids = all_module_ids(); + let src = tsjs_unified_script_src(); assert_eq!( - tsjs_unified_script_src(), + src, tsjs_script_src(&ids), "should hash all module IDs for the unified script source" ); assert_eq!( tsjs_unified_script_tag(), - tsjs_script_tag(&ids), - "should wrap the all-module unified script source" + format!(""), + "should keep the all-module generic tag byte-for-byte unmarked" ); } From 0bfee82a58863c5c11268ff3069ae307b3fc23c9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 20:26:23 +0530 Subject: [PATCH 171/195] Cover disabled GPT attribution --- .../trusted-server-core/src/html_processor.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 12d5b3636..10dce6658 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -841,15 +841,15 @@ mod tests { #[test] fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { - fn process(gam_attribution_enabled: Option) -> String { - let integrations = if let Some(gam_attribution_enabled) = gam_attribution_enabled { + fn process(gpt_config: Option<(bool, bool)>) -> String { + let integrations = if let Some((enabled, gam_attribution_enabled)) = gpt_config { let mut settings = create_test_settings(); settings .integrations .insert_config( "gpt", &json!({ - "enabled": true, + "enabled": enabled, "gam_attribution_enabled": gam_attribution_enabled }), ) @@ -868,11 +868,12 @@ mod tests { String::from_utf8(output).expect("should produce valid UTF-8") } - let attributed = process(Some(true)); - let unattributed = process(Some(false)); + let attributed = process(Some((true, true))); + let unattributed = process(Some((true, false))); + let disabled_gpt = process(Some((false, true))); let without_gpt = process(None); - for html in [&attributed, &unattributed, &without_gpt] { + for html in [&attributed, &unattributed, &disabled_gpt, &without_gpt] { assert_eq!( html.matches("id=\"trustedserver-js\"").count(), 1, @@ -887,6 +888,10 @@ mod tests { !unattributed.contains("data-ts-gam-attribution"), "should leave an attribution-disabled GPT publisher bundle unmarked" ); + assert!( + !disabled_gpt.contains("data-ts-gam-attribution"), + "should let the GPT master switch suppress attribution metadata" + ); assert!( !without_gpt.contains("data-ts-gam-attribution"), "should leave a non-GPT publisher bundle unmarked" From 06ec21326e7f2cc88cbfc637e197ebaffd89e358 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 20:31:13 +0530 Subject: [PATCH 172/195] Queue page-level GAM attribution --- .../src/integrations/gpt.rs | 29 ++++ .../src/integrations/gpt_bootstrap.js | 22 +++- .../integrations/gpt/gpt_bootstrap.test.ts | 124 ++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index f4f09bbff..533944027 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1427,6 +1427,35 @@ mod tests { ); } + #[test] + fn head_inserts_queue_gam_attribution_before_guard_and_ad_requests() { + let targeting_index = GPT_BOOTSTRAP_JS + .find("gpt.setConfig({ targeting: { ts: 'true' } })") + .expect("should apply the fixed page-level GAM targeting pair"); + let guard_index = GPT_BOOTSTRAP_JS + .find("if (ts.adInit) return;") + .expect("should retain the preinstalled adInit guard"); + let display_index = GPT_BOOTSTRAP_JS + .find("googletag.display(divId);") + .expect("should retain the executable GPT display call"); + let refresh_index = GPT_BOOTSTRAP_JS + .find("googletag.pubads().refresh(slotsNeedingRefresh);") + .expect("should retain the bounded GPT refresh call"); + + assert!( + targeting_index < guard_index, + "should enqueue attribution before the preinstalled adInit guard" + ); + assert!( + targeting_index < display_index, + "should enqueue attribution before the executable display call" + ); + assert!( + targeting_index < refresh_index, + "should enqueue attribution before the executable refresh call" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 86b51ffa7..e336d0438 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -17,6 +17,24 @@ (function () { if (typeof window === "undefined") return; var ts = (window.tsjs = window.tsjs || {}); + var tag; + + if (window.__tsjs_gam_attribution_enabled === true) { + tag = window.googletag = window.googletag || { cmd: [] }; + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { + try { + var gpt = window.googletag; + if (gpt && typeof gpt.setConfig === "function") { + // "ts" is the fixed GAM key, not the local window.tsjs alias. + gpt.setConfig({ targeting: { ts: 'true' } }); + } + } catch (_) { + // Attribution must not interrupt the existing bootstrap queue. + } + }); + } + if (ts.adInit) return; // Track whether the publisher disabled GPT initial load. Read the effective @@ -38,7 +56,9 @@ return true; } - (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { + tag = tag || (window.googletag = window.googletag || { cmd: [] }); + tag.cmd = tag.cmd || []; + tag.cmd.push(function () { var gpt = window.googletag; syncInitialLoadDisabled(gpt); if ( diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index d3e1d7099..4ec4110b5 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -33,6 +33,8 @@ interface MockGoogleTag { pubads: () => unknown; enableServices: () => void; display: (divId: string) => void; + getConfig?: (key: string) => Record; + setConfig?: (config: Record) => void; } // `tsjs` is declared globally as the full `TsjsApi`; `Omit` drops it from @@ -40,8 +42,25 @@ interface MockGoogleTag { type TestWindow = Omit & { googletag?: MockGoogleTag; tsjs?: Partial; + __tsjs_gam_attribution_enabled?: boolean; }; +function makeGoogleTag(overrides: Partial = {}): MockGoogleTag { + const pubads = { + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + + return { + cmd: [], + defineSlot: vi.fn(), + pubads: vi.fn(() => pubads), + enableServices: vi.fn(), + display: vi.fn(), + ...overrides, + }; +} + function runBootstrap(): void { // Evaluate in the jsdom global scope, exactly as an inline '); + clonedDocument.close(); + executingScript = clonedDocument.querySelector('script'); + const queue: Array<() => void> = []; + const setConfig = vi.fn(); + win.googletag = makeGoogleTag({ cmd: queue, setConfig }); + + await importFreshGptBundle(); + queue[0](); + + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + }); + + it.each(['missing', 'throwing'])( + 'keeps module installation working with %s setConfig', + async (setConfigMode) => { + const queue: Array<() => void> = []; + const setConfig = + setConfigMode === 'throwing' + ? vi.fn(() => { + throw new Error('publisher setConfig failed'); + }) + : undefined; + win.googletag = makeGoogleTag({ cmd: queue, setConfig }); + win.__tsjs_slim_prebid_url = 'https://cdn.example.com/slim-prebid.js'; + executingScript = attributedScript(); + + await importFreshGptBundle(); + + expect(() => [...queue].forEach((command) => command())).not.toThrow(); + expect(typeof win.tsjs?.adInit).toBe('function'); + expect(typeof win.tsjs?.scheduleInitialAdInit).toBe('function'); + expect(win.tsjs?.spaHookInstalled).toBe(true); + expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('load', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('message', expect.any(Function)); + if (setConfig) { + expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); + } + } + ); + + it('preserves GPT-enabled shim behavior without queuing attribution when unmarked', async () => { + const queue: Array<() => void> = []; + const setConfig = vi.fn(); + const tag = makeGoogleTag({ cmd: queue, setConfig }); + win.googletag = tag; + win.__tsjs_gpt_enabled = true; + executingScript = document.createElement('script'); + + await importFreshGptBundle(); + [...queue].forEach((command) => command()); + const guard = await importGuardModule(); + + expect(guard.isGuardInstalled()).toBe(true); + expect(win.googletag).toBe(tag); + expect(win.googletag!.cmd).toBe(queue); + expect(setConfig).not.toHaveBeenCalled(); + expect(typeof win.tsjs?.adInit).toBe('function'); + }); +}); + describe('GPT debug ADM iframe hardening', () => { it('sandbox token list omits allow-same-origin', async () => { const mod = await import('../../../src/integrations/gpt/index'); From d65f6562e4315cd7c6167526b09487db6f4d958b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:31:52 +0530 Subject: [PATCH 174/195] Characterize targeting collisions --- .../src/creative_opportunities.rs | 9 ++- crates/trusted-server-core/src/publisher.rs | 11 +++- .../lib/test/integrations/gpt/ad_init.test.ts | 3 +- .../test/integrations/prebid/index.test.ts | 58 ++++++++++++++++--- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index e44b0cbcf..6fc9c8dc5 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -1711,11 +1711,18 @@ mod tests { #[test] fn to_ad_slot_sets_floor_price_and_formats() { - let slot = make_slot("atf", vec!["/"]); + let mut slot = make_slot("atf", vec!["/"]); + slot.targeting + .insert("ts".to_string(), "operator-value".to_string()); let ad_slot = slot.to_ad_slot(); assert_eq!(ad_slot.id, "atf"); assert_eq!(ad_slot.floor_price, Some(0.50)); assert_eq!(ad_slot.formats.len(), 1); + assert_eq!( + ad_slot.targeting.get("ts"), + Some(&serde_json::Value::String("operator-value".to_owned())), + "should preserve operator-provided ts targeting verbatim" + ); } #[test] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4bed98327..00c403408 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -8736,9 +8736,14 @@ mod tests { #[test] fn ad_slots_script_contains_slot_data() { - let slots = vec![make_slot()]; + let mut slot = make_slot(); + slot.targeting + .insert("ts".to_string(), "operator-value".to_string()); + let slots = vec![slot]; let config = make_config(); let script = build_ad_slots_script(&slots, &config, "/"); + let slot_json = crate::publisher::build_slot_json(&slots[0], &config, "example") + .expect("should build slot JSON"); assert!( script.contains("window.tsjs=window.tsjs||{}"), "should initialise tsjs namespace" @@ -8753,6 +8758,10 @@ mod tests { !script.contains("__ts_request_id"), "must NOT contain request_id" ); + assert_eq!( + slot_json["targeting"]["ts"], "operator-value", + "should forward operator-provided ts targeting verbatim" + ); } #[test] diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 179a810d5..7e7c6d10a 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -1874,7 +1874,7 @@ describe('installTsAdInit', () => { setTargeting: vi.fn().mockReturnThis(), clearTargeting, getSlotElementId: vi.fn().mockReturnValue('div-old-route'), - getTargeting: vi.fn().mockReturnValue([]), + getTargeting: vi.fn((key: string) => (key === 'ts' ? ['publisher-value'] : [])), }; const mockPubads = { enableSingleRequest: vi.fn(), @@ -1908,6 +1908,7 @@ describe('installTsAdInit', () => { expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(clearTargeting).toHaveBeenCalledWith('pos'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); expect(mockPubads.refresh).not.toHaveBeenCalled(); expect((window as TestWindow).tsjs!.divToSlotId).toEqual({}); expect((window as TestWindow).tsjs!.prevSlotTargetingKeys).toEqual({}); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 9f9a3f977..8ead01aa8 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -414,9 +414,11 @@ describe('prebid/installPrebidNpm', () => { delete testWindow.__tsjs_prebid_diagnostics; delete testWindow.tsjs; delete mockPbjs['__tsApsBidResponseListenerInstalled']; + delete mockPbjs.bidderSettings; }); afterEach(() => { + delete mockPbjs.bidderSettings; vi.restoreAllMocks(); }); @@ -1130,6 +1132,35 @@ describe('prebid/installPrebidNpm', () => { }); describe('requestBids shim', () => { + it('preserves publisher ts adserverTargeting while adding trustedServer settings', () => { + const publisherTargeting = [{ key: 'ts', val: () => 'publisher-value' }]; + mockPbjs.bidderSettings = { + exampleBidder: { adserverTargeting: publisherTargeting }, + }; + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'exampleBidder', params: {} }] }], + } as unknown as RequestBidsArg); + + const bidderSettings = mockPbjs.bidderSettings as { + exampleBidder: { adserverTargeting: typeof publisherTargeting }; + trustedServer: { + allowAlternateBidderCodes: boolean; + allowedAlternateBidderCodes: string[]; + }; + }; + expect(bidderSettings.exampleBidder.adserverTargeting).toBe(publisherTargeting); + expect(bidderSettings.exampleBidder.adserverTargeting[0].key).toBe('ts'); + expect(bidderSettings.exampleBidder.adserverTargeting[0].val()).toBe('publisher-value'); + expect(bidderSettings.trustedServer).toEqual( + expect.objectContaining({ + allowAlternateBidderCodes: true, + allowedAlternateBidderCodes: ['*'], + }) + ); + }); + it('injects trustedServer bidder into every ad unit', () => { const pbjs = installPrebidNpm(); @@ -1850,25 +1881,33 @@ describe('prebid/installRefreshHandler', () => { it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { const originalRefresh = vi.fn(); - const clearTargeting = vi.fn(); + const slotTargeting = new Map([ + ['ts_initial', ['1']], + ['zone', ['homepage']], + ]); + const clearTargeting = vi.fn((key: string) => { + slotTargeting.delete(key); + }); + const setTargeting = vi.fn((key: string, value: string | string[]) => { + slotTargeting.set(key, Array.isArray(value) ? value : [value]); + }); const gptSlot = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn((key: string) => { - if (key === 'ts_initial') return ['1']; - if (key === 'zone') return ['homepage']; - return []; - }), + getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), getSizes: vi.fn(() => [ { getWidth: () => 970, getHeight: () => 250 }, { getWidth: () => 728, getHeight: () => 90 }, ]), clearTargeting, + setTargeting, }; const pubads = { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - const setTargetingForGPTAsync = vi.fn(); + const setTargetingForGPTAsync = vi.fn(() => { + gptSlot.setTargeting('ts', 'prebid-value'); + }); mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, @@ -1918,12 +1957,17 @@ describe('prebid/installRefreshHandler', () => { expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); expect(originalRefresh).not.toHaveBeenCalled(); const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; bidsBackHandler(); expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(setTargetingForGPTAsync.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(slotTargeting.get('ts')).toEqual(['prebid-value']); expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); }); From ef37c1154fc847eacfd159fba8b8cd9bba44b626 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:33:56 +0530 Subject: [PATCH 175/195] Define GAM attribution streaming semantics --- crates/trusted-server-core/src/publisher.rs | 44 ++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 00c403408..d95f82567 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -7754,7 +7754,15 @@ mod tests { } fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { - let settings = Arc::new(create_test_settings()); + streaming_finalize_response_with_settings(params, body, create_test_settings()) + } + + fn streaming_finalize_response_with_settings( + params: OwnedProcessResponseParams, + body: EdgeBody, + settings: Settings, + ) -> EdgeBody { + let settings = Arc::new(settings); let registry = Arc::new( IntegrationRegistry::new(&settings).expect("should create integration registry"), ); @@ -7807,6 +7815,40 @@ mod tests { } } + #[test] + fn streaming_finalize_emits_gam_attribution_head_before_origin_eof() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "gpt", + &serde_json::json!({ + "enabled": true, + "gam_attribution_enabled": true + }), + ) + .expect("should insert GPT config"); + + let body = streaming_finalize_response_with_settings( + html_stream_params("", None), + origin_chunk_then_pending(bytes::Bytes::from_static( + b"

origin remains pending

", + )), + settings, + ); + let html = String::from_utf8(first_lazy_body_chunk(body).to_vec()) + .expect("should emit UTF-8 HTML"); + + assert!( + html.contains("__tsjs_gam_attribution_enabled=true"), + "first rewritten head chunk should carry the primary activation flag: {html}" + ); + assert!( + html.contains("data-ts-gam-attribution=\"true\""), + "first rewritten head chunk should authorize the bundle fallback: {html}" + ); + } + #[test] fn streaming_finalize_emits_compressed_html_before_origin_eof() { // The FCP regression from #849: the lazy body must emit its first From 17494141ca593bea370a8b34feaede9105d1588d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 09:57:52 +0530 Subject: [PATCH 176/195] Document GAM attribution configuration --- .../tests/config_env_overlay.rs | 44 +++++++++++ .../tests/shared/script-injection.spec.ts | 1 + .../configs/trusted-server.integration.toml | 1 + docs/guide/integrations/gpt.md | 78 +++++++++++++++++-- trusted-server.example.toml | 3 + 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-cli/tests/config_env_overlay.rs b/crates/trusted-server-cli/tests/config_env_overlay.rs index 39345137b..35263c0eb 100644 --- a/crates/trusted-server-cli/tests/config_env_overlay.rs +++ b/crates/trusted-server-cli/tests/config_env_overlay.rs @@ -29,6 +29,7 @@ ids = ["trusted_server_secrets"] "#; const REWRITE_ENV: &str = "TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES"; const SANITIZE_ENV: &str = "TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES"; +const GAM_ATTRIBUTION_ENV: &str = "TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED"; struct MigratedProject { directory: TempDir, @@ -112,6 +113,49 @@ fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { ); } +#[test] +fn migrated_legacy_config_applies_gam_attribution_environment_override() { + let project = migrated_legacy_project(); + let output = Command::new(env!("CARGO_BIN_EXE_ts")) + .args(["config", "push", "--adapter", "axum", "--manifest"]) + .arg(&project.manifest_path) + .arg("--app-config") + .arg(&project.config_path) + .args(["--yes", "--no-diff"]) + .current_dir(project.directory.path()) + .env(GAM_ATTRIBUTION_ENV, "true") + .output() + .expect("should run ts config push"); + + assert!( + output.status.success(), + "valid boolean overlay should push successfully: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let local_store_path = project + .directory + .path() + .join(".edgezero/local-config-trusted_server_config.json"); + let local_store: serde_json::Value = serde_json::from_str( + &fs::read_to_string(local_store_path).expect("should read pushed local config"), + ) + .expect("should parse local config store"); + let envelope_json = local_store + .as_object() + .and_then(|entries| entries.values().next()) + .and_then(serde_json::Value::as_str) + .expect("should contain a blob envelope"); + let envelope: serde_json::Value = + serde_json::from_str(envelope_json).expect("should parse blob envelope"); + + assert_eq!( + envelope["data"]["integrations"]["gpt"]["gam_attribution_enabled"], + serde_json::Value::Bool(true), + "pushed config should contain the GAM attribution environment override" + ); +} + #[test] fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { let project = migrated_legacy_project(); diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts index 3a38aa746..78059596f 100644 --- a/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/shared/script-injection.spec.ts @@ -9,6 +9,7 @@ test.describe("Script injection", () => { const src = await scriptTag.getAttribute("src"); expect(src).toContain("/static/tsjs="); + await expect(scriptTag).not.toHaveAttribute("data-ts-gam-attribution"); }); test("no unexpected console errors on page load", async ({ page }) => { diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index 17d7c2713..d8e35d179 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -86,6 +86,7 @@ rewrite_sdk = true [integrations.gpt] enabled = false +gam_attribution_enabled = false script_url = "https://ads.example.com/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index f38f68231..174301ba1 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -53,6 +53,7 @@ Add GPT configuration to `trusted-server.toml`: ```toml [integrations.gpt] enabled = true +gam_attribution_enabled = false script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true @@ -60,12 +61,18 @@ rewrite_script = true ### Configuration Options -| Field | Type | Required | Default | Description | -| ------------------- | ------- | -------- | ------------------------------------------------------ | ------------------------------------------ | -| `enabled` | boolean | No | `true` | Enable/disable the integration | -| `script_url` | string | No | `https://securepubads.g.doubleclick.net/tag/js/gpt.js` | URL for the GPT bootstrap script | -| `cache_ttl_seconds` | integer | No | `3600` | Cache TTL for proxied scripts (60--86400s) | -| `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | +| Field | Type | Required | Default | Description | +| ------------------------- | ------- | -------- | ------------------------------------------------------ | ----------------------------------------------------------------- | +| `enabled` | boolean | No | `true` | Enable/disable the integration | +| `gam_attribution_enabled` | boolean | No | `false` | Add fixed page-level `ts=true` targeting for GAM cohort reporting | +| `script_url` | string | No | `https://securepubads.g.doubleclick.net/tag/js/gpt.js` | URL for the GPT bootstrap script | +| `cache_ttl_seconds` | integer | No | `3600` | Cache TTL for proxied scripts (60--86400s) | +| `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | + +The environment override +`TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` works only when +`gam_attribution_enabled` is already present under `[integrations.gpt]` in the +TOML file. EdgeZero v0.0.4 cannot create a missing configuration leaf. ## Endpoints @@ -109,6 +116,65 @@ Takes over `googletag.cmd` so every queued callback is wrapped before GPT execut - Consent gating of ad requests - Ad-unit path rewriting for A/B testing +### GAM Treatment Attribution + +Setting `gam_attribution_enabled = true` adds the fixed page-level GPT targeting +value `ts=true`. It is applied before publisher GPT initialization and remains +for the browser document's lifetime, so initial, lazy, refresh, publisher-owned, +and SPA-route requests inherit it unless another targeting consumer clears or +overrides the key. The attribution switch is independently controlled and +defaults to `false`, but the GPT integration's `enabled` master switch must also +be `true`. + +This key is distinct from the existing slot-level `ts_initial=1` value. +`ts_initial` retains its current cleanup lifecycle; Trusted Server does not +clear the page-level `ts` value during Prebid refresh or SPA cleanup. + +For an eligible publisher document whose activation script was not cloned, +`ts=true` means Trusted Server emitted the rewritten document head before the +GPT request. It does not prove that the response body completed, that a Trusted +Server bid won, or that an impression was caused by treatment. A publisher can +copy the activation script with `srcdoc` or `document.write`; treat any marker +on an unrewritten nested document as contamination, not attribution proof. + +Before enabling attribution in a cohort: + +1. Complete privacy and CSP review, create the reportable predefined `true` + value in the target GAM network, and verify the chosen GAM reporting surface + and billing approval. +2. Audit the short `ts` key across publisher GPT code, effective Prebid + `bidderSettings[*].adserverTargeting` output (including + `setTargetingForGPTAsync`), the effective creative-opportunity targeting map, + and every GAM consumer that can affect eligibility, pricing, protection, or + routing. Trusted Server accepts and forwards operator targeting verbatim; it + does not reserve, filter, or intercept a slot-level `ts` key at runtime. +3. With treatment routing stopped, deploy attribution enabled and validate + initial, lazy, refresh, publisher-owned, and SPA requests. Confirm every + excluded path reports zero marked requests, then save a short paired-report + dry run that satisfies the invariants below before starting the cohort. + +For reporting, save one exact eligible universe: GAM network, inventory units, +routes, formats, time zone, date window, metrics, and all exclusions. Report A +is the nonduplicated total for that universe. Report B uses identical filters +and metrics plus exactly `ts=true`. If Enhanced Key-Value reporting is +unavailable, unapproved, or incompatible, use an exactly filtered legacy +key-value report and never sum its repeated key-value rows. Derive control as +`A - B`, and require `0 <= B <= A` for every metric. A violation invalidates the +whole report pair; never clamp a negative result. Use the same reporting-latency +and invalid-traffic maturation window for both reports. + +GAM results are descriptive delivery attribution, not a causal treatment +effect. Aggregate monitoring and synthetic/manual samples can detect obvious +failures but cannot prove marker completeness on every production request +without request-correlated telemetry. + +For a normal rollback, first stop and verify new treatment assignment at the +router, record a clean reporting boundary, and let already-open documents drain. +Exclude the drain interval, then set `gam_attribution_enabled = false` after +marked traffic reaches zero for the agreed interval. An emergency kill may flip +the setting immediately, but the affected interval and subsequent drain must be +treated as invalid for experiment reporting. + ## Use Cases ### First-Party Ad Delivery diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 19ecda4a5..38af8a1ec 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -104,6 +104,9 @@ rewrite_sdk = true [integrations.gpt] enabled = false +# Keep this leaf present when using the corresponding EdgeZero v0.0.4 +# environment override. Attribution remains off until explicitly enabled. +gam_attribution_enabled = false script_url = "https://ads.example.com/gpt.js" cache_ttl_seconds = 3600 rewrite_script = true From ef35343fff2b6938ee2a3fdd153f19dec792eaf0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 10:01:49 +0530 Subject: [PATCH 177/195] Use explicit GAM attribution branch --- crates/trusted-server-core/src/integrations/gpt.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 533944027..2c2c94ba7 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -491,11 +491,11 @@ impl IntegrationHeadInjector for GptIntegration { /// route changes (see `auction/endpoints.rs`). /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - let gam_attribution_flag = self - .config - .gam_attribution_enabled - .then_some("window.__tsjs_gam_attribution_enabled=true;") - .unwrap_or_default(); + let gam_attribution_flag = if self.config.gam_attribution_enabled { + "window.__tsjs_gam_attribution_enabled=true;" + } else { + "" + }; let mut scripts = vec![ format!( From 97e097d4108a40c9bad4bf6d36bbacb3c2ce295c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 15 Aug 2026 10:12:29 +0530 Subject: [PATCH 178/195] Align GAM rollback and failure coverage --- .../integrations/gpt/gpt_bootstrap.test.ts | 16 +++++++- .../2026-07-15-gam-ts-cohort-attribution.md | 16 ++++---- ...-07-15-gam-ts-cohort-attribution-design.md | 37 ++++++++++--------- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index 4ec4110b5..96f377933 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -174,8 +174,18 @@ describe('gpt_bootstrap.js fallback', () => { const setConfig = vi.fn(() => { throw new Error('publisher setConfig failed'); }); + const disableInitialLoad = vi.fn(); + const pubads = { + disableInitialLoad, + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; const publisherCommand = vi.fn(); - (window as TestWindow).googletag = makeGoogleTag({ cmd: queue, setConfig }); + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + setConfig, + pubads: vi.fn(() => pubads), + }); (window as TestWindow).__tsjs_gam_attribution_enabled = true; runBootstrap(); @@ -184,6 +194,10 @@ describe('gpt_bootstrap.js fallback', () => { expect(() => [...queue].forEach((command) => command())).not.toThrow(); expect(setConfig).toHaveBeenCalledWith({ targeting: { ts: 'true' } }); expect(publisherCommand).toHaveBeenCalledTimes(1); + expect(typeof (window as TestWindow).tsjs!.adInit).toBe('function'); + pubads.disableInitialLoad(); + expect(disableInitialLoad).toHaveBeenCalledTimes(1); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); }); it('still tracks the wrapped legacy disableInitialLoad path', () => { diff --git a/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md b/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md index d7763e046..eaf07ca51 100644 --- a/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md +++ b/docs/superpowers/plans/2026-07-15-gam-ts-cohort-attribution.md @@ -905,8 +905,9 @@ This task must not add buffering or adapter logic. - saved report pairs are exported after the same reporting-latency and invalid-traffic window, and monitoring/synthetic samples do not prove per-response marker completeness; - - normal rollback stops and verifies routing before flipping the setting, - then excludes the open-document drain interval; + - normal rollback stops and verifies routing, records the boundary, keeps + attribution enabled through the excluded open-document drain, and flips the + setting only after marked traffic reaches zero; - an emergency kill flips immediately and invalidates the affected/drain interval. @@ -1018,7 +1019,7 @@ This task must not add buffering or adapter logic. git log --oneline --decorate -8 ``` - Expected: a clean worktree, seven focused implementation commits, and no + Expected: a clean worktree, focused implementation commits, and no production changes to creative-opportunity filtering, Prebid interception, streaming buffering, or adapter behavior. @@ -1052,10 +1053,11 @@ remain separate from Tasks 1-8 because they require publisher/router/GAM access. `0 <= Report B <= Report A` for every selected metric. - [ ] Start the sticky treatment cohort only after every gate passes; use GAM share versus router allocation only as a diagnostic. -- [ ] For normal rollback, close the clean report boundary, stop and verify new - treatment routing, disable attribution, and exclude the marked-document - drain interval. For an emergency kill, disable immediately and invalidate - the affected plus drain intervals. +- [ ] For normal rollback, stop and verify new treatment routing, close the + clean report boundary, keep attribution enabled while marked documents + drain through the excluded interval, and disable only after marked traffic + reaches zero for the agreed interval. For an emergency kill, disable + immediately and invalidate the affected plus drain intervals. ## Definition of done diff --git a/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md index 5e9172d4c..38ffa7ef4 100644 --- a/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md +++ b/docs/superpowers/specs/2026-07-15-gam-ts-cohort-attribution-design.md @@ -986,19 +986,20 @@ deployment: diagnostic before interpreting descriptive delivery results. Rollback is ordered so newly routed treatment traffic cannot become unmarked -control. First record the last clean reporting boundary and stop new treatment -assignment/routing. Verify through router or access logs that routing stopped; -then set `gam_attribution_enabled = false` and deploy the kill switch. Fresh -Trusted Server documents must keep normal GPT behavior while omitting the -marker. Already-open documents—including long-lived SPA sessions and any marked -document restored from a cache—retain page-level targeting and may continue -issuing marked lazy or refreshed requests. Exclude the entire post-boundary -drain interval from both cohorts. The drain ends only after router/access logs -and GAM show no remaining `ts=true` traffic for one complete, runbook-defined -reporting interval and a fresh synthetic navigation confirms that new documents -are unmarked. If marked traffic persists, the interval remains excluded rather -than being inferred as control. Historical GAM rows remain valid, and the GAM -key may stay defined and reportable for historical analysis. +control. First stop new treatment assignment/routing and verify through router +or access logs that routing stopped, then record the last clean reporting +boundary. Keep `gam_attribution_enabled = true` while already-open +documents—including long-lived SPA sessions and any marked document restored +from a cache—drain; they retain page-level targeting and may continue issuing +marked lazy or refreshed requests. Exclude the entire post-boundary drain +interval from both cohorts. The drain ends only after router/access logs and GAM +show no remaining `ts=true` traffic for one complete, runbook-defined reporting +interval. Then set `gam_attribution_enabled = false`, deploy the kill switch, +and use a fresh synthetic Trusted Server navigation to confirm normal GPT +behavior while the marker is absent. If marked traffic persists, keep +attribution enabled and the interval excluded rather than inferring it as +control. Historical GAM rows remain valid, and the GAM key may stay defined and +reportable for historical analysis. If an active privacy, targeting-collision, or ad-delivery incident requires an immediate kill, deploy `gam_attribution_enabled = false` without waiting for @@ -1133,8 +1134,8 @@ baseline limitation and requires coverage checks. 13. CSP-compatible inline execution is proven before launch. A fallback-only page remains attributed to treatment but raises an incident and cannot be treated as a healthy experiment page. -14. Normal rollback stops and verifies new treatment routing before deploying - the attribution kill switch, and reporting excludes already-open or cached - marked documents until observed traffic drains according to the documented - boundary rule. An emergency kill invalidates the affected and drain windows - instead of treating newly unmarked traffic as control. +14. Normal rollback stops and verifies new treatment routing, records the clean + boundary, and keeps attribution enabled until already-open or cached marked + documents drain according to the documented rule. Only then is the kill + switch deployed. An emergency kill invalidates the affected and drain + windows instead of treating newly unmarked traffic as control. From 54203900d8fb7da9338a39c5e47822d55aba6219 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:59:57 -0700 Subject: [PATCH 179/195] Sync EdgeZero to latest deploy-actions tip and adopt config gc Re-resolve the six edgezero-* deps from bb441162 to 908e229a (current tip of feature/edgezero-deploy-actions, PR #316), and adapt the ts CLI to its surface changes: - Wire the new `ts config gc` subcommand (reclaims orphaned config-store chunk entries) to edgezero_cli::run_config_gc, with parse coverage for the preview default, destructive --yes/--older-than sweep, and the --dry-run/--yes conflict. - Lock the hardened deploy staging behavior: --stage was renamed to --staging and deploy passthrough is now last=true, so a stray --stage fails closed at parse time instead of routing a staging-intended deploy to production. Add tests for the rejection and for post---- passthrough capture. --- Cargo.lock | 33 +++++++---- crates/trusted-server-cli/src/run.rs | 88 +++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5caa1a7a..32e0bd696 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-trait", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-stream", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-trait", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "anyhow", "async-compression", @@ -1569,14 +1569,14 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" dependencies = [ "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 3.0.3", "toml", "validator", ] @@ -4769,6 +4769,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5908,7 +5919,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 7374c56a7..a395281be 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -2,8 +2,8 @@ use std::process; use clap::{Parser, Subcommand}; use edgezero_cli::args::{ - ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, - DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, + ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, + ConfigValidateArgs, DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; use trusted_server_core::config::TrustedServerAppConfig; @@ -55,6 +55,8 @@ enum ConfigCommand { Init(ConfigInitArgs), /// Diff `trusted-server.toml` against the live `EdgeZero` config. Diff(ConfigDiffArgs), + /// Reclaim orphaned chunk entries leaked from prior oversized pushes. + Gc(ConfigGcArgs), /// Push `trusted-server.toml` as a blob envelope through `EdgeZero`. Push(ConfigPushArgs), /// Validate `edgezero.toml` and the typed Trusted Server config. @@ -102,6 +104,7 @@ fn dispatch(args: Args) -> Result<(), String> { Err(err) => Err(err), } } + Command::Config(ConfigCommand::Gc(args)) => edgezero_cli::run_config_gc(&args), Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) } @@ -299,6 +302,39 @@ mod tests { assert!(deploy.staging); } + #[test] + fn deploy_rejects_renamed_stage_flag_before_separator() { + // `--stage` was renamed to `--staging`, and adapter passthrough is + // `last = true` (only captured after `--`). A stray `--stage` before the + // separator must fail closed at parse time rather than being swallowed as + // passthrough, which would leave `staging` false and route a + // staging-intended deploy to production. + Args::try_parse_from(["ts", "deploy", "--adapter", "fastly", "--stage"]) + .expect_err("should reject the renamed-away --stage flag, not route it to production"); + } + + #[test] + fn deploy_captures_adapter_passthrough_after_separator() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--", + "--comment", + "ci", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); + }; + assert!(!deploy.staging, "should default to a production deploy"); + assert_eq!( + deploy.adapter_args, + vec!["--comment", "ci"], + "should capture args after -- as adapter passthrough" + ); + } + #[test] fn parses_audit_with_default_outputs() { let args = parse(&["ts", "audit", "https://publisher.example"]); @@ -438,6 +474,54 @@ mod tests { assert!(!diff.no_env); } + #[test] + fn config_gc_previews_by_default() { + let args = parse(&["ts", "config", "gc", "--adapter", "fastly"]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert_eq!(gc.adapter, "fastly"); + assert_eq!( + gc.older_than, None, + "should not require an older-than window to preview" + ); + assert!(!gc.dry_run); + assert!(!gc.no_env); + } + + #[test] + fn config_gc_parses_destructive_sweep() { + let args = parse(&[ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--yes", + "--older-than", + "7d", + ]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert!(gc.yes); + assert_eq!(gc.older_than, Some("7d".to_owned())); + } + + #[test] + fn config_gc_rejects_dry_run_with_yes() { + Args::try_parse_from([ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--dry-run", + "--yes", + ]) + .expect_err("should reject conflicting --dry-run and --yes"); + } + #[test] fn config_validate_uses_edgezero_app_config_flag() { let args = parse(&[ From fe5767e61aa8cdbeb17cb1b2b5e59a00a03640d1 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Sun, 16 Aug 2026 00:14:38 -0700 Subject: [PATCH 180/195] Sync EdgeZero to deploy-actions tip 5f3d648c Re-resolve the six edgezero-* deps from 908e229a to 5f3d648c (current tip of feature/edgezero-deploy-actions, PR #316). The upstream change is an internal review-addressing pass (redact config-store errors, fix version parse, log cleanup, docs) confined to the Fastly adapter CLI; no ts CLI surface change, so no run.rs adaptation is needed. --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ea08f4cc..5e3388bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-stream", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-trait", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#908e229a424e837717450c36c656d73891466d08" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#5f3d648c3c6c38fc6e6b22b5c65c66177363aad8" dependencies = [ "log", "proc-macro2", @@ -5920,7 +5920,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From 31de3f63298c15b6875f5e6753a7ead13879e24c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 10:22:55 +0530 Subject: [PATCH 181/195] Fix ad-template CLI section rendering and harden the audit commands The CLI still called `resolved_gam_unit_path`, which core replaced with the path-aware `render_gam_unit_path` when `{section}` templating landed, so the crate no longer compiled. Both call sites now derive the section through `CreativeOpportunitiesConfig::section_for_path` and render the template, and `ExpectedSlot`/`ConfiguredJson` carry an optional unit path so an over-limit dynamic render is reported rather than silently matched against the wrong unit. Also resolves the outstanding review findings on these paths: - Write the operator config through a same-directory temp file, fsync, and rename, so a failed write cannot truncate `trusted-server.toml`. - Validate TLS certificates in both audit browser sessions; opting out now requires `--danger-accept-invalid-certs`. - Refuse a redirect that leaves the requested origin during verify unless `--allow-cross-origin-redirect` is passed, so another origin's evidence cannot satisfy `--strict`. - Reject page patterns the runtime cannot compile before they reach the file, through a new shared `compile_page_pattern` in core. - Reject `creative_opportunities` declared in a form the line-based splice cannot edit, instead of appending a duplicate table. - Drop non-integer GPT sizes in the collector so one fluid size cannot fail deserialization of the whole evidence payload. - Escape control characters in page-controlled text written to the terminal. --- .../src/ad_templates/compare.rs | 54 ++++- .../src/ad_templates/expected.rs | 86 +++++++- .../src/ad_templates/output.rs | 77 ++++++- .../commands/audit/ad_template_collector.js | 34 ++- .../src/commands/audit/ad_templates.rs | 195 +++++++++++++++-- .../src/commands/audit/browser.rs | 31 ++- .../src/commands/audit/collector.rs | 9 + .../audit/generate/browser_collector.rs | 5 + .../src/commands/audit/generate/mod.rs | 203 +++++++++++++++++- .../src/commands/audit/generate/slot_toml.rs | 141 +++++++++++- .../src/commands/audit/mod.rs | 7 + .../src/commands/audit/page.rs | 13 +- .../src/commands/config/ad_templates.rs | 35 ++- .../src/creative_opportunities.rs | 71 +++--- docs/guide/cli.md | 21 ++ 15 files changed, 903 insertions(+), 79 deletions(-) diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs index 18e47fb0b..215a6a300 100644 --- a/crates/trusted-server-cli/src/ad_templates/compare.rs +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -251,13 +251,27 @@ pub fn compare_page_evidence( for slot in expected { let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); let resolved_id = resolved.map(|dom| dom.dom_id.clone()); - let gpt_idx = evidence.gpt_slots.iter().position(|gpt| { - gpt.gam_unit_path == slot.gam_unit_path - && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) }); let banner = banner_sizes(slot); let mut warnings = Vec::new(); + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime rejects this config", + slot.id + ), + )); + } let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { consumed_gpt[idx] = true; @@ -434,7 +448,7 @@ mod tests { ExpectedSlot { id: id.to_string(), div_id: div_id.to_string(), - gam_unit_path: gam_unit_path.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), formats: sizes .iter() .map(|&(width, height)| ExpectedFormat { @@ -453,7 +467,7 @@ mod tests { ExpectedSlot { id: id.to_string(), div_id: div_id.to_string(), - gam_unit_path: gam_unit_path.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), formats: vec![ExpectedFormat { width: 0, height: 0, @@ -487,6 +501,36 @@ mod tests { ); } + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + #[test] fn dom_only_is_partial() { let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs index dd8e1055d..549ec0a57 100644 --- a/crates/trusted-server-cli/src/ad_templates/expected.rs +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -24,8 +24,14 @@ pub struct ExpectedSlot { pub id: String, /// Resolved HTML `div` element ID (override or the slot id). pub div_id: String, - /// Resolved GAM unit path (override or `//`). - pub gam_unit_path: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` when a dynamic template renders beyond GAM's unit-path byte limit + /// for this path's section. Runtime validation rejects such a config, so + /// this only occurs for a config that would fail to load; the slot is then + /// reported unconfirmable rather than matched against a wrong path. + pub gam_unit_path: Option, /// Configured ad formats. pub formats: Vec, /// Configured provider names, in `aps`, `prebid` order. @@ -53,16 +59,22 @@ pub struct ExpectedFormat { /// Uses [`match_slots`] so glob semantics stay identical to the runtime, and /// preserves configured slot order. `path` is assumed already normalized via /// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. // Shared projection used by the audit verifier; the static commands match slots // directly against the runtime matcher. #[must_use] pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); let slots = match_slots(&config.slot, path) .into_iter() .map(|slot| ExpectedSlot { id: slot.id.clone(), div_id: slot.resolved_div_id().to_string(), - gam_unit_path: slot.resolved_gam_unit_path(&config.gam_network_id), + gam_unit_path: slot.render_gam_unit_path(&config.gam_network_id, §ion), formats: slot .formats .iter() @@ -182,7 +194,10 @@ mod tests { ["atf"] ); assert_eq!(expected.slots[0].div_id, "ad-atf-"); - assert_eq!(expected.slots[0].gam_unit_path, "/123/news/atf"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); assert_eq!(expected.slots[0].providers, ["prebid"]); assert_eq!( expected.slots[0].formats, @@ -208,10 +223,71 @@ mod tests { let expected = expected_slots_for_path("/", &config); assert_eq!(expected.slots[0].div_id, "footer"); - assert_eq!(expected.slots[0].gam_unit_path, "/42/footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); assert!(expected.slots[0].providers.is_empty()); } + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_report_unrenderable_dynamic_template_as_none() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. `validate_runtime` rejects this config, so the verifier reports + // the slot as unconfirmable rather than matching a truncated path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert_eq!( + expected.slots[0].gam_unit_path, None, + "an over-limit dynamic render should project as None" + ); + } + #[test] fn normalize_path_or_url_strips_query_and_fragment() { assert_eq!( diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs index 51658d04c..9a1190652 100644 --- a/crates/trusted-server-cli/src/ad_templates/output.rs +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -14,10 +14,44 @@ reason = "wire model assembled by the audit verifier in a later task" )] +use std::borrow::Cow; + use serde::{Deserialize, Serialize}; use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 || (0x7f..=0x9f).contains(&code) +} + /// Confirmation status for a single configured slot. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -175,8 +209,9 @@ pub struct SlotJson { pub struct ConfiguredJson { /// Resolved div element ID. pub div_id: String, - /// Resolved GAM unit path. - pub gam_unit_path: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, /// Configured formats. pub formats: Vec, /// Configured provider names. @@ -260,7 +295,7 @@ impl VerificationReport { phase: EvidencePhaseJson::InitialLoad, configured: ConfiguredJson { div_id: "ad-atf-".to_string(), - gam_unit_path: "/123/news/atf".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), formats: vec![FormatJson { width: 300, height: 250, @@ -324,6 +359,42 @@ impl VerificationReport { mod tests { use super::*; + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + } + #[test] fn verification_json_contains_gate_state_and_extra_evidence() { let result = VerificationReport::example_confirmed_with_extra_evidence(); diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js index c01074376..1133d46b6 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -25,6 +25,16 @@ function __ts_push(list, entry) { if (list.length < __ts_max_entries) list.push(entry) } +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0) return null + return [width, height] +} + function __ts_normalize_sizes(sizes) { const out = [] if (!Array.isArray(sizes)) return out @@ -32,8 +42,9 @@ function __ts_normalize_sizes(sizes) { const pairs = typeof sizes[0] === "number" ? [sizes] : sizes for (const size of pairs) { if (out.length >= __ts_max_entries) break - if (Array.isArray(size) && typeof size[0] === "number" && typeof size[1] === "number") { - out.push([size[0], size[1]]) + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) } else { __ts_push(__ts_ev.warnings, { code: "fluid_size_ignored", @@ -148,10 +159,21 @@ window.__tsCollectAdTemplateEvidence = function () { const sizes = [] for (const size of rawSizes) { if (sizes.length >= __ts_max_entries) break - if (size && typeof size.getWidth === "function") { - sizes.push([size.getWidth(), size.getHeight()]) - } else if (Array.isArray(size) && typeof size[0] === "number") { - sizes.push([size[0], size[1]]) + let pair = null + if (size && typeof size.getWidth === "function" && typeof size.getHeight === "function") { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + __ts_push(__ts_ev.warnings, { + code: "fluid_size_ignored", + message: "non-numeric GPT size ignored", + }) } } const exists = __ts_ev.gpt_slots.some( diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs index 8fd72b58c..8250e1cde 100644 --- a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -20,7 +20,7 @@ use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, norma use crate::ad_templates::output::{ ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, - VerificationReport, Warning, + VerificationReport, Warning, escape_terminal_text, }; use crate::commands::audit::AuditAdTemplatesVerifyArgs; use crate::commands::audit::collector::{ @@ -41,8 +41,11 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String loaded.settings.creative_opportunities.as_ref(), loaded.settings.auction.enabled, &args.urls, - args.strict, - args.scroll, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, &args.cookies, ); @@ -61,6 +64,17 @@ pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result<(), String } } +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + /// Builds the verification report for `urls` using `collector`. /// /// `creative` is the effective `[creative_opportunities]` config (if any) and @@ -70,8 +84,7 @@ fn build_report( creative: Option<&CreativeOpportunitiesConfig>, auction_enabled: bool, urls: &[url::Url], - strict: bool, - scroll: bool, + options: VerifyOptions, cookies: &[(String, String)], ) -> VerificationReport { let init_script = build_init_script(creative); @@ -84,7 +97,7 @@ fn build_report( let request = BrowserCollectRequest { url: url.clone(), init_scripts: init_script.clone().into_iter().collect(), - scroll, + scroll: options.scroll, collect_ad_evidence: true, cookies: cookies.to_vec(), }; @@ -94,9 +107,20 @@ fn build_report( any_error = true; pages.push(error_page(url, &message)); } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } Ok(collected) => { let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); - if strict && strict_failed { + if options.strict && strict_failed { any_strict_fail = true; } pages.push(page); @@ -104,15 +128,20 @@ fn build_report( } } - let ok = !(any_error || (strict && any_strict_fail)); + let ok = !(any_error || (options.strict && any_strict_fail)); VerificationReport { ok, - strict, + strict: options.strict, pages, warnings: Vec::new(), } } +/// Whether navigation left the requested URL's origin (scheme, host, or port). +fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + requested.origin() != final_url.origin() +} + /// Builds the read-only collector init script from the configured slots. fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Option { let config = AdTemplateCollectorConfig { @@ -227,6 +256,37 @@ fn error_page(requested: &url::Url, message: &str) -> PageJson { } } +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + fn empty_evidence() -> BrowserAdEvidence { BrowserAdEvidence { dom_ids: Vec::new(), @@ -325,10 +385,29 @@ fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), St } fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + for page in &report.pages { writeln!(out, "url: {}", page.url).map_err(write_err)?; if let Some(error) = &page.error { - writeln!(out, " error [{}]: {}", error.code, error.message).map_err(write_err)?; + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; continue; } if let Some(path) = &page.path { @@ -338,13 +417,11 @@ fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), S writeln!(out, " slot {}: {}", slot.id, status_label(slot.status)) .map_err(write_err)?; for warning in &slot.warnings { - writeln!(out, " warning [{}]: {}", warning.code, warning.message) - .map_err(write_err)?; + write_warning(out, " ", warning)?; } } for warning in &page.warnings { - writeln!(out, " warning [{}]: {}", warning.code, warning.message) - .map_err(write_err)?; + write_warning(out, " ", warning)?; } } writeln!(out, "ok: {}", report.ok).map_err(write_err) @@ -449,6 +526,24 @@ mod tests { auction_enabled: bool, strict: bool, urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, ) -> VerificationReport { let config = news_config(); let parsed: Vec = urls @@ -460,8 +555,7 @@ mod tests { Some(&config), auction_enabled, &parsed, - strict, - false, + options, &[], ) } @@ -487,6 +581,75 @@ mod tests { ); } + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + #[test] fn confirmed_page_is_ok_in_default_mode() { let collector = FakeCollector::page( diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs index a67591b59..ab998adec 100644 --- a/crates/trusted-server-cli/src/commands/audit/browser.rs +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -61,6 +61,8 @@ pub struct BrowserCollector { settle_quiet: Duration, /// Hard cap on settling. settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, } impl Default for BrowserCollector { @@ -77,6 +79,7 @@ impl BrowserCollector { chrome: None, settle_quiet: Duration::from_millis(DEFAULT_SETTLE_QUIET_MS), settle_max: Duration::from_millis(DEFAULT_SETTLE_MAX_MS), + accept_invalid_certs: false, } } @@ -87,6 +90,7 @@ impl BrowserCollector { chrome: opts.chrome.clone(), settle_quiet: Duration::from_millis(opts.settle_quiet_ms), settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, } } } @@ -206,8 +210,17 @@ impl AuditCollector for BrowserCollector { // so audit output stays clean, then restore the prior threshold. let previous_level = log::max_level(); log::set_max_level(log::LevelFilter::Error); - let result = runtime - .block_on(async move { collect(&chrome, profile.path(), request, settle).await }); + let accept_invalid_certs = self.accept_invalid_certs; + let result = runtime.block_on(async move { + collect( + &chrome, + profile.path(), + request, + settle, + accept_invalid_certs, + ) + .await + }); log::set_max_level(previous_level); result } @@ -219,10 +232,20 @@ async fn collect( profile_dir: &std::path::Path, request: BrowserCollectRequest, settle_config: SettleConfig, + accept_invalid_certs: bool, ) -> Result { - let config = BrowserConfig::builder() + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let mut builder = BrowserConfig::builder() .chrome_executable(chrome) - .user_data_dir(profile_dir) + .user_data_dir(profile_dir); + if !accept_invalid_certs { + builder = builder.respect_https_errors(); + } + let config = builder .build() .map_err(|error| format!("failed to build browser config: {error}"))?; diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index aca6814ca..ff7a45867 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -28,6 +28,15 @@ pub struct BrowserOpts { /// Hard cap in milliseconds on waiting for the page to settle. #[arg(long, default_value_t = 10_000)] pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, } /// A request to collect a single page. diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index b1446933c..077c5550a 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -63,10 +63,15 @@ async fn collect_page_via_browser_async( "failed to create temporary browser profile for audit: {error}" )) })?; + // chromiumoxide ignores TLS errors by default. `generate` sends operator + // cookies and writes what it scrapes into the operator's config, so a + // certificate-invalid impersonator could both harvest the session and seed + // the config with slots of its choosing. Validate certificates. let config = BrowserConfig::builder() .chrome_executable(chrome_executable) .user_data_dir(user_data_dir.path()) .new_headless_mode() + .respect_https_errors() .build() .map_err(|error| { report_error(format!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 43a8e7f8f..1feec9329 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -10,7 +10,9 @@ use std::io::Write; use std::path::{Path, PathBuf}; use serde::Serialize; -use trusted_server_core::creative_opportunities::CreativeOpportunitiesConfig; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, compile_page_pattern, +}; use url::Url; use crate::commands::audit::generate::collector::AuditCollector; @@ -23,6 +25,45 @@ use crate::error::{CliResult, cli_error, report_error}; use analyzer::{analyze_collected_page, extract_gtm_container_id}; +/// Writes `contents` to `path` atomically: a same-directory temp file is +/// written and fsynced, then renamed over the target, then the directory entry +/// is fsynced. +/// +/// A plain `fs::write` truncates the destination before writing, so a full disk +/// or an interrupted run would leave an operator's `trusted-server.toml` empty +/// or half-written. `rename` within a directory is atomic, so a reader sees +/// either the old file or the complete new one. +/// +/// The target's existing permissions are carried onto the replacement, since +/// the temp file is created 0600 and the config may intentionally be broader. +/// +/// # Errors +/// +/// Returns the underlying I/O error when the temp file cannot be created, +/// written, synced, or renamed over `path`. +fn write_file_atomically(path: &Path, contents: &str) -> std::io::Result<()> { + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + let mut temp = tempfile::Builder::new() + .prefix(".ts-audit-") + .tempfile_in(directory)?; + temp.write_all(contents.as_bytes())?; + temp.as_file().sync_all()?; + if let Ok(metadata) = fs::metadata(path) { + temp.as_file().set_permissions(metadata.permissions())?; + } + temp.persist(path).map_err(|error| error.error)?; + + // Best-effort durability for the rename itself. Opening a directory handle + // is not portable (Windows rejects it), and the content is already safely + // on disk either way, so a failure here is not worth failing the command. + let _ = fs::File::open(directory).and_then(|handle| handle.sync_all()); + Ok(()) +} + /// Arguments for `ts audit generate ` — bootstraps draft Trusted Server /// config and JavaScript asset audit files from a live page (issue #800). #[derive(Debug, clap::Args)] @@ -229,7 +270,7 @@ fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliRes let mut written_paths = Vec::new(); if let Some(path) = &plan.js_assets_path { - fs::write(path, &outputs.js_assets_toml).map_err(|error| { + write_file_atomically(path, &outputs.js_assets_toml).map_err(|error| { report_error(format!( "failed to write JS asset audit {}: {error}", path.display() @@ -238,7 +279,7 @@ fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliRes written_paths.push(path.display().to_string()); } if let Some(path) = &plan.config_path { - fs::write(path, &outputs.draft_config_toml).map_err(|error| { + write_file_atomically(path, &outputs.draft_config_toml).map_err(|error| { report_error(format!( "failed to write draft config {}: {error}", path.display() @@ -488,6 +529,11 @@ pub(crate) fn run_update_slots( } else { page_patterns.to_vec() }; + // Reject a pattern the runtime cannot compile before it reaches the file: + // a persisted invalid glob either fails the next config load or is silently + // dropped at pattern-compile time, leaving the slot matching fewer pages + // than the config claims. + validate_page_patterns(&run_patterns)?; let merged = merge_slots(existing_creative, &discovered, &run_patterns, replace); let network_id = resolve_network_id( @@ -503,7 +549,7 @@ pub(crate) fn run_update_slots( .map_err(|error| report_error(format!("failed to write preview: {error}")))?; return Ok(()); } - fs::write(config_path, &updated).map_err(|error| { + write_file_atomically(config_path, &updated).map_err(|error| { report_error(format!( "failed to write config {}: {error}", config_path.display() @@ -518,6 +564,30 @@ pub(crate) fn run_update_slots( ) .map_err(|error| report_error(format!("failed to write command output: {error}"))) } +/// Rejects any page pattern the runtime's glob compiler would not accept. +/// +/// Uses [`compile_page_pattern`] so the accepted set is exactly what +/// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the +/// `**`→`*` normalisation. All patterns are reported at once so an operator +/// passing several `--page-pattern` values fixes them in one pass. +/// +/// # Errors +/// +/// Returns a user-facing error listing every pattern that does not compile. +fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { + let invalid: Vec = patterns + .iter() + .filter_map(|pattern| compile_page_pattern(pattern).err()) + .collect(); + if invalid.is_empty() { + return Ok(()); + } + cli_error(format!( + "refusing to write invalid page pattern(s): {}", + invalid.join("; ") + )) +} + /// The default page pattern for a scraped URL: its path, or `/` for the root. fn default_page_pattern(target_url: &Url) -> String { let path = target_url.path(); @@ -591,6 +661,19 @@ mod tests { } } + /// A collected page carrying one discoverable GPT slot, for `run_update_slots`. + fn collected_page_with_header_slot() -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + collected + } + fn audit_args(url: &str) -> GenerateArgs { GenerateArgs { url: url.to_string(), @@ -955,6 +1038,118 @@ mod tests { ); } + #[test] + fn update_slots_rejects_invalid_page_pattern_without_touching_config() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + let error = run_update_slots( + "https://publisher.example/", + &config_path, + None, + &["[".to_string()], + false, + &[], + false, + &collector, + &mut out, + ) + .expect_err("should reject an invalid glob"); + + assert!( + format!("{error:?}").contains("page pattern '['"), + "error should name the offending pattern, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a rejected pattern must leave the operator config untouched" + ); + } + + #[test] + fn update_slots_accepts_double_star_pattern_like_the_runtime() { + // `/20**` does not compile directly but the runtime normalises it to + // `/20*`; validation must accept exactly what the runtime accepts. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &["/20**".to_string()], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should accept a runtime-normalisable pattern"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/20**") + ); + } + + #[test] + fn update_slots_write_replaces_the_config_without_leaving_temp_files() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &[], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should update slots"); + + let entries: Vec = fs::read_dir(temp.path()) + .expect("should read temp dir") + .map(|entry| { + entry + .expect("should read entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entries, + ["trusted-server.toml"], + "the atomic write should leave no stray temp file behind" + ); + let written = fs::read_to_string(&config_path).expect("should read config"); + toml::from_str::(&written).expect("rewritten config is valid TOML"); + } + #[test] fn update_slots_dry_run_does_not_persist_environment_overlay_config() { let temp = TempDir::new().expect("should create temp dir"); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 63f81b50c..580fe39ad 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -345,11 +345,26 @@ pub(super) fn splice_creative_slots( let rendered = rendered_slots.trim_matches('\n'); let existing = remove_inline_slot_value(existing)?; - // No section yet — append a fresh one with the network id and slots. - if !existing + // Presence is decided structurally (toml_edit), but the splice below is a + // line edit that only recognises a canonical `[creative_opportunities]` + // header. Reconciling the two here keeps a valid-but-unrecognised form — + // a quoted `["creative_opportunities"]` header, a top-level + // `creative_opportunities = { ... }` inline table, or a section implied + // only by its subtables — from being treated as absent and getting a + // duplicate table appended, which would produce invalid TOML. + let has_canonical_header = existing .lines() - .any(|line| is_table_header(line, "[creative_opportunities]")) - { + .any(|line| is_table_header(line, "[creative_opportunities]")); + if section_is_present(&existing)? && !has_canonical_header { + return cli_error( + "target config declares `creative_opportunities` in a form this updater cannot \ + edit safely; rewrite it as a `[creative_opportunities]` table (with \ + `[[creative_opportunities.slot]]` entries) and re-run", + ); + } + + // No section yet — append a fresh one with the network id and slots. + if !has_canonical_header { let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); @@ -445,6 +460,22 @@ pub(super) fn splice_creative_slots( Ok(result) } +/// Whether `document` declares `creative_opportunities` at all, in any valid +/// TOML representation (canonical table, quoted header, inline table, or a +/// section implied only by its subtables). +/// +/// # Errors +/// +/// Returns an error when the document does not parse as TOML. +fn section_is_present(document: &str) -> CliResult { + let parsed = document.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + Ok(parsed.get("creative_opportunities").is_some()) +} + /// Removes a scalar `creative_opportunities.slot` value so it can be replaced /// with the generated array-of-tables representation. fn remove_inline_slot_value(document: &str) -> CliResult { @@ -637,6 +668,108 @@ mod tests { toml::from_str::(&out).expect("spliced config is valid TOML"); } + #[test] + fn splice_rejects_quoted_section_header_instead_of_duplicating_it() { + // A quoted header is valid TOML but the line-based splice does not + // recognise it; appending a second `[creative_opportunities]` would + // produce a document that no longer parses. + let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; + + let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect_err("should refuse an unrecognised section form"); + + assert!( + format!("{error:?}").contains("cannot edit safely"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + #[test] + fn splice_rejects_top_level_inline_creative_opportunities_table() { + let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; + + let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect_err("should refuse a top-level inline table"); + + assert!( + format!("{error:?}").contains("cannot edit safely"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + #[test] + fn splice_appends_section_when_config_has_none() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("appended config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222") + ); + } + + #[test] + fn splice_preserves_section_scalars_and_provider_subtables() { + // Mirrors the templated operator shape: section policy scalars in the + // head block and a per-slot prebid provider subtable. + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + auction_timeout_ms = 2000\n\ + section_root = \"homepage\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-header-0\"\n\ + div_id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {}\n\n\ + [auction]\nenabled = true\n"; + let existing_config = existing_config( + &existing + .replace("[creative_opportunities]\n", "") + .replace("[[creative_opportunities.slot]]", "[[slot]]") + .replace("[creative_opportunities.slot.", "[slot.") + .replace("\n[auction]\nenabled = true\n", ""), + ); + let discovered = discovered_header_slot(); + let merged = merge_slots( + Some(&existing_config), + &discovered, + &["/news/*".to_string()], + false, + ); + + let out = splice_creative_slots(existing, Some("111"), &render_slots(&merged)) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "section policy scalars must survive the splice" + ); + assert_eq!(creative["auction_timeout_ms"].as_integer(), Some(2000)); + assert_eq!( + creative["slot"][0]["gam_unit_path"].as_str(), + Some("/{network_id}/example/{section}"), + "an existing templated unit path must not be rewritten to a literal" + ); + assert!( + creative["slot"][0]["providers"]["prebid"]["bidders"].is_table(), + "the prebid provider subtable must be re-emitted" + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "trailing sections must be preserved" + ); + } + #[test] fn splice_preserves_crlf_line_endings() { let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 1a33b890a..e024eb8b1 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -156,6 +156,13 @@ pub(crate) struct AuditAdTemplatesVerifyArgs { /// Perform a deterministic scroll pass after the initial settle. #[arg(long)] pub scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + /// + /// Off by default: slots are matched on the post-redirect path, so an + /// off-origin page could otherwise satisfy `--strict`. Enable only for a + /// known redirect between your own properties (e.g. apex to `www`). + #[arg(long)] + pub allow_cross_origin_redirect: bool, /// Cookie to send with each page request, as `name=value`. Repeatable. /// Use to carry an existing session (e.g. a valid bot-protection clearance /// cookie) so the origin serves the real page instead of a challenge. diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs index af0144fe9..6df54032d 100644 --- a/crates/trusted-server-cli/src/commands/audit/page.rs +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -4,6 +4,7 @@ use std::io::{self, Write}; use clap::Args; +use crate::ad_templates::output::escape_terminal_text; use crate::commands::audit::browser::BrowserCollector; use crate::commands::audit::collector::{ AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, @@ -57,11 +58,19 @@ fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> R let to_err = |error: io::Error| format!("failed to write command output: {error}"); writeln!(out, "url: {url}").map_err(to_err)?; writeln!(out, "final url: {}", page.final_url).map_err(to_err)?; - writeln!(out, "title: {}", page.title).map_err(to_err)?; + // The title and collector warning messages are page-controlled, so escape + // control characters before they reach the operator's terminal. + writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; for warning in &page.warnings { - writeln!(out, "warning [{}]: {}", warning.code, warning.message).map_err(to_err)?; + writeln!( + out, + "warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(to_err)?; } Ok(()) } diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index 60deda068..4216db382 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -195,7 +195,14 @@ fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), Str }; let matched = match_slots(&config.slot, &path); - write_match_result(out, &path, &matched, &config.gam_network_id, args.details) + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + args.details, + ) } fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { @@ -258,7 +265,14 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), }; let matched = match_slots(&config.slot, &path); - write_match_result(out, &path, &matched, &config.gam_network_id, true)?; + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; let method_pass = args.method.eq_ignore_ascii_case("GET"); let navigation_pass = !args.non_navigation; @@ -314,6 +328,7 @@ fn write_match_result( path: &str, matched: &[&CreativeOpportunitySlot], gam_network_id: &str, + section: &str, details: bool, ) -> Result<(), String> { if matched.is_empty() { @@ -330,7 +345,8 @@ fn write_match_result( if details { for slot in matched { - writeln!(out, "- {}", format_slot(slot, gam_network_id)).map_err(output_error)?; + writeln!(out, "- {}", format_slot(slot, gam_network_id, section)) + .map_err(output_error)?; } } @@ -341,7 +357,11 @@ fn write_gate(out: &mut dyn Write, label: &str, pass: bool) -> Result<(), String writeln!(out, "gate {label}: {}", if pass { "pass" } else { "block" }).map_err(output_error) } -fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str) -> String { +/// Formats one matched slot for `--details` output. +/// +/// `section` is the value the runtime derives from the evaluated path, so a +/// `{section}` template renders the same unit path the live request would use. +fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &str) -> String { let formats = slot .formats .iter() @@ -349,11 +369,16 @@ fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str) -> String { .collect::>() .join(", "); let providers = format_providers(slot); + // `None` means a dynamic template renders past GAM's unit-path byte limit — + // a config the runtime rejects, so surface it rather than printing a path. + let gam_unit_path = slot + .render_gam_unit_path(gam_network_id, section) + .unwrap_or_else(|| "".to_string()); format!( "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", slot.id, slot.resolved_div_id(), - slot.resolved_gam_unit_path(gam_network_id), + gam_unit_path, slot.page_patterns.join(", "), formats, providers, diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 594dbb6f9..f63f92d8e 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -531,15 +531,7 @@ impl CreativeOpportunitySlot { // skip `compile_patterns`). Re-compiles on every call. self.page_patterns .iter() - .any(|pattern| match Pattern::new(pattern) { - Ok(p) => p.matches(path), - Err(_) => { - let normalised = pattern.replace("**", "*"); - Pattern::new(&normalised) - .map(|p| p.matches(path)) - .unwrap_or(false) - } - }) + .any(|pattern| compile_page_pattern(pattern).is_ok_and(|p| p.matches(path))) } /// Compile [`page_patterns`](Self::page_patterns) into the @@ -556,22 +548,20 @@ impl CreativeOpportunitySlot { self.compiled_patterns = self .page_patterns .iter() - .filter_map(|pattern| { - match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { - Ok(compiled) => Some(compiled), - Err(_) => { - // Build-time validation only requires *one* valid pattern - // per slot, so a mixed valid/invalid set passes the build - // with the bad pattern silently dropped here. Warn so the - // operator can see the slot matches fewer pages than - // configured. - log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", - self.id, - pattern - ); - None - } + .filter_map(|pattern| match compile_page_pattern(pattern) { + Ok(compiled) => Some(compiled), + Err(_) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", + self.id, + pattern + ); + None } }) .collect(); @@ -834,6 +824,37 @@ pub struct PrebidSlotParams { pub bidders: HashMap, } +/// Compiles a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This is the single definition of what the runtime accepts as a page glob: +/// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that +/// [`CreativeOpportunitySlot::compile_patterns`] and +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. Tooling that +/// writes patterns into operator config validates them through this function so +/// it cannot persist a pattern the runtime would silently drop. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// normalisation. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::creative_opportunities::compile_page_pattern; +/// +/// assert!(compile_page_pattern("/news/*").is_ok()); +/// // `**` in a position the glob crate rejects is normalised to `*`. +/// assert!(compile_page_pattern("/20**").is_ok()); +/// assert!(compile_page_pattern("[").is_err()); +/// ``` +pub fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + /// Validates that a slot ID contains only safe characters. /// /// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). diff --git a/docs/guide/cli.md b/docs/guide/cli.md index bd1157937..e0baac367 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -138,6 +138,27 @@ ts audit generate https://publisher.example --force The legacy `ts audit ` form remains a compatibility alias for artifact generation. New automation should use `ts audit generate `. +### Audit safety defaults + +Every `ts audit` browser session validates TLS certificates. This matters +because `--cookie` sends a real session to the origin and the page's own +response becomes the audit's evidence, so a certificate-invalid host could both +harvest the session and fabricate what the audit reports. Override only for a +host you control with a known self-signed certificate: + +```bash +ts audit page https://staging.publisher.example --danger-accept-invalid-certs +``` + +`ts audit ad-templates verify` matches configured slots against the +**post-redirect** path, so it refuses a redirect that leaves the requested +origin rather than accepting another site's evidence as verification. Allow it +for a known redirect between your own properties (for example apex to `www`): + +```bash +ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-redirect +``` + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. From 07b37a1f0a2a4436437b57bf5b634dd3cf330aa7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 10:47:37 +0530 Subject: [PATCH 182/195] Verify generated ad-template config before it replaces the operator file `ts audit ad-templates generate` derived everything it wrote from a live, page-controlled ad stack and never checked the result, so several reachable inputs produced a config that cannot load. An unloadable `trusted-server.toml` is not a degraded ad stack: `build_state` fails and the adapter answers every route from the startup error router, so the whole site returns 500 once pushed. Add a write-side gate that runs the candidate through `Settings::from_toml`, the same `finalize_deserialized` chain the runtime uses at startup. It runs on the `--dry-run` path too, so a clean preview is now evidence the config loads. When the target config was already unloadable before the run, the gate reports that as a warning instead of blaming this run, so a freshly bootstrapped file carrying placeholder secrets can still be updated. Close the three reachable paths at their source as well: - Skip a scraped slot whose ad-unit path contains `{` or `}`. The path is a template and there is no escape syntax, so a literal brace either fails config load or is silently reinterpreted as a placeholder. - Skip a slot whose div id normalizes to nothing (a wholly ephemeral id such as a React SSR marker). An empty `div_id` fails config load, and as a runtime prefix it would bind the slot to the first id-bearing element on the page. - Refuse to create a `[creative_opportunities]` section with no GAM network id rather than writing one that omits the required key. This is reachable because the network id is only recovered from an all-digit leading segment, which an MCM child-network path does not have. --- .../src/commands/audit/generate/gpt_slots.rs | 89 ++++++++++++++ .../src/commands/audit/generate/mod.rs | 72 +++++++++++ .../src/commands/audit/generate/slot_toml.rs | 34 +++++- .../src/commands/audit/generate/validate.rs | 112 ++++++++++++++++++ 4 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/validate.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index fea34dd1f..365a5b696 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -130,6 +130,9 @@ fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option if is_multi_slot_div(&entry.div_id) { return None; } + if !is_usable_unit_path(&entry.gam_unit_path) { + return None; + } let formats: Vec<(u32, u32)> = entry .sizes .iter() @@ -140,6 +143,14 @@ fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option return None; } let div_stem = normalize_div_stem(&entry.div_id); + // Normalization truncates at the first ephemeral marker, so a div id that is + // *entirely* ephemeral (`_R_9sl…`, or exactly `-container`) reduces to the + // empty string. An empty `div_id` override fails config load outright, and + // an empty prefix would bind the slot to the first id-bearing element on the + // page, so such a slot is unusable rather than merely imprecise. + if div_stem.is_empty() { + return None; + } Some(DiscoveredSlot { id: slot_id_from_div(&div_stem), div_id: div_stem, @@ -155,6 +166,17 @@ fn is_multi_slot_div(div_id: &str) -> bool { div_id.contains('~') } +/// Whether a scraped GAM ad-unit path can be represented in config. +/// +/// `gam_unit_path` is a template: `{` and `}` delimit placeholders and +/// [`parse_unit_template`](trusted_server_core::creative_opportunities) offers no +/// escape syntax. A live path containing a brace would either fail config load +/// or, worse, be silently reinterpreted as a placeholder-bearing template. A +/// blank path is rejected for the same reason config load rejects it. +fn is_usable_unit_path(path: &str) -> bool { + !path.trim().is_empty() && !path.contains(['{', '}']) +} + /// Strips ephemeral GPT div-id noise so the stored id is stable across renders. /// /// Removes a trailing `-container` wrapper, then truncates at the first ephemeral @@ -220,6 +242,9 @@ fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? .to_string(); let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + if !is_usable_unit_path(&gam_unit_path) { + return None; + } // A usable unit path needs the network id plus at least one path segment. parts.next()?; @@ -232,6 +257,11 @@ fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot)> { return None; } let div_id = normalize_div_stem(&raw_div); + // See `slot_from_registry`: a fully ephemeral div id normalizes to nothing, + // which is neither a valid config value nor a usable runtime prefix. + if div_id.is_empty() { + return None; + } let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); if formats.is_empty() { @@ -466,6 +496,65 @@ mod tests { } } + #[test] + fn registry_slot_with_brace_in_unit_path_is_skipped() { + // `gam_unit_path` is a template and there is no escape syntax, so a + // literal brace either fails config load or is silently reinterpreted as + // a placeholder. Neither is acceptable to persist. + let registry = vec![ + registry_slot("/123/home/{section}", "div-gpt-ad-a", &[(300, 250)]), + registry_slot("/123/home/ok", "div-gpt-ad-b", &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "the brace-bearing slot should be dropped, the clean one kept" + ); + assert_eq!(discovered.slots[0].gam_unit_path, "/123/home/ok"); + } + + #[test] + fn registry_slot_whose_div_id_is_entirely_ephemeral_is_skipped() { + // `_R_…` is a React SSR marker; normalizing truncates at it, leaving an + // empty stem. An empty div_id fails config load, and as a runtime prefix + // it would match the first id-bearing element on the page. + let registry = vec![registry_slot( + "/123/home/header", + "_R_9slkta7pd6", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a slot with no stable div stem should be dropped, got {:?}", + discovered.slots + ); + } + + #[test] + fn volatile_guid_div_id_still_normalizes_to_a_usable_prefix() { + // The live autoblog shape: a GUID between two copies of the slot name. + // This must survive - only a stem that normalizes to *nothing* is dropped. + let registry = vec![registry_slot( + "/88059007/autoblog/homepage", + "ad-in_content-0949b6c5726343bf8bbec2ac47b494b4-in_content-0", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!( + discovered.slots[0].div_id, "ad-in_content", + "the GUID and trailing index should be truncated to a stable prefix" + ); + } + #[test] fn reads_slots_from_live_registry() { let registry = vec![registry_slot( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 1feec9329..d3125a1f5 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod browser_collector; pub(crate) mod collector; mod gpt_slots; mod slot_toml; +mod validate; use std::collections::BTreeSet; use std::fs; @@ -544,6 +545,15 @@ pub(crate) fn run_update_slots( let rendered_slots = render_slots(&merged); let updated = splice_creative_slots(&existing, network_id.as_deref(), &rendered_slots)?; + // Everything above is derived from a live, page-controlled ad stack, so the + // candidate has to clear the runtime's own load path before it can replace + // the operator's file. This runs on the dry-run path too — otherwise "the + // preview looked fine" would not be evidence that the config loads. + for warning in validate::check_candidate(&updated, &existing)? { + writeln!(out, "warning: {warning}") + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + if dry_run { writeln!(out, "{updated}") .map_err(|error| report_error(format!("failed to write preview: {error}")))?; @@ -1150,6 +1160,68 @@ mod tests { toml::from_str::(&written).expect("rewritten config is valid TOML"); } + /// A full, loadable config with real secrets substituted, so the write-side + /// validation gate is live rather than downgraded by a broken baseline. + fn loadable_config() -> String { + EXAMPLE_CONFIG + .replace( + "replace-with-admin-password-32-bytes", + "test-admin-password-32-bytes-minimum", + ) + .replace( + "trusted-server-placeholder-secret", + "test-ec-passphrase-32-bytes-minimum", + ) + .replace( + "change-me-proxy-secret", + "test-proxy-secret-32-bytes-minimum", + ) + } + + #[test] + fn generated_config_loads_through_the_runtime_settings_path() { + // The end-to-end contract: whatever `generate` writes must survive the + // same load path the adapter runs at startup. An unloadable config is a + // full-site outage once pushed, not a degraded ad stack. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let baseline = loadable_config(); + trusted_server_core::settings::Settings::from_toml(&baseline) + .expect("test baseline must itself be loadable or the gate is not exercised"); + fs::write(&config_path, &baseline).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + "https://publisher.example/", + &config_path, + None, + &[], + false, + &[], + false, + &collector, + &mut out, + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let settings = trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + let creative = settings + .creative_opportunities + .expect("generated config should carry creative opportunities"); + assert_eq!( + creative.slot.len(), + 1, + "the discovered slot should be present after a real load" + ); + assert_eq!( + creative.slot[0].div_id.as_deref(), + Some("div-gpt-ad-header") + ); + } + #[test] fn update_slots_dry_run_does_not_persist_environment_overlay_config() { let temp = TempDir::new().expect("should create temp dir"); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 580fe39ad..880ce1cac 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -365,14 +365,25 @@ pub(super) fn splice_creative_slots( // No section yet — append a fresh one with the network id and slots. if !has_canonical_header { + // `gam_network_id` is a required field, so creating the section without + // one writes a config that cannot load at all. This is reachable: the + // network id is only recovered when the scraped unit path starts with an + // all-digit segment, which an MCM/child-network path like + // `/1234,5678/home/header` does not. + let Some(network_id) = network_id else { + return cli_error( + "refusing to create a `[creative_opportunities]` section without a \ + GAM network id: none could be determined from the audited page, and \ + the key is required. Add `[creative_opportunities]` with a \ + `gam_network_id` to the config and re-run", + ); + }; let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); } result.push_str("\n[creative_opportunities]\n"); - if let Some(network_id) = network_id { - result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); - } + result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); result.push_str(rendered); result.push('\n'); return Ok(result); @@ -697,6 +708,23 @@ mod tests { ); } + #[test] + fn splice_refuses_fresh_section_without_a_network_id() { + // Reachable whenever the scraped unit path has no all-digit leading + // segment (MCM/child-network paths). Writing the section anyway produces + // a config missing a required field, which fails load and takes every + // route to the startup error router once pushed. + let existing = "[publisher]\ndomain = \"x\"\n"; + + let error = splice_creative_slots(existing, None, &header_rendered()) + .expect_err("should refuse to create a section with no network id"); + + assert!( + format!("{error:?}").contains("without a GAM network id"), + "error should name the missing network id, got {error:?}" + ); + } + #[test] fn splice_appends_section_when_config_has_none() { let existing = "[publisher]\ndomain = \"x\"\n"; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs new file mode 100644 index 000000000..721ba4046 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -0,0 +1,112 @@ +//! Write-side validation for generated ad-template config. +//! +//! Everything the generator writes is derived from a live, page-controlled ad +//! stack, so the candidate document has to clear the same bar the runtime +//! applies at startup *before* it replaces the operator's file. A config the +//! runtime rejects is not a degraded ad stack — `build_state` fails and the +//! adapter answers every route from the startup error router, so an unloadable +//! `trusted-server.toml` is a full-site outage once pushed. + +use trusted_server_core::settings::Settings; + +use crate::error::{CliResult, cli_error}; + +/// Validates the candidate config text the generator is about to persist. +/// +/// Runs [`Settings::from_toml`], which drives the identical +/// `finalize_deserialized` chain the runtime uses — serde (`deny_unknown_fields` +/// plus required fields), then `compile_slots` → `compile_unit_templates` → +/// `validate_runtime`, then the validator pass — with no I/O. +/// +/// `baseline` is the config as it was read from disk. When the baseline is +/// *already* unloadable, this run cannot be blamed for it: the candidate is +/// accepted and the pre-existing error is returned as a warning instead. Without +/// that escape hatch a freshly bootstrapped config carrying placeholder secrets +/// could never be updated by `generate`. +/// +/// # Errors +/// +/// Returns a user-facing error when the candidate fails to load and the baseline +/// loaded cleanly — that is, when this run introduced the failure. +pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult> { + let Err(candidate_error) = Settings::from_toml(candidate) else { + return Ok(Vec::new()); + }; + + if let Err(baseline_error) = Settings::from_toml(baseline) { + return Ok(vec![format!( + "target config was already invalid before this run, so the generated \ + result could not be verified: {baseline_error}" + )]); + } + + cli_error(format!( + "refusing to write: the generated config would fail to load, which would \ + take the service down once pushed: {candidate_error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A minimal config that loads cleanly, used as the valid baseline. + fn baseline() -> String { + crate::commands::config::init::EXAMPLE_CONFIG + .replace( + "replace-with-admin-password-32-bytes", + "test-admin-password-32-bytes-minimum", + ) + .to_string() + } + + #[test] + fn valid_candidate_passes_without_warnings() { + let config = baseline(); + + let warnings = check_candidate(&config, &config).expect("valid candidate should pass"); + + assert!( + warnings.is_empty(), + "a clean candidate should not warn, got {warnings:?}" + ); + } + + #[test] + fn candidate_this_run_broke_is_refused() { + let good = baseline(); + // An empty div_id override is exactly what a div id normalized down to + // nothing would produce, and `validate_runtime` rejects it. + let broken = format!( + "{good}\n[[creative_opportunities.slot]]\n\ + id = \"broken\"\ndiv_id = \"\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n" + ); + + let error = check_candidate(&broken, &good).expect_err("should refuse a broken candidate"); + + assert!( + format!("{error:?}").contains("refusing to write"), + "error should name the refusal, got {error:?}" + ); + } + + #[test] + fn pre_existing_breakage_downgrades_to_a_warning() { + // The operator's file was already unloadable; `generate` must still be + // able to update it rather than blaming this run for the old error. + let broken_baseline = "[creative_opportunities]\n"; + let broken_candidate = "[creative_opportunities]\n"; + + let warnings = check_candidate(broken_candidate, broken_baseline) + .expect("a pre-existing failure should not block the write"); + + assert_eq!(warnings.len(), 1, "should surface exactly one warning"); + assert!( + warnings[0].contains("already invalid"), + "warning should name the pre-existing failure, got {:?}", + warnings[0] + ); + } +} From e7f8268743e847bdd4e59df4a8285390fd3e866d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 10:55:22 +0530 Subject: [PATCH 183/195] Add multi-page collection and crawl planning for ad-template generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for discovering ad slots across a site's sections rather than from a single page. Nothing calls this yet; `run_update_slots` is unchanged. `AuditCollector` gains a defaulted `collect_pages` that streams each page to a sink, so every existing implementor keeps working and the caller can fold a page into its evidence and drop the DOM immediately instead of holding every serialization at once. The browser collector overrides it to launch Chrome once for the whole crawl: a cold start plus a fresh profile dominates the cost of a multi-page run, and the shared profile carries a bot-protection clearance cookie earned on the first page across the rest of the walk. Page discovery reads the hydrated DOM rather than the served markup, because an app-router page keeps its link graph in the framework payload — parsing raw HTML finds only a fraction of a site's sections. Sitemaps are fetched from inside the open page via `fetch` plus `DOMParser`, which inherits the session's cookies and Chrome's TLS fingerprint, gets transparent gzip and XML parsing, and so needs no new Rust dependency. `crawl_plan` turns links and sitemap entries into a bounded page set: one landing page and one article per section, ranked by whether navigation and the sitemap corroborate each other, capped by section and page budgets. Sections dropped for budget are reported rather than silently omitted. Same-origin is enforced on links and on sitemap entries alike, since a `Sitemap:` directive can name any host and the crawl carries operator cookies. --- .../src/commands/audit/generate/analyzer.rs | 12 + .../audit/generate/browser_collector.rs | 217 +++++++- .../src/commands/audit/generate/collector.rs | 78 +++ .../src/commands/audit/generate/crawl_plan.rs | 521 ++++++++++++++++++ .../src/commands/audit/generate/mod.rs | 5 + 5 files changed, 823 insertions(+), 10 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs b/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs index dc1ea9ffe..06d784b7a 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/analyzer.rs @@ -286,6 +286,8 @@ mod tests { resource_type: Some("Script".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: vec!["partial settle".to_string()], }; @@ -323,6 +325,8 @@ mod tests { script_tags: Vec::new(), network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -341,6 +345,8 @@ mod tests { script_tags: Vec::new(), network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -366,6 +372,8 @@ mod tests { resource_type: Some("script".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -401,6 +409,8 @@ mod tests { ], network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; @@ -432,6 +442,8 @@ mod tests { script_tags: Vec::new(), network_requests: Vec::new(), gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index 077c5550a..b1c504cd5 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -13,7 +13,8 @@ use url::Url; use which::which; use crate::commands::audit::generate::collector::{ - AuditCollector, CollectedGptSlot, CollectedPage, CollectedRequest, CollectedScriptTag, + AuditCollector, CollectedGptSlot, CollectedLink, CollectedPage, CollectedRequest, + CollectedScriptTag, ControlFlow, PageSink, }; use crate::error::{CliResult, report_error}; @@ -49,14 +50,56 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(collect_page_via_browser_async(target_url, cookies)) + runtime.block_on(async { + let mut collected = None; + with_browser( + std::slice::from_ref(target_url), + cookies, + &mut |_, result| { + collected = Some(result); + Ok(ControlFlow::Stop) + }, + ) + .await?; + collected.unwrap_or_else(|| Err(report_error("browser session produced no page"))) + }) + } + + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_page: PageSink<'_>, + ) -> CliResult<()> { + if targets.is_empty() { + return Ok(()); + } + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| { + report_error(format!( + "failed to build Tokio runtime for browser audit: {error}" + )) + })?; + + runtime.block_on(with_browser(targets, cookies, on_page)) } } -async fn collect_page_via_browser_async( - target_url: &Url, +/// Launches one browser, walks `targets` on it, and hands each result to `sink`. +/// +/// One launch for the whole crawl rather than one per page: a cold Chrome start +/// plus a fresh profile dominates the cost of a multi-page run. The shared +/// profile is also load-bearing — a bot-protection clearance cookie earned on +/// the first page carries to the rest of the crawl, which is what makes a +/// multi-section walk of a protected site viable at all. The tradeoff is that +/// paywall meters and personalization also accumulate across the run. +async fn with_browser( + targets: &[Url], cookies: &[(String, String)], -) -> CliResult { + sink: PageSink<'_>, +) -> CliResult<()> { let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { report_error(format!( @@ -93,13 +136,25 @@ async fn collect_page_via_browser_async( } }); - let result = collect_page_from_browser(&mut browser, target_url, cookies).await; + // Sitemap discovery is a whole-site fact, so only the first target pays for it. + let mut result = Ok(()); + for (index, target) in targets.iter().enumerate() { + let collected = collect_page_from_browser(&mut browser, target, cookies, index == 0).await; + match sink(target, collected) { + Ok(ControlFlow::Continue) => {} + Ok(ControlFlow::Stop) => break, + Err(error) => { + result = Err(error); + break; + } + } + } let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) .await .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { + .and_then(|closed| { + closed.map_err(|error| { report_error(format!("failed to close browser after audit: {error}")) }) }); @@ -109,15 +164,21 @@ async fn collect_page_via_browser_async( let _ = handler_task.await; match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), + (Ok(()), Ok(_)) => Ok(()), + (Ok(()), Err(error)) | (Err(error), _) => Err(error), } } +/// Collects one page on an already-launched browser. +/// +/// `discover_sitemap` runs the `robots.txt`/sitemap fetch from inside this +/// page's context. It is meaningful only once per crawl (the site's sitemap does +/// not change per page), so callers pass `true` for the root page only. async fn collect_page_from_browser( browser: &mut Browser, target_url: &Url, cookies: &[(String, String)], + discover_sitemap: bool, ) -> CliResult { let page = browser.new_page("about:blank").await.map_err(|error| { report_error(format!("failed to create browser page for audit: {error}")) @@ -238,6 +299,34 @@ async fn collect_page_from_browser( Err(_) => Vec::new(), }; + // Links come from the hydrated DOM, not the served markup: an app-router + // page keeps its link graph in the framework payload, so parsing the raw + // HTML finds only a fraction of the site's sections. Best-effort — an empty + // list just means crawl planning falls back to other sources. + let links: Vec = match page.evaluate(LINKS_SCRIPT).await { + Ok(result) => result.into_value().unwrap_or_default(), + Err(_) => Vec::new(), + }; + + // Sitemap discovery is a whole-site fact, so it runs once per crawl. A miss + // is normal (no sitemap, robots 404, fetch blocked) and leaves planning to + // the link graph alone. + let mut sitemap_locs: Vec = if discover_sitemap { + match page.evaluate(SITEMAP_SCRIPT).await { + Ok(result) => result.into_value().unwrap_or_default(), + Err(_) => Vec::new(), + } + } else { + Vec::new() + }; + sitemap_locs.truncate(MAX_SITEMAP_LOCS); + if discover_sitemap && sitemap_locs.is_empty() { + warnings.push( + "no sitemap was reachable; site sections were inferred from page links only" + .to_string(), + ); + } + Ok(CollectedPage { requested_url: target_url.to_string(), final_url, @@ -258,10 +347,118 @@ async fn collect_page_from_browser( }) .collect(), gpt_slots, + links, + sitemap_locs, warnings, }) } +/// Maximum sitemap `` entries kept. Section discovery needs one page per +/// section, so a 50,000-URL catalog sitemap is truncated hard. +const MAX_SITEMAP_LOCS: usize = 5000; + +/// Reads same-origin `a[href]` targets from the hydrated DOM. +/// +/// `anchor.href` is absolutized by the DOM already, and `in_nav` records whether +/// the anchor sits inside site navigation — navigation is the publisher's own +/// declaration of its taxonomy, so those links rank higher when picking sections. +/// +/// Reading the DOM rather than the served markup is deliberate: an app-router +/// page keeps its link graph in the framework payload, so parsing raw HTML finds +/// only a fraction of a site's sections. +const LINKS_SCRIPT: &str = r#"() => { + try { + const navAnchors = new Set( + Array.from(document.querySelectorAll( + 'nav a[href], header a[href], [role="navigation"] a[href]' + )) + ); + const out = []; + const seen = new Set(); + for (const anchor of document.querySelectorAll('a[href]')) { + if (out.length >= 2000) break; + const href = anchor.href; + if (!href || seen.has(href)) continue; + if (!href.startsWith(location.origin)) continue; + seen.add(href); + out.push({ url: href, in_nav: navAnchors.has(anchor) }); + } + return out; + } catch (error) { + return []; + } +}"#; + +/// Discovers sitemap page URLs from inside the page, starting at `robots.txt`. +/// +/// Runs in the browser rather than through a Rust HTTP client on purpose: the +/// in-page `fetch` carries the session's cookies and Chrome's TLS fingerprint, +/// so a bot-protection layer that would answer a bare client with a challenge +/// serves the real document instead. It also gets transparent gzip and an XML +/// parser for free, which is why sitemap support needs no new Rust dependency. +/// +/// Same-origin is enforced here *and* again in Rust: a `Sitemap:` directive can +/// name any host, and this crawl carries operator-supplied cookies. +const SITEMAP_SCRIPT: &str = r#"async () => { + const sameOrigin = (raw) => { + try { + return new URL(raw, location.origin).origin === location.origin; + } catch (error) { + return false; + } + }; + const fetchText = async (url) => { + try { + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) return null; + return await response.text(); + } catch (error) { + return null; + } + }; + const parseLocs = (text) => { + try { + const doc = new DOMParser().parseFromString(text, 'application/xml'); + if (doc.querySelector('parsererror')) return { pages: [], indexes: [] }; + const indexes = Array.from(doc.querySelectorAll('sitemapindex > sitemap > loc')) + .map((node) => (node.textContent || '').trim()).filter(sameOrigin); + const pages = Array.from(doc.querySelectorAll('urlset > url > loc')) + .map((node) => (node.textContent || '').trim()).filter(sameOrigin); + return { pages, indexes }; + } catch (error) { + return { pages: [], indexes: [] }; + } + }; + + const roots = []; + const robots = await fetchText('/robots.txt'); + if (robots) { + for (const line of robots.split(/\r?\n/)) { + const match = /^\s*sitemap\s*:\s*(\S+)/i.exec(line); + if (match && sameOrigin(match[1])) roots.push(match[1]); + } + } + if (roots.length === 0) roots.push('/sitemap.xml', '/sitemap_index.xml'); + + const pages = []; + let childrenFollowed = 0; + for (const root of roots) { + if (pages.length >= 5000) break; + const text = await fetchText(root); + if (!text) continue; + const parsed = parseLocs(text); + pages.push(...parsed.pages); + for (const child of parsed.indexes) { + if (childrenFollowed >= 10 || pages.length >= 5000) break; + childrenFollowed += 1; + const childText = await fetchText(child); + if (!childText) continue; + pages.push(...parseLocs(childText).pages); + } + } + return pages.slice(0, 5000); +}"#; + /// Reads the live GPT slot registry into `{gam_unit_path, div_id, sizes}` rows. /// /// Mirrors the ad-template verifier's `getSlots()` scrape: it defends against a diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 2a31c763b..625ac660e 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -3,6 +3,26 @@ use url::Url; use crate::error::CliResult; +/// Sink invoked once per collected page during a batch crawl. +/// +/// Receives the per-page outcome so a failed page can be folded into the run as +/// a warning rather than aborting it; returning `Err` stops the crawl. +pub(crate) type PageSink<'a> = + &'a mut dyn FnMut(&Url, CliResult) -> CliResult; + +/// Whether a batch crawl should keep going after a page. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ControlFlow { + /// Collect the next target. + #[allow( + dead_code, + reason = "constructed by run_update_slots once it orchestrates the crawl" + )] + Continue, + /// Stop the crawl without an error (budget reached, challenge rate exceeded). + Stop, +} + pub(crate) trait AuditCollector { /// Collects a live page. `cookies` are `(name, value)` pairs set on the /// browser context before navigation (scoped to `target_url`) so an existing @@ -13,6 +33,41 @@ pub(crate) trait AuditCollector { target_url: &Url, cookies: &[(String, String)], ) -> CliResult; + + /// Collects several pages in one session, handing each result to `on_page`. + /// + /// The default implementation loops over [`collect_page`](Self::collect_page), + /// which keeps every existing implementor working unchanged. The browser + /// collector overrides it to reuse one Chrome instance and profile across the + /// crawl — a fresh launch per page dominates the cost of a multi-page run, + /// and a shared profile carries bot-protection clearance cookies site-wide. + /// + /// Results are streamed rather than returned as a `Vec` so the caller can + /// fold each page into its evidence and drop the page's HTML immediately, + /// instead of holding every DOM serialization at once. + /// + /// # Errors + /// + /// Returns an error when `on_page` does, or when the session itself cannot + /// be established. Individual page failures are delivered to `on_page`. + #[allow( + dead_code, + reason = "called by run_update_slots once it orchestrates the crawl" + )] + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_page: PageSink<'_>, + ) -> CliResult<()> { + for target in targets { + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -29,9 +84,32 @@ pub(crate) struct CollectedPage { /// when the ad request never fires (consent-gated or iframe-issued). #[serde(default)] pub(crate) gpt_slots: Vec, + /// Same-origin `a[href]` targets read from the hydrated DOM, absolutized. + /// + /// Read from the live DOM rather than the served HTML on purpose: an + /// app-router page keeps its link graph in the framework payload, so parsing + /// the raw markup finds only a fraction of the site's sections. + #[serde(default)] + pub(crate) links: Vec, + /// Sitemap `` entries discovered from `robots.txt`, when fetched. + /// + /// Empty unless sitemap discovery ran (root page only). + #[serde(default)] + pub(crate) sitemap_locs: Vec, pub(crate) warnings: Vec, } +/// A same-origin link observed in the hydrated DOM. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedLink { + /// Absolute URL of the link target. + pub(crate) url: String, + /// Whether the anchor sits inside site navigation (`nav`, `header`, + /// `[role="navigation"]`). Nav links are the publisher's own declaration of + /// its taxonomy, so they rank above body links when choosing sections. + pub(crate) in_nav: bool, +} + /// A single slot read from the page's live GPT registry. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct CollectedGptSlot { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs new file mode 100644 index 000000000..5b959dc4f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -0,0 +1,521 @@ +//! Pure crawl planning: turn discovered links and sitemap entries into the +//! bounded set of pages worth loading in a browser. +//! +//! The goal is deliberately *not* site coverage. Ad slots repeat per site +//! section, and the generated config needs one glob pair per section +//! (`/news` and `/news/*`), so one representative page per section is enough. +//! That keeps the crawl proportional to the publisher's taxonomy (a dozen +//! sections) rather than its catalog (tens of thousands of articles). +//! +//! Two sources feed the plan and each supplies a half the other cannot: +//! +//! - **Navigation links** give section *landing* paths (`/news`), which +//! sitemaps routinely omit, and are the publisher's own taxonomy declaration. +//! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), +//! which is where in-content slots live, and reveal sections hidden behind a +//! navigation overflow menu. +#![allow( + dead_code, + reason = "planner is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::BTreeMap; + +use url::Url; + +use super::collector::CollectedLink; + +/// Path segments that are never a content section worth sampling. +/// +/// These carry either no ad stack at all or an unrepresentative one, and +/// crawling them spends budget that a real section needs. +const NOISE_SEGMENTS: &[&str] = &[ + "about", + "about-us", + "account", + "author", + "cart", + "contact", + "editorial-policy", + "login", + "logout", + "newsletter", + "page", + "press", + "privacy", + "register", + "search", + "sitemap", + "subscribe", + "terms", +]; + +/// File extensions that are assets rather than pages. +const NON_PAGE_EXTENSIONS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +/// Bounds on how much of a site a single run will load. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct CrawlBudget { + /// Maximum number of sections to sample. + pub(super) max_sections: usize, + /// Maximum number of pages to load in total, including the root. + pub(super) max_pages: usize, +} + +impl Default for CrawlBudget { + fn default() -> Self { + Self { + max_sections: 8, + max_pages: 17, + } + } +} + +/// One section selected for sampling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlannedSection { + /// The first path segment identifying the section (`news`). + pub(super) segment: String, + /// The section landing page, when one was observed. + pub(super) landing: Option, + /// A representative content page inside the section, when one was observed. + pub(super) article: Option, +} + +impl PlannedSection { + /// The pages to load for this section, landing first. + fn targets(&self) -> impl Iterator { + self.landing.iter().chain(self.article.iter()) + } +} + +/// The bounded outcome of planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CrawlPlan { + /// Sections selected for sampling, highest confidence first. + pub(super) sections: Vec, + /// Sections found but dropped because the budget was already spent. + pub(super) dropped_sections: Vec, + /// Human-readable notes about how the plan was reached. + pub(super) notes: Vec, +} + +impl CrawlPlan { + /// Page URLs to load, in crawl order. The root is *not* included — the + /// caller has already collected it in order to plan at all. + pub(super) fn targets(&self) -> Vec { + self.sections + .iter() + .flat_map(PlannedSection::targets) + .cloned() + .collect() + } +} + +/// Evidence gathered about one candidate section before ranking. +#[derive(Debug, Default)] +struct SectionCandidate { + landing: Option, + article: Option, + in_nav: bool, + in_sitemap: bool, + link_count: usize, +} + +impl SectionCandidate { + /// Confidence ordering: corroborated by both sources beats either alone, + /// and navigation beats a sitemap-only hit because navigation is the + /// publisher's own statement of what its sections are. + fn rank(&self) -> u8 { + match (self.in_nav, self.in_sitemap) { + (true, true) => 3, + (true, false) => 2, + (false, true) => 1, + (false, false) => 0, + } + } +} + +/// Plans the crawl from the root page's links and any sitemap entries. +/// +/// `root` bounds the crawl: every candidate must share its origin, which also +/// stops a hostile or misconfigured `robots.txt` from redirecting the crawl (and +/// the operator's cookies) at an unrelated host. +pub(super) fn plan_crawl( + root: &Url, + links: &[CollectedLink], + sitemap_locs: &[String], + budget: CrawlBudget, +) -> CrawlPlan { + let mut candidates: BTreeMap = BTreeMap::new(); + let mut notes = Vec::new(); + + for link in links { + let Some(url) = same_origin_page_url(root, &link.url) else { + continue; + }; + let Some(segment) = first_segment(&url) else { + continue; + }; + let entry = candidates.entry(segment).or_default(); + entry.in_nav |= link.in_nav; + entry.link_count += 1; + record_url(entry, &url); + } + + let mut sitemap_pages = 0_usize; + for loc in sitemap_locs { + let Some(url) = same_origin_page_url(root, loc) else { + continue; + }; + let Some(segment) = first_segment(&url) else { + continue; + }; + sitemap_pages += 1; + let entry = candidates.entry(segment).or_default(); + entry.in_sitemap = true; + record_url(entry, &url); + } + + if !sitemap_locs.is_empty() { + notes.push(format!( + "sitemap contributed {sitemap_pages} same-origin page(s) across {} section(s)", + candidates.values().filter(|c| c.in_sitemap).count() + )); + } + if links.iter().all(|link| !link.in_nav) && !links.is_empty() { + notes.push( + "no navigation links were found; sections were inferred from body links only" + .to_string(), + ); + } + + // Rank before truncating: confidence first, then how heavily the section is + // linked, then the segment name so runs are reproducible. + let mut ranked: Vec<(String, SectionCandidate)> = candidates.into_iter().collect(); + ranked.sort_by(|(left_segment, left), (right_segment, right)| { + right + .rank() + .cmp(&left.rank()) + .then(right.link_count.cmp(&left.link_count)) + .then(left_segment.cmp(right_segment)) + }); + + let mut sections = Vec::new(); + let mut dropped_sections = Vec::new(); + // The root page is already collected and counts against the page budget. + let mut pages_used = 1_usize; + for (segment, candidate) in ranked { + let planned = PlannedSection { + segment: segment.clone(), + landing: candidate.landing, + article: candidate.article, + }; + let cost = planned.targets().count(); + if cost == 0 { + continue; + } + if sections.len() >= budget.max_sections || pages_used + cost > budget.max_pages { + dropped_sections.push(segment); + continue; + } + pages_used += cost; + sections.push(planned); + } + + if !dropped_sections.is_empty() { + notes.push(format!( + "budget reached: {} section(s) not sampled ({}); raise --max-sections/--max-pages to include them", + dropped_sections.len(), + dropped_sections.join(", ") + )); + } + + CrawlPlan { + sections, + dropped_sections, + notes, + } +} + +/// Files a URL as the section's landing page or its representative article. +/// +/// The first candidate of each kind wins, so a run is stable given stable input. +fn record_url(entry: &mut SectionCandidate, url: &Url) { + if segment_count(url) == 1 { + if entry.landing.is_none() { + entry.landing = Some(url.clone()); + } + } else if entry.article.is_none() { + entry.article = Some(url.clone()); + } +} + +/// Parses `raw` against `root` and keeps it only if it is a same-origin page. +/// +/// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or +/// utility paths. Query and fragment are dropped so `/news?page=2` and +/// `/news#top` collapse onto `/news`. +fn same_origin_page_url(root: &Url, raw: &str) -> Option { + let mut url = root.join(raw).ok()?; + if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { + return None; + } + url.set_query(None); + url.set_fragment(None); + + let path = url.path().to_ascii_lowercase(); + if NON_PAGE_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + { + return None; + } + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.is_empty() { + return None; + } + if NOISE_SEGMENTS.contains(&segments[0]) { + return None; + } + // `/news/page/2` is the same inventory as `/news`, so it is not a second + // sample worth spending a page load on. + if segments.contains(&"page") { + return None; + } + Some(url) +} + +/// The first non-empty path segment, lowercased. +fn first_segment(url: &Url) -> Option { + url.path() + .split('/') + .find(|part| !part.is_empty()) + .map(str::to_ascii_lowercase) +} + +/// Count of non-empty path segments. +fn segment_count(url: &Url) -> usize { + url.path() + .split('/') + .filter(|part| !part.is_empty()) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> Url { + Url::parse("https://publisher.example/").expect("valid root") + } + + fn nav(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + } + } + + fn body(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: false, + } + } + + fn segments(plan: &CrawlPlan) -> Vec<&str> { + plan.sections + .iter() + .map(|section| section.segment.as_str()) + .collect() + } + + #[test] + fn pairs_a_landing_page_with_an_article_from_the_sitemap() { + let plan = plan_crawl( + &root(), + &[nav("/news")], + &["https://publisher.example/news/story-abc".to_string()], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["news"]); + let section = &plan.sections[0]; + assert_eq!( + section.landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news") + ); + assert_eq!( + section.article.as_ref().map(Url::as_str), + Some("https://publisher.example/news/story-abc") + ); + assert_eq!(plan.targets().len(), 2, "should load landing then article"); + } + + #[test] + fn cross_origin_candidates_are_dropped() { + // Guards both the sitemap (a `Sitemap:` directive can point anywhere) + // and links: the crawl carries operator cookies, so it must not leave + // the requested origin. + let plan = plan_crawl( + &root(), + &[CollectedLink { + url: "https://tracker.example/news".to_string(), + in_nav: true, + }], + &["https://other.example/deals/x".to_string()], + CrawlBudget::default(), + ); + + assert!( + plan.sections.is_empty(), + "no off-origin section should survive, got {:?}", + segments(&plan) + ); + } + + #[test] + fn utility_paths_and_assets_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/about-us"), + nav("/search"), + nav("/editorial-policy"), + nav("/logo.png"), + nav("/feed.xml"), + nav("/news/page/2"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the real content section should remain" + ); + } + + #[test] + fn query_and_fragment_collapse_onto_one_landing_page() { + let plan = plan_crawl( + &root(), + &[nav("/news?utm_source=x"), nav("/news#top"), nav("/news")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["news"]); + assert_eq!( + plan.sections[0].landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news"), + "tracking query and fragment should be stripped" + ); + } + + #[test] + fn nav_and_sitemap_corroboration_outranks_either_alone() { + let plan = plan_crawl( + &root(), + &[nav("/features"), body("/reviews")], + &[ + "https://publisher.example/features/story".to_string(), + "https://publisher.example/deals/x".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan)[0], + "features", + "nav + sitemap should rank first, got {:?}", + segments(&plan) + ); + } + + #[test] + fn budget_truncates_and_reports_what_was_dropped() { + let links: Vec = ["a", "b", "c", "d"] + .iter() + .map(|segment| nav(&format!("/{segment}"))) + .collect(); + + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 2, + max_pages: 17, + }, + ); + + assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); + assert_eq!(plan.dropped_sections.len(), 2); + assert!( + plan.notes + .iter() + .any(|note| note.contains("budget reached")), + "dropping sections must be reported, not silent: {:?}", + plan.notes + ); + } + + #[test] + fn page_budget_counts_the_already_collected_root() { + // max_pages = 3 leaves room for exactly one landing+article pair on top + // of the root page the caller already loaded. + let plan = plan_crawl( + &root(), + &[nav("/news"), nav("/deals")], + &[ + "https://publisher.example/news/a".to_string(), + "https://publisher.example/deals/b".to_string(), + ], + CrawlBudget { + max_sections: 8, + max_pages: 3, + }, + ); + + assert_eq!( + plan.targets().len(), + 2, + "root + 2 pages fills max_pages = 3" + ); + assert_eq!(plan.dropped_sections.len(), 1); + } + + #[test] + fn body_only_links_still_yield_sections_with_a_note() { + let plan = plan_crawl( + &root(), + &[body("/news"), body("/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["deals", "news"]); + assert!( + plan.notes + .iter() + .any(|note| note.contains("no navigation links")), + "a nav-less page should say so: {:?}", + plan.notes + ); + } + + #[test] + fn empty_input_plans_nothing_rather_than_panicking() { + let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); + + assert!(plan.sections.is_empty()); + assert!(plan.targets().is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index d3125a1f5..8d2491801 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -1,6 +1,7 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; +mod crawl_plan; mod gpt_slots; mod slot_toml; mod validate; @@ -667,6 +668,8 @@ mod tests { resource_type: Some("script".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), } } @@ -953,6 +956,8 @@ mod tests { resource_type: Some("fetch".to_string()), }], gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), warnings: Vec::new(), }; From 73ac39c1cecfd9b38edccc4e479aefd41312b8d8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:04:55 +0530 Subject: [PATCH 184/195] Accumulate cross-page slot evidence for ad-template generate Template inference needs the set of observations per slot, not one snapshot: a single page cannot distinguish a literal ad-unit path from a templated one, so the divergence across pages is the only signal available. Add the table that holds it. Nothing calls this yet. Slots are keyed on the normalized div stem, since raw GPT div ids carry per-render framework hashes and would otherwise look like a new slot on every page. Three reconciliations happen here and nowhere else: - Formats union across pages. A size that renders only on article pages, such as a 300x600 rail, has to survive alongside the homepage's sizes; taking the first page's list would silently narrow the slot. - Divergent unit paths are retained as separate rows rather than collapsed, because discarding them is what makes templating impossible. - Network ids must agree. Two GAM networks in one crawl means the pages are not one property, so this is a hard error naming both rather than a guess that would bid against the wrong inventory. Pages that yield no slots are recorded rather than dropped, so a caller can recognise a bot challenge serving interstitials and refuse to write a half-empty config. --- .../src/commands/audit/generate/evidence.rs | 372 ++++++++++++++++++ .../src/commands/audit/generate/mod.rs | 1 + 2 files changed, 373 insertions(+) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/evidence.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs new file mode 100644 index 000000000..e1b1bf81b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -0,0 +1,372 @@ +//! Cross-page slot evidence: what each slot looked like on every page it was +//! observed on. +//! +//! A single page cannot distinguish a literal ad-unit path from a templated one, +//! so inference needs the *set* of observations per slot rather than one +//! snapshot. This module accumulates that set and is deliberately the only place +//! that reconciles a slot seen more than once: +//! +//! - **Formats union.** A size that appears only on article pages (a 300x600 +//! rail, say) must survive alongside the homepage's sizes. Taking the first +//! page's formats would silently narrow the slot. +//! - **Unit paths are kept, not collapsed.** Divergence across pages is the +//! signal inference reads; discarding it is what makes templating impossible. +//! - **Network ids must agree.** Two different GAM networks in one crawl means +//! the pages are not one property, and writing either one would be a guess. +//! +//! Slots are keyed on the *normalized div stem* produced by +//! [`discover_gpt_slots`](super::gpt_slots::discover_gpt_slots), because raw GPT +//! div ids carry per-render framework hashes and would otherwise look like a new +//! slot on every page. + +#![allow( + dead_code, + reason = "table is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use super::gpt_slots::DiscoveredSlots; +use crate::error::{CliResult, cli_error}; + +/// One observation of a slot on one page. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct EvidenceRow { + /// The page path the slot was observed on, normalized (leading `/`, no + /// query or fragment). + pub(super) path: String, + /// The literal GAM ad-unit path the live page used for this slot. + pub(super) unit_path: String, +} + +/// Everything observed about one slot across the crawl. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SlotEvidence { + /// Config slot id derived from the div stem. + pub(super) id: String, + /// Normalized div stem, used as the runtime `div_id` prefix. + pub(super) div_id: String, + /// Union of every pixel size observed for this slot, smallest first. + pub(super) formats: BTreeSet<(u32, u32)>, + /// Whether any page carrying this slot showed header-bidding signals. + pub(super) has_prebid: bool, + /// Distinct `(path, unit_path)` observations, in a stable order. + pub(super) rows: BTreeSet, +} + +impl SlotEvidence { + /// The distinct literal unit paths observed for this slot. + pub(super) fn unit_paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.unit_path.as_str()).collect() + } + + /// The distinct page paths this slot was observed on. + pub(super) fn paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.path.as_str()).collect() + } +} + +/// Slot evidence accumulated across every collected page. +#[derive(Debug, Clone, Default)] +pub(super) struct EvidenceTable { + slots: BTreeMap, + /// Div stems in first-seen order, so generated config keeps crawl order + /// rather than alphabetical order. + order: Vec, + network_ids: BTreeSet, + /// Every page path folded in, including those that yielded no slots. + pages: BTreeSet, + /// Page paths that produced no slot evidence at all. + empty_pages: BTreeSet, +} + +impl EvidenceTable { + /// Folds one page's discovered slots into the table. + /// + /// `path` is the page's normalized request path; it is what page patterns + /// and `{section}` derivation are computed from later, so it must be the + /// post-redirect path actually audited. + pub(super) fn fold_page(&mut self, path: &str, discovered: &DiscoveredSlots) { + self.pages.insert(path.to_string()); + if let Some(network_id) = &discovered.gam_network_id { + self.network_ids.insert(network_id.clone()); + } + if discovered.slots.is_empty() { + self.empty_pages.insert(path.to_string()); + return; + } + + for slot in &discovered.slots { + let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { + self.order.push(slot.div_id.clone()); + SlotEvidence { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + formats: BTreeSet::new(), + has_prebid: false, + rows: BTreeSet::new(), + } + }); + // Union rather than replace: a size seen only on one page type is + // still a size this slot serves. + entry.formats.extend(slot.formats.iter().copied()); + entry.has_prebid |= slot.has_prebid; + entry.rows.insert(EvidenceRow { + path: path.to_string(), + unit_path: slot.gam_unit_path.clone(), + }); + } + } + + /// Slots in first-seen order. + pub(super) fn slots(&self) -> impl Iterator { + self.order + .iter() + .filter_map(|div_id| self.slots.get(div_id)) + } + + /// Number of distinct slots observed. + pub(super) fn slot_count(&self) -> usize { + self.slots.len() + } + + /// Every page path folded in, whether or not it yielded slots. + pub(super) fn pages(&self) -> &BTreeSet { + &self.pages + } + + /// Page paths that produced no slot evidence. + /// + /// A high proportion of these is the signature of a bot challenge serving + /// interstitials instead of the real site, which is worth refusing to write + /// from rather than persisting a half-empty config. + pub(super) fn empty_pages(&self) -> &BTreeSet { + &self.empty_pages + } + + /// Whether any slot was observed at all. + pub(super) fn is_empty(&self) -> bool { + self.slots.is_empty() + } + + /// The single GAM network id observed across the crawl. + /// + /// # Errors + /// + /// Returns an error when pages disagreed. Two networks in one crawl means + /// the pages are not one property (a syndicated subdomain, a child network, + /// an off-origin redirect that slipped through), and picking either would be + /// a guess that silently bids against the wrong inventory. + pub(super) fn network_id(&self) -> CliResult> { + let mut found = self.network_ids.iter(); + let Some(first) = found.next() else { + return Ok(None); + }; + if self.network_ids.len() > 1 { + let all: Vec<&str> = self.network_ids.iter().map(String::as_str).collect(); + return cli_error(format!( + "the crawled pages reported more than one GAM network id ({}); \ + they do not appear to be one property, so no network id can be \ + chosen safely. Audit a single property, or pass explicit URLs", + all.join(", ") + )); + } + Ok(Some(first.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// One live slot as `(unit path, div id, sizes)`. + type SlotFixture<'a> = (&'a str, &'a str, &'a [(u32, u32)]); + + fn page(slots: &[SlotFixture<'_>], has_prebid: bool) -> DiscoveredSlots { + let registry: Vec = slots + .iter() + .map(|(unit_path, div_id, sizes)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: sizes.to_vec(), + }) + .collect(); + discover_gpt_slots(®istry, &[], has_prebid) + } + + #[test] + fn formats_union_across_pages_instead_of_first_seen_winning() { + // The 300x600 rail only ever renders on article pages. Keeping the + // homepage's format list alone would silently narrow the slot. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-rail", &[(300, 250)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-rail", &[(300, 600)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.formats.iter().copied().collect::>(), + [(300, 250), (300, 600)], + "both pages' sizes should survive" + ); + assert_eq!(table.slot_count(), 1, "one div stem is one slot"); + } + + #[test] + fn divergent_unit_paths_are_preserved_as_separate_rows() { + // This divergence is the entire signal template inference reads. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.unit_paths().into_iter().collect::>(), + ["/123/site/home", "/123/site/news"], + "both observed unit paths must be retained" + ); + assert_eq!( + slot.paths().into_iter().collect::>(), + ["/", "/news/story"] + ); + } + + #[test] + fn repeated_identical_observations_collapse() { + let mut table = EvidenceTable::default(); + let observed = page(&[("/123/site/home", "ad-header", &[(728, 90)])], false); + table.fold_page("/", &observed); + table.fold_page("/", &observed); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!(slot.rows.len(), 1, "the same page twice is one observation"); + } + + #[test] + fn prebid_is_sticky_once_any_page_shows_it() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], true), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert!( + slot.has_prebid, + "a slot proven to run prebid on any page runs prebid" + ); + } + + #[test] + fn slots_keep_first_seen_order_not_alphabetical_order() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page( + &[ + ("/123/site/home", "zeta-slot", &[(728, 90)]), + ("/123/site/home", "alpha-slot", &[(300, 250)]), + ], + false, + ), + ); + + let ids: Vec<&str> = table.slots().map(|slot| slot.div_id.as_str()).collect(); + assert_eq!( + ids, + ["zeta-slot", "alpha-slot"], + "generated config should follow crawl order" + ); + } + + #[test] + fn conflicting_network_ids_are_a_hard_error() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/111/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/222/site/news", "ad-header", &[(728, 90)])], false), + ); + + let error = table + .network_id() + .expect_err("two networks in one crawl should not resolve"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("111") && rendered.contains("222"), + "the error should name both observed ids, got {rendered}" + ); + } + + #[test] + fn agreeing_network_ids_resolve_to_one_value() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + assert_eq!( + table.network_id().expect("agreeing ids should resolve"), + Some("123".to_string()) + ); + } + + #[test] + fn pages_without_slots_are_recorded_for_challenge_detection() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page("/blocked", &page(&[], false)); + + assert_eq!( + table + .empty_pages() + .iter() + .map(String::as_str) + .collect::>(), + ["/blocked"], + "a slot-less page must be visible to the caller, not silently dropped" + ); + assert_eq!( + table.pages().len(), + 2, + "every folded page should be counted" + ); + } + + #[test] + fn empty_table_resolves_no_network_id_rather_than_erroring() { + let table = EvidenceTable::default(); + + assert!(table.is_empty()); + assert_eq!(table.network_id().expect("empty is not a conflict"), None); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 8d2491801..649a57047 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -2,6 +2,7 @@ mod analyzer; pub(crate) mod browser_collector; pub(crate) mod collector; mod crawl_plan; +mod evidence; mod gpt_slots; mod slot_toml; mod validate; From f32a1acf5d175d2fcc005eb4ac6a4e57043d3ccd Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:12:26 +0530 Subject: [PATCH 185/195] Infer section ad-unit templates from cross-page evidence Adds the inference that turns literal scraped ad-unit paths into a `{network_id}`/`{section}` template plus the section policy it depends on. Nothing calls this yet. A wrong template makes a publisher bid against inventory that does not exist, which is worse than a narrow literal path, so this refuses rather than guesses. Three rules carry that: - `{network_id}` binds positionally to unit segment 0 and only when that segment already equals the resolved id. Substring replacement would rewrite `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. - Exactly one unit segment may vary. Zero proves nothing and stays literal; two means the unit tracks a dimension the request path cannot supply, such as a device or geo split, and is refused with that reason. - Two pages must witness both a different derived section and a different unit segment before anything is templated. Round-trip verification cannot supply this: a single observation is reproduced equally well by a literal path, a `{network_id}`-only template, and a `{section}` template, so only variation distinguishes them. `section_segment` is chosen by partitioning observations into pages that have a section segment and pages that do not, the latter fixing `section_root`. An index that cannot be witnessed is rejected, an unwitnessed root leaves the path literal rather than guessing, and two indices that both fit are ambiguous and template nothing. Every accepted template is then replayed through the runtime's own `render_gam_unit_path` and `derive_section` against every observation, so a section slug the path cannot reproduce is caught and downgraded. `derive_section` becomes public for exactly this: the check has to use the runtime's derivation rather than a second implementation that could drift from it. --- .../src/commands/audit/generate/mod.rs | 1 + .../commands/audit/generate/unit_template.rs | 841 ++++++++++++++++++ .../src/creative_opportunities.rs | 6 +- 3 files changed, 847 insertions(+), 1 deletion(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 649a57047..c90235649 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -5,6 +5,7 @@ mod crawl_plan; mod evidence; mod gpt_slots; mod slot_toml; +mod unit_template; mod validate; use std::collections::BTreeSet; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs new file mode 100644 index 000000000..790a89859 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -0,0 +1,841 @@ +//! Infers a `{network_id}`/`{section}` ad-unit template from observed evidence. +//! +//! The generator otherwise writes the literal path each page happened to +//! request, which pins a slot to the one section it was scraped from. A template +//! generalizes across sections — but a *wrong* template makes the publisher bid +//! against inventory that does not exist, which is worse than a narrow literal. +//! So this module is built to refuse rather than guess. +//! +//! Three rules do the load-bearing work: +//! +//! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only +//! if that segment is the resolved network id. Substring replacement would +//! corrupt `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. +//! 2. **Exactly one varying segment.** Zero means nothing was proven and the +//! path stays literal; two means the unit varies along a dimension the +//! request path cannot supply (device, geo, experiment), so it is refused. +//! 3. **The witness rule.** Two pages must show *different* derived sections +//! *and* different unit segments. Without it a single-page crawl is +//! indistinguishable from a static path — literal, `{network_id}`-only and +//! `{section}` all reproduce one observation equally well, and round-trip +//! verification cannot tell them apart. Only variation can. +//! +//! Every accepted template is then replayed through the runtime's own +//! [`render_gam_unit_path`](CreativeOpportunitySlot::render_gam_unit_path) and +//! [`derive_section`] against every observation. A template that does not +//! reproduce what the live page actually requested is downgraded, not written. + +#![allow( + dead_code, + reason = "inference is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; + +use super::evidence::{EvidenceTable, SlotEvidence}; +use super::slot_toml::toml_string; + +/// Candidate `section_segment` values considered, `0..=MAX_SECTION_SEGMENT`. +/// +/// A locale-prefixed site (`/en/news/story`) needs 1. Beyond 2 the "section" is +/// no longer a taxonomy the operator would recognise, and every extra candidate +/// is another chance for two indices to both fit and force a refusal. +const MAX_SECTION_SEGMENT: usize = 2; + +/// The config-level section policy an inferred template depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SectionPolicy { + /// Value substituted for `{section}` on paths with no section segment. + pub(super) section_root: String, + /// Index of the path segment `{section}` is taken from. + pub(super) section_segment: usize, +} + +/// What to write for one slot's `gam_unit_path`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum SlotDecision { + /// Write this templated path; it reproduced every observation. + Template(String), + /// Write this literal path; nothing generalizable was proven. + Literal(String), + /// Write no path at all — the observations cannot be represented. + Refuse { + /// Operator-facing explanations, one per reason. + reasons: Vec, + }, +} + +/// The outcome of inference across the whole evidence table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct InferenceOutcome { + /// Section policy to write, present only when some slot templated. + pub(super) policy: Option, + /// Per-slot decision, keyed by div stem, in evidence order. + pub(super) decisions: Vec<(String, SlotDecision)>, + /// Operator-facing notes about why inference went the way it did. + pub(super) diagnostics: Vec, +} + +impl InferenceOutcome { + /// The decision for a slot, by div stem. + pub(super) fn decision(&self, div_id: &str) -> Option<&SlotDecision> { + self.decisions + .iter() + .find(|(key, _)| key == div_id) + .map(|(_, decision)| decision) + } +} + +/// Per-slot analysis under one candidate `section_segment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SlotAnalysis { + /// Templatable: unit segment `varying` tracks the derived section, and root + /// pages agreed on `section_root`. + Templatable { + varying: usize, + section_root: String, + }, + /// The unit path never varied, so nothing about `{section}` was proven. + Static, + /// Cannot be represented; carries the operator-facing reason. + Refuse(String), + /// Would be templatable but no root page was observed, so `section_root` + /// is undetermined under this candidate. + RootUnwitnessed, +} + +/// Infers unit-path templates for every slot in `table`. +/// +/// `network_id` is the resolved GAM network id; `{network_id}` is only ever +/// bound to a unit segment that already equals it. +pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> InferenceOutcome { + let slots: Vec<&SlotEvidence> = table.slots().collect(); + let mut diagnostics = Vec::new(); + + // Evaluate every candidate index independently; ambiguity between two that + // both fit is a refusal, not a preference for the smaller one. + let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + for segment in 0..=MAX_SECTION_SEGMENT { + let analyses: BTreeMap = slots + .iter() + .map(|slot| (slot.div_id.clone(), analyse_slot(slot, network_id, segment))) + .collect(); + + let roots: BTreeSet<&str> = analyses + .values() + .filter_map(|analysis| match analysis { + SlotAnalysis::Templatable { section_root, .. } => Some(section_root.as_str()), + _ => None, + }) + .collect(); + // Slots must agree: `section_root` is one config-level value, so two + // slots claiming different roots means this index is not the real one. + let Some(root) = roots.iter().next().copied() else { + continue; + }; + if roots.len() > 1 { + continue; + } + if !witnessed(&slots, &analyses, segment) { + continue; + } + qualifying.push((segment, root.to_string(), analyses)); + } + + let chosen = match qualifying.len() { + 0 => None, + 1 => qualifying.into_iter().next(), + _ => { + let indices: Vec = qualifying + .iter() + .map(|(segment, _, _)| segment.to_string()) + .collect(); + diagnostics.push(format!( + "more than one section_segment ({}) explains the observed ad-unit paths \ + equally well, so no template can be chosen safely; keeping literal paths", + indices.join(", ") + )); + None + } + }; + + let Some((section_segment, section_root, analyses)) = chosen else { + if diagnostics.is_empty() { + diagnostics.push( + "no ad-unit path varied by page section across the crawl, so paths were kept \ + literal; crawl more sections to enable a {section} template" + .to_string(), + ); + } + return InferenceOutcome { + policy: None, + decisions: literal_decisions(&slots), + diagnostics, + }; + }; + + let mut decisions = Vec::with_capacity(slots.len()); + let mut templated = 0_usize; + for slot in &slots { + let analysis = analyses + .get(&slot.div_id) + .cloned() + .unwrap_or(SlotAnalysis::Static); + let decision = match analysis { + SlotAnalysis::Templatable { varying, .. } => { + let template = build_template(slot, varying); + match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) + { + Ok(()) => { + templated += 1; + SlotDecision::Template(template) + } + Err(reason) => { + diagnostics.push(format!( + "slot `{}` template `{template}` did not reproduce the observed \ + ad-unit paths ({reason}); keeping the literal path", + slot.id + )); + literal_decision(slot) + } + } + } + SlotAnalysis::Static | SlotAnalysis::RootUnwitnessed => literal_decision(slot), + SlotAnalysis::Refuse(reason) => SlotDecision::Refuse { + reasons: vec![reason], + }, + }; + decisions.push((slot.div_id.clone(), decision)); + } + + if templated == 0 { + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + }; + } + + diagnostics.push(format!( + "inferred section_segment = {section_segment} and section_root = \"{section_root}\" \ + from {} page(s); {templated} slot(s) templated", + table.pages().len() + )); + InferenceOutcome { + policy: Some(SectionPolicy { + section_root, + section_segment, + }), + decisions, + diagnostics, + } +} + +/// Whether the accepted analyses actually witnessed section variation. +/// +/// Requires two rows with both a different derived section and a different +/// value in the varying unit segment. Round-trip verification cannot supply +/// this: one observation is reproduced equally well by a literal path, a +/// `{network_id}`-only template, and a `{section}` template. +fn witnessed( + slots: &[&SlotEvidence], + analyses: &BTreeMap, + section_segment: usize, +) -> bool { + for slot in slots { + let Some(SlotAnalysis::Templatable { + varying, + section_root, + }) = analyses.get(&slot.div_id) + else { + continue; + }; + let mut sections = BTreeSet::new(); + let mut units = BTreeSet::new(); + for row in &slot.rows { + sections.insert(derive_section(&row.path, section_root, section_segment)); + if let Some(value) = segment_at(&row.unit_path, *varying) { + units.insert(value.to_string()); + } + } + if sections.len() >= 2 && units.len() >= 2 { + return true; + } + } + false +} + +/// Checks the properties of a slot's observations that do not depend on which +/// `section_segment` is being considered. +/// +/// Kept separate because these refusals are final: no candidate index can +/// rescue a slot whose observations are not one template with a single hole in +/// them, and the operator needs the specific reason rather than a generic one. +/// +/// Returns the single varying unit segment, `None` when nothing varied, or the +/// reason the observations cannot be represented at all. +fn structural_check(slot: &SlotEvidence) -> Result, String> { + // One page reporting two different ad-unit paths for the same slot means the + // unit varies along something the request path cannot express — a device or + // geo split, or two profiles disagreeing. Nothing here can represent that. + let mut per_path: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for row in &slot.rows { + per_path + .entry(row.path.as_str()) + .or_default() + .insert(row.unit_path.as_str()); + } + if let Some((path, units)) = per_path.iter().find(|(_, units)| units.len() > 1) { + let observed: Vec<&str> = units.iter().copied().collect(); + return Err(format!( + "page `{path}` requested more than one ad-unit path for this slot ({}); \ + the unit varies by something the request path cannot derive", + observed.join(", ") + )); + } + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + let Some(first) = split.first() else { + return Ok(None); + }; + // Differing shapes are not one template with a hole in it. + if split.iter().any(|parts| parts.len() != first.len()) { + return Err( + "the observed ad-unit paths have different segment counts, so they are not \ + one template" + .to_string(), + ); + } + + let varying: Vec = (0..first.len()) + .filter(|index| { + split + .iter() + .map(|parts| parts[*index]) + .collect::>() + .len() + > 1 + }) + .collect(); + match varying.len() { + 0 => Ok(None), + 1 if varying[0] == 0 => { + Err("the network-id segment of the ad-unit path varied across pages".to_string()) + } + 1 => Ok(Some(varying[0])), + count => Err(format!( + "{count} ad-unit segments vary across pages, so the path does not track the \ + page section alone" + )), + } +} + +/// Analyses one slot under a candidate `section_segment`. +fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) -> SlotAnalysis { + let varying = match structural_check(slot) { + Err(reason) => return SlotAnalysis::Refuse(reason), + Ok(None) => return SlotAnalysis::Static, + Ok(Some(varying)) => varying, + }; + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + // `{network_id}` binds positionally and only to the resolved id. Substring + // replacement would rewrite an unrelated segment that merely contains it. + if split.first().and_then(|parts| parts.first()) != Some(&network_id) { + return SlotAnalysis::Static; + } + + // Partition observations into pages that have a section segment and pages + // that do not; the latter are what determine `section_root`. + let mut root_values = BTreeSet::new(); + for (row, parts) in slot.rows.iter().zip(split.iter()) { + let observed = parts[varying]; + if path_segments(&row.path).len() > section_segment { + // The empty root is unused here: the path has this segment. + if derive_section(&row.path, "", section_segment) != observed { + return SlotAnalysis::Static; + } + } else { + root_values.insert(observed); + } + } + + let mut roots = root_values.into_iter(); + let Some(section_root) = roots.next() else { + // Without a root observation, `section_root` would be a guess that + // silently mis-renders every short path. + return SlotAnalysis::RootUnwitnessed; + }; + if roots.next().is_some() { + return SlotAnalysis::Static; + } + // A root that is not `[A-Za-z0-9_-]+` makes any `{section}` template fail + // config load; catch it here rather than at push time. + if section_root.is_empty() + || !section_root + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + { + return SlotAnalysis::Static; + } + + SlotAnalysis::Templatable { + varying, + section_root: section_root.to_string(), + } +} + +/// Builds the template text by substituting the two proven placeholders. +fn build_template(slot: &SlotEvidence, varying: usize) -> String { + let first = slot + .rows + .iter() + .next() + .map(|row| row.unit_path.as_str()) + .unwrap_or_default(); + let rendered: Vec = segments(first) + .into_iter() + .enumerate() + .map(|(index, value)| { + if index == 0 { + "{network_id}".to_string() + } else if index == varying { + "{section}".to_string() + } else { + value.to_string() + } + }) + .collect(); + format!("/{}", rendered.join("/")) +} + +/// Replays `template` through the runtime renderer against every observation. +/// +/// This is the gate that catches a section slug the path cannot reproduce — a +/// publisher whose `/car-research` pages request `.../carresearch`, say, where +/// the derived section and the observed segment differ. +fn verify_round_trip( + template: &str, + slot: &SlotEvidence, + network_id: &str, + section_root: &str, + section_segment: usize, +) -> Result<(), String> { + let probe = probe_slot(template)?; + for row in &slot.rows { + let section = derive_section(&row.path, section_root, section_segment); + match probe.render_gam_unit_path(network_id, §ion) { + Some(rendered) if rendered == row.unit_path => {} + Some(rendered) => { + return Err(format!( + "on `{}` it renders `{rendered}` but the page requested `{}`", + row.path, row.unit_path + )); + } + None => { + return Err(format!( + "on `{}` it renders past the GAM ad-unit path byte limit", + row.path + )); + } + } + } + Ok(()) +} + +/// Builds a throwaway slot carrying `template`, for rendering only. +/// +/// Deserializing is how the runtime itself builds slots, so this exercises the +/// same template parsing rather than a parallel implementation. +fn probe_slot(template: &str) -> Result { + let document = format!( + "id = \"probe\"\ngam_unit_path = {}\npage_patterns = [\"/\"]\n\ + formats = [{{ width = 1, height = 1 }}]\n", + toml_string(template) + ); + toml::from_str::(&document) + .map_err(|error| format!("template is not representable in config: {error}")) +} + +/// The decision for a slot no template was proven for. +/// +/// A structural refusal wins over the generic "several paths" message, so the +/// operator sees *why* the slot could not be represented (a device split, an +/// extra varying dimension) rather than only that it could not. +fn literal_decision(slot: &SlotEvidence) -> SlotDecision { + if let Err(reason) = structural_check(slot) { + return SlotDecision::Refuse { + reasons: vec![reason], + }; + } + let units = slot.unit_paths(); + let mut found = units.iter(); + match (found.next(), found.next()) { + (Some(only), None) => SlotDecision::Literal((*only).to_string()), + (Some(_), Some(_)) => SlotDecision::Refuse { + reasons: vec![format!( + "the slot used several ad-unit paths ({}) and none generalized, so no \ + single literal path is correct", + units.into_iter().collect::>().join(", ") + )], + }, + _ => SlotDecision::Refuse { + reasons: vec!["no ad-unit path was observed for this slot".to_string()], + }, + } +} + +fn literal_decisions(slots: &[&SlotEvidence]) -> Vec<(String, SlotDecision)> { + slots + .iter() + .map(|slot| (slot.div_id.clone(), literal_decision(slot))) + .collect() +} + +/// Non-empty path segments of an ad-unit path. +fn segments(unit_path: &str) -> Vec<&str> { + unit_path + .split('/') + .filter(|part| !part.is_empty()) + .collect() +} + +/// Non-empty path segments of a request path. +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +/// The ad-unit path segment at `index`, if present. +fn segment_at(unit_path: &str, index: usize) -> Option<&str> { + segments(unit_path).into_iter().nth(index) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// Folds `(path, unit_path)` observations for one div into a table. + fn table_for(div_id: &str, observations: &[(&str, &str)]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, unit_path) in observations { + let registry = vec![CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: div_id.to_string(), + sizes: vec![(728, 90)], + }]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + fn only_decision(outcome: &InferenceOutcome) -> &SlotDecision { + assert_eq!(outcome.decisions.len(), 1, "fixture should have one slot"); + &outcome.decisions[0].1 + } + + #[test] + fn templates_a_section_varying_unit_path() { + // The shape the operator writes by hand today. + let table = table_for( + "ad-header", + &[ + ("/", "/88059007/autoblog/homepage"), + ("/news/story-abc", "/88059007/autoblog/news"), + ("/deals/thing", "/88059007/autoblog/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "88059007"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }) + ); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/autoblog/{section}".to_string()) + ); + } + + #[test] + fn a_single_page_never_templates() { + // Literal, {network_id}-only and {section} all reproduce one observation, + // so only variation can distinguish them. This is the witness rule. + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/news".to_string()) + ); + } + + #[test] + fn a_static_unit_path_across_sections_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/fixed"), + ("/news/story", "/123/site/fixed"), + ("/deals/x", "/123/site/fixed"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None, "nothing varied, so nothing is proven"); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/fixed".to_string()) + ); + } + + #[test] + fn a_device_split_is_refused_rather_than_guessed() { + // Two units for the SAME path: the desktop/mobile cross-check surfaces + // here, and the request path cannot express the difference. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/news/story", "/123/mobile/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!( + "a device split must refuse, got {:?}", + only_decision(&outcome) + ); + }; + assert!( + reasons[0].contains("more than one ad-unit path"), + "reason should name the conflict, got {reasons:?}" + ); + } + + #[test] + fn two_varying_segments_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/deals/x", "/123/mobile/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two varying dimensions must refuse"); + }; + assert!( + reasons[0].contains("segments vary"), + "reason should name the extra dimension, got {reasons:?}" + ); + } + + #[test] + fn a_slug_the_path_cannot_reproduce_stays_literal() { + // `/car-research` requests `.../carresearch`: the derived section and + // the observed segment differ, so the template would render the wrong + // unit. Round-trip verification is what catches this. + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news"), + ("/car-research/x", "/123/site/carresearch"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "a section whose slug is not derivable must not template" + ); + assert!(matches!( + only_decision(&outcome), + SlotDecision::Refuse { .. } + )); + } + + #[test] + fn an_unwitnessed_root_does_not_template() { + // Every crawled page had a section, so `section_root` would be a guess + // that silently mis-renders the homepage. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/site/news"), + ("/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + let SlotDecision::Refuse { .. } = only_decision(&outcome) else { + panic!("two literal paths and no template is not representable as one literal"); + }; + } + + #[test] + fn a_locale_prefixed_site_infers_the_deeper_segment() { + let table = table_for( + "ad-header", + &[ + ("/en", "/123/site/homepage"), + ("/en/news/story", "/123/site/news"), + ("/en/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }), + "the locale prefix should push the section one segment deeper" + ); + } + + #[test] + fn network_id_is_bound_positionally_not_by_substring() { + // `sports123` merely contains the network id; substring replacement + // would corrupt it into `sports{network_id}`. + let table = table_for( + "ad-header", + &[ + ("/", "/123/sports123/homepage"), + ("/news/story", "/123/sports123/news"), + ("/deals/x", "/123/sports123/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/sports123/{section}".to_string()), + "only segment 0 may become {{network_id}}" + ); + } + + #[test] + fn a_unit_path_not_starting_with_the_network_id_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/999/site/homepage"), + ("/news/story", "/999/site/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "segment 0 must equal the resolved network id" + ); + } + + #[test] + fn differing_segment_counts_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news/extra"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("differing shapes are not one template"); + }; + assert!( + reasons[0].contains("segment counts"), + "reason should name the shape mismatch, got {reasons:?}" + ); + } + + #[test] + fn a_static_slot_stays_literal_alongside_a_templated_one() { + let mut table = EvidenceTable::default(); + for (path, section_unit) in [ + ("/", "homepage"), + ("/news/story", "news"), + ("/deals/x", "deals"), + ] { + let registry = vec![ + CollectedGptSlot { + gam_unit_path: format!("/123/site/{section_unit}"), + div_id: "ad-header".to_string(), + sizes: vec![(728, 90)], + }, + CollectedGptSlot { + gam_unit_path: "/123/site/sticky".to_string(), + div_id: "ad-sticky".to_string(), + sizes: vec![(300, 250)], + }, + ]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert!(outcome.policy.is_some(), "the varying slot should template"); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.decision("ad-sticky"), + Some(&SlotDecision::Literal("/123/site/sticky".to_string())), + "a genuinely static slot must not be dragged into the template" + ); + } + + #[test] + fn diagnostics_explain_why_nothing_templated() { + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("crawl more sections")), + "the operator should learn why, got {:?}", + outcome.diagnostics + ); + } +} diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index f63f92d8e..c90c8bd63 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -171,8 +171,12 @@ fn sanitize_section(segment: &str) -> String { /// The path is used **raw** (not percent-decoded) so this stays consistent with /// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the /// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +/// +/// Public so operator tooling that *infers* a `{section}` template from observed +/// ad-unit paths can check its inference against the exact derivation the +/// runtime will perform, rather than reimplementing the sanitization rules. #[must_use] -fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { +pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { match path .split('/') .filter(|segment| !segment.is_empty()) From deced196987f151c94d7246b5e0198a5d82f0261 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:19:02 +0530 Subject: [PATCH 186/195] Let the ad-template writer express section policy and per-section patterns Two gaps between what inference produces and what the writer could put on disk. Nothing calls the new code yet. `page_patterns` expands the paths a slot was observed on into globs. Each witnessed section contributes a pair, because one glob cannot cover both halves: `*` crosses `/` in this dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing page, and emitting only the star form would silently drop the landing page from the slot. Nothing extrapolates past a witnessed section, so a crawl that never visited `/reviews` never claims it. `replace_key_in_section` can only rewrite a key that is already present, so it could not add `section_root` or `section_segment` to a config that predates them, which is every config a first templated run touches. Add `upsert_key_in_section`, which inserts immediately after the section header so the new key lands in the section's scalar block rather than after a subtable, where TOML would read it as belonging to that subtable instead. `splice_creative_slots` now takes the section keys as a struct rather than a bare network id. It omits `section_root` and `section_segment` entirely unless a slot actually templated: both are `deny_unknown_fields` additions, so writing them into a config that does not need them would make it unloadable by an older binary for no benefit. --- .../src/commands/audit/generate/mod.rs | 10 +- .../commands/audit/generate/page_patterns.rs | 137 +++++++++ .../src/commands/audit/generate/slot_toml.rs | 275 ++++++++++++++++-- 3 files changed, 391 insertions(+), 31 deletions(-) create mode 100644 crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index c90235649..8c1e793ee 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod collector; mod crawl_plan; mod evidence; mod gpt_slots; +mod page_patterns; mod slot_toml; mod unit_template; mod validate; @@ -546,7 +547,14 @@ pub(crate) fn run_update_slots( replace, ); let rendered_slots = render_slots(&merged); - let updated = splice_creative_slots(&existing, network_id.as_deref(), &rendered_slots)?; + let updated = splice_creative_slots( + &existing, + &slot_toml::CreativeSectionKeys { + network_id: network_id.as_deref(), + ..slot_toml::CreativeSectionKeys::default() + }, + &rendered_slots, + )?; // Everything above is derived from a live, page-controlled ad stack, so the // candidate has to clear the runtime's own load path before it can replace diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs new file mode 100644 index 000000000..70b18a6ae --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -0,0 +1,137 @@ +//! Derives `page_patterns` globs from the paths a slot was actually observed on. +//! +//! A slot seen on `/news/story-abc` should serve every article in that section, +//! not just that one URL — but nothing here extrapolates beyond a *witnessed* +//! section. Each observed path contributes the section prefix it belongs to and +//! nothing else, so a crawl that never visited `/reviews` never claims it. +//! +//! Each section yields a pair, because one glob cannot cover both halves: +//! `*` crosses `/` in this glob dialect, so `/news/*` matches `/news/a/b` but +//! **not** the bare `/news` landing page. Emitting only the star form silently +//! drops the landing page from the slot. + +#![allow( + dead_code, + reason = "expansion is exercised by tests until run_update_slots orchestrates the crawl" +)] + +use std::collections::BTreeSet; + +/// The root pattern, matching only the site root. +const ROOT_PATTERN: &str = "/"; + +/// Expands observed page paths into the glob set a slot should carry. +/// +/// `section_segment` is the index the section is taken from, matching the +/// config key of the same name: a path is reduced to its first +/// `section_segment + 1` segments, which is the prefix every page of that +/// section shares. Paths shorter than that are root pages and contribute `/`. +/// +/// Results are deduplicated and ordered with `/` first, then alphabetically, so +/// re-running against unchanged evidence produces an unchanged file. +pub(super) fn patterns_for_paths<'a>( + paths: impl IntoIterator, + section_segment: usize, +) -> Vec { + let mut patterns: BTreeSet = BTreeSet::new(); + let mut has_root = false; + + for path in paths { + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.len() <= section_segment { + has_root = true; + continue; + } + let prefix = format!("/{}", segments[..=section_segment].join("/")); + // The landing page and everything beneath it. + patterns.insert(prefix.clone()); + patterns.insert(format!("{prefix}/*")); + } + + let mut out = Vec::with_capacity(patterns.len() + usize::from(has_root)); + if has_root { + out.push(ROOT_PATTERN.to_string()); + } + out.extend(patterns); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_section_article_yields_both_halves_of_the_pair() { + // `/news/*` alone would not match the bare `/news` landing page, because + // `*` crosses `/` but does not match the empty remainder. + let patterns = patterns_for_paths(["/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"]); + } + + #[test] + fn the_root_path_contributes_the_root_pattern_first() { + let patterns = patterns_for_paths(["/deals/x", "/", "/news/y"], 0); + + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "root first, then sections alphabetically" + ); + } + + #[test] + fn a_landing_page_and_its_article_collapse_to_one_pair() { + let patterns = patterns_for_paths(["/news", "/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"], "no duplicate entries"); + } + + #[test] + fn a_locale_prefixed_site_keeps_the_locale_in_the_prefix() { + // section_segment = 1 means the section is the second segment, so the + // shared prefix every page of that section carries includes the locale. + let patterns = patterns_for_paths(["/en/news/story", "/en/deals/x", "/en"], 1); + + assert_eq!( + patterns, + ["/", "/en/deals", "/en/deals/*", "/en/news", "/en/news/*"] + ); + } + + #[test] + fn unwitnessed_sections_are_never_invented() { + let patterns = patterns_for_paths(["/news/story"], 0); + + assert_eq!( + patterns, + ["/news", "/news/*"], + "only the crawled section may appear" + ); + } + + #[test] + fn output_is_stable_regardless_of_input_order() { + let one = patterns_for_paths(["/news/a", "/deals/b", "/"], 0); + let two = patterns_for_paths(["/", "/deals/b", "/news/a"], 0); + + assert_eq!(one, two, "re-running should not reorder the written file"); + } + + #[test] + fn every_emitted_pattern_compiles_as_a_runtime_glob() { + let patterns = patterns_for_paths(["/", "/news/story", "/car-research/x"], 0); + + for pattern in &patterns { + trusted_server_core::creative_opportunities::compile_page_pattern(pattern) + .unwrap_or_else(|error| { + panic!("emitted pattern `{pattern}` must compile: {error}") + }); + } + } + + #[test] + fn no_paths_yield_no_patterns() { + assert!(patterns_for_paths([], 0).is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index 880ce1cac..a0fc51da7 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -337,11 +337,50 @@ fn toml_inline_value(value: &serde_json::Value) -> String { /// /// If the config has no `[creative_opportunities]` section, a fresh one is /// appended so `generate` works against a config that omits it. +/// The config-level values a splice writes alongside the slot array. +#[derive(Debug, Clone, Default)] +pub(super) struct CreativeSectionKeys<'a> { + /// GAM network id, when one was resolved. + pub(super) network_id: Option<&'a str>, + /// `section_root`, written only when a slot uses a `{section}` template. + pub(super) section_root: Option<&'a str>, + /// `section_segment`, written only alongside `section_root`. + pub(super) section_segment: Option, +} + +impl CreativeSectionKeys<'_> { + /// The `key = value` lines this policy contributes, in config order. + fn lines(&self) -> Vec<(&'static str, String)> { + let mut out = Vec::new(); + if let Some(network_id) = self.network_id { + out.push(( + "gam_network_id", + format!("gam_network_id = {}", toml_string(network_id)), + )); + } + // Both keys are omitted unless a template needs them. They are + // `deny_unknown_fields` additions, so writing them into a config that + // does not need them would make it unloadable by an older binary for no + // benefit. + if let Some(section_root) = self.section_root { + out.push(( + "section_root", + format!("section_root = {}", toml_string(section_root)), + )); + if let Some(segment) = self.section_segment { + out.push(("section_segment", format!("section_segment = {segment}"))); + } + } + out + } +} + pub(super) fn splice_creative_slots( existing: &str, - network_id: Option<&str>, + keys: &CreativeSectionKeys<'_>, rendered_slots: &str, ) -> CliResult { + let network_id = keys.network_id; let rendered = rendered_slots.trim_matches('\n'); let existing = remove_inline_slot_value(existing)?; @@ -378,28 +417,27 @@ pub(super) fn splice_creative_slots( `gam_network_id` to the config and re-run", ); }; + let _ = network_id; let mut result = existing; if !result.is_empty() && !result.ends_with('\n') { result.push('\n'); } result.push_str("\n[creative_opportunities]\n"); - result.push_str(&format!("gam_network_id = {}\n", toml_string(network_id))); + for (_, line) in keys.lines() { + result.push_str(&line); + result.push('\n'); + } result.push_str(rendered); result.push('\n'); return Ok(result); } - // Section exists — update `gam_network_id` (best-effort) and replace slots. + // Section exists — set the scalar keys, then replace the slot array. + // `upsert` rather than `replace`: `section_root`/`section_segment` are new + // keys that a config predating templating simply does not have. let mut document = existing.clone(); - if let Some(network_id) = network_id - && let Ok(updated) = replace_key_in_section( - &document, - "creative_opportunities", - "gam_network_id", - &format!("gam_network_id = {}", toml_string(network_id)), - ) - { - document = updated; + for (key, line) in keys.lines() { + document = upsert_key_in_section(&document, "creative_opportunities", key, &line)?; } let lines: Vec<&str> = document.lines().collect(); @@ -594,6 +632,51 @@ pub(super) fn replace_key_in_section( Ok(output) } +/// Sets `key` in `section`, replacing an existing assignment or inserting one. +/// +/// [`replace_key_in_section`] can only rewrite a key that is already present, so +/// it cannot add `section_root` or `section_segment` to a config that predates +/// them — which is every config a first templated run touches. This inserts +/// immediately after the section header instead, keeping the new key inside the +/// section's scalar block rather than stranding it after a subtable, where TOML +/// would read it as belonging to that subtable. +/// +/// # Errors +/// +/// Returns an error when `section` is not present in the document. +pub(super) fn upsert_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + if let Ok(replaced) = replace_key_in_section(document, section, key, replacement_line) { + return Ok(replaced); + } + + let section_header = format!("[{section}]"); + let Some(header_index) = document + .lines() + .position(|line| is_table_header(line, §ion_header)) + else { + return cli_error(format!( + "failed to update config because section `{section_header}` was not found" + )); + }; + + let mut lines: Vec = document.lines().map(str::to_string).collect(); + lines.insert(header_index + 1, replacement_line.to_string()); + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + fn is_key_line(trimmed_line: &str, key: &str) -> bool { trimmed_line .strip_prefix(key) @@ -643,6 +726,14 @@ mod tests { render_slots(&merged) } + /// Section keys carrying only a network id, the common test case. + fn network_keys(network_id: &str) -> CreativeSectionKeys<'_> { + CreativeSectionKeys { + network_id: Some(network_id), + ..CreativeSectionKeys::default() + } + } + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { toml::from_str::(toml_str).expect("valid creative config") } @@ -656,7 +747,7 @@ mod tests { formats = [{ width = 300, height = 250 }]\n\n\ [auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); assert!( @@ -686,7 +777,7 @@ mod tests { // produce a document that no longer parses. let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; - let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect_err("should refuse an unrecognised section form"); assert!( @@ -699,7 +790,7 @@ mod tests { fn splice_rejects_top_level_inline_creative_opportunities_table() { let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; - let error = splice_creative_slots(existing, Some("222"), &header_rendered()) + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect_err("should refuse a top-level inline table"); assert!( @@ -708,6 +799,126 @@ mod tests { ); } + /// Section keys for a templated run: network id plus the section policy. + fn template_keys<'a>( + network_id: &'a str, + root: &'a str, + segment: usize, + ) -> CreativeSectionKeys<'a> { + CreativeSectionKeys { + network_id: Some(network_id), + section_root: Some(root), + section_segment: Some(segment), + } + } + + #[test] + fn splice_inserts_section_policy_keys_a_config_does_not_have_yet() { + // The whole point of `upsert`: every config predating templating lacks + // these keys, so a replace-only writer could never add them. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "inserting must not disturb later sections" + ); + } + + #[test] + fn splice_replaces_section_policy_keys_that_are_already_present() { + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\ + section_root = \"old\"\nsection_segment = 2\n"; + + let out = splice_creative_slots( + existing, + &template_keys("111", "homepage", 1), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(1)); + assert_eq!( + out.matches("section_root").count(), + 1, + "the key must be replaced, not duplicated" + ); + } + + #[test] + fn splice_omits_section_policy_when_no_slot_needs_it() { + // `section_root`/`section_segment` are `deny_unknown_fields` additions: + // writing them into a config that does not need them would make it + // unloadable by an older binary for no benefit. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.contains("section_root") && !out.contains("section_segment"), + "an untemplated run must not add rollback-fatal keys, got:\n{out}" + ); + } + + #[test] + fn splice_writes_section_policy_into_a_freshly_created_section() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + } + + #[test] + fn upsert_keeps_an_inserted_key_inside_the_section_scalar_block() { + // Appending at the end of the section would land the key after a + // subtable, where TOML reads it as part of that subtable instead. + let document = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [[creative_opportunities.slot]]\nid = \"a\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 1, height = 1 }]\n"; + + let out = upsert_key_in_section( + document, + "creative_opportunities", + "section_root", + "section_root = \"homepage\"", + ) + .expect("should insert"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["section_root"].as_str(), + Some("homepage"), + "the key must belong to the section, not the slot subtable" + ); + } + #[test] fn splice_refuses_fresh_section_without_a_network_id() { // Reachable whenever the scraped unit path has no all-digit leading @@ -716,8 +927,12 @@ mod tests { // route to the startup error router once pushed. let existing = "[publisher]\ndomain = \"x\"\n"; - let error = splice_creative_slots(existing, None, &header_rendered()) - .expect_err("should refuse to create a section with no network id"); + let error = splice_creative_slots( + existing, + &CreativeSectionKeys::default(), + &header_rendered(), + ) + .expect_err("should refuse to create a section with no network id"); assert!( format!("{error:?}").contains("without a GAM network id"), @@ -729,7 +944,7 @@ mod tests { fn splice_appends_section_when_config_has_none() { let existing = "[publisher]\ndomain = \"x\"\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should append a fresh section"); let value = toml::from_str::(&out).expect("appended config is valid TOML"); @@ -771,7 +986,7 @@ mod tests { false, ); - let out = splice_creative_slots(existing, Some("111"), &render_slots(&merged)) + let out = splice_creative_slots(existing, &network_keys("111"), &render_slots(&merged)) .expect("should splice"); let value = toml::from_str::(&out).expect("spliced config is valid TOML"); @@ -803,7 +1018,7 @@ mod tests { let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ [auction]\r\nenabled = true\r\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); assert!( @@ -846,7 +1061,7 @@ mod tests { // Config with no [creative_opportunities] at all — generate should append it. let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); let value = toml::from_str::(&out).expect("valid TOML"); @@ -872,14 +1087,14 @@ mod tests { // header comment; it must keep exactly one copy, not append another. let first = splice_creative_slots( "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", - Some("222"), + &network_keys("222"), &header_rendered(), ) .expect("first splice"); - let second = - splice_creative_slots(&first, Some("222"), &header_rendered()).expect("second splice"); - let third = - splice_creative_slots(&second, Some("222"), &header_rendered()).expect("third splice"); + let second = splice_creative_slots(&first, &network_keys("222"), &header_rendered()) + .expect("second splice"); + let third = splice_creative_slots(&second, &network_keys("222"), &header_rendered()) + .expect("third splice"); assert_eq!( third @@ -899,7 +1114,7 @@ mod tests { let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ [auction] # flags\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); assert_eq!( @@ -931,7 +1146,7 @@ mod tests { let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should splice"); let value = toml::from_str::(&out).expect("valid TOML"); @@ -958,7 +1173,7 @@ mod tests { slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ [auction]\nenabled = true\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should replace inline slot array"); let value = toml::from_str::(&out).expect("spliced config should be valid"); @@ -980,7 +1195,7 @@ mod tests { gam_network_id = \"111\"\n\ slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; - let out = splice_creative_slots(existing, Some("222"), &header_rendered()) + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) .expect("should replace inline slot map"); let value = toml::from_str::(&out).expect("spliced config should be valid"); From 6722e199675b7e4e6ca69f82712239c4973a80b7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:30:58 +0530 Subject: [PATCH 187/195] Crawl site sections in ad-template generate and write inferred templates Connects the crawl, evidence, inference and writer pieces: a bare `ts audit ad-templates generate ` now samples the site's sections, reconciles each slot across them, infers a `{section}` ad-unit template where the evidence proves one, and writes the section policy alongside the slots. The flow is collect root, plan the crawl from its links and sitemap, walk the planned pages on one browser, fold each into the evidence table, infer, then merge, render, splice and validate as before. Page patterns now come from the sections a slot was actually seen on, so a slot scraped from one article serves its whole section instead of that single URL. Failure handling follows what the evidence can support. A page that will not collect is reported and skipped, because one blocked page should not discard the sections that worked. But if more than a quarter of crawled pages yield no slots the run refuses outright: that is the signature of bot protection serving challenge interstitials, and writing from it would silently narrow the operator's slot set. Pages disagreeing about the GAM network id is likewise a refusal rather than a guess. A run that templates prints the deploy-ordering contract, because the config it just wrote is not rollback-safe: `section_root` and `section_segment` are `deny_unknown_fields` additions, so an older binary rejects the whole config and serves an error on every route. `--max-pages` and `--max-sections` bound the crawl; `--max-pages 1` restores single-page behavior exactly, and an explicit `--page-pattern` still applies to every slot and skips pattern inference. `run_update_slots` takes a request struct, since a nine-argument signature could not absorb the crawl bounds. Removes `default_page_pattern`, superseded by section-derived patterns, and narrows the single-page `merge_slots` path to test scaffolding. --- .../src/commands/audit/generate/collector.rs | 8 - .../src/commands/audit/generate/crawl_plan.rs | 10 +- .../src/commands/audit/generate/evidence.rs | 5 - .../src/commands/audit/generate/mod.rs | 675 ++++++++++++++---- .../commands/audit/generate/page_patterns.rs | 5 - .../src/commands/audit/generate/slot_toml.rs | 50 ++ .../commands/audit/generate/unit_template.rs | 5 - .../src/commands/audit/mod.rs | 37 +- 8 files changed, 635 insertions(+), 160 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs index 625ac660e..e11cf2634 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -14,10 +14,6 @@ pub(crate) type PageSink<'a> = #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ControlFlow { /// Collect the next target. - #[allow( - dead_code, - reason = "constructed by run_update_slots once it orchestrates the crawl" - )] Continue, /// Stop the crawl without an error (budget reached, challenge rate exceeded). Stop, @@ -50,10 +46,6 @@ pub(crate) trait AuditCollector { /// /// Returns an error when `on_page` does, or when the session itself cannot /// be established. Individual page failures are delivered to `on_page`. - #[allow( - dead_code, - reason = "called by run_update_slots once it orchestrates the crawl" - )] fn collect_pages( &self, targets: &[Url], diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs index 5b959dc4f..4796e91dd 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -14,10 +14,6 @@ //! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), //! which is where in-content slots live, and reveal sections hidden behind a //! navigation overflow menu. -#![allow( - dead_code, - reason = "planner is exercised by tests until run_update_slots orchestrates the crawl" -)] use std::collections::BTreeMap; @@ -58,11 +54,11 @@ const NON_PAGE_EXTENSIONS: &[&str] = &[ /// Bounds on how much of a site a single run will load. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) struct CrawlBudget { +pub(crate) struct CrawlBudget { /// Maximum number of sections to sample. - pub(super) max_sections: usize, + pub(crate) max_sections: usize, /// Maximum number of pages to load in total, including the root. - pub(super) max_pages: usize, + pub(crate) max_pages: usize, } impl Default for CrawlBudget { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index e1b1bf81b..bfa004b94 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -19,11 +19,6 @@ //! div ids carry per-render framework hashes and would otherwise look like a new //! slot on every page. -#![allow( - dead_code, - reason = "table is exercised by tests until run_update_slots orchestrates the crawl" -)] - use std::collections::{BTreeMap, BTreeSet}; use super::gpt_slots::DiscoveredSlots; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 8c1e793ee..4584fd9ba 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -22,14 +22,15 @@ use url::Url; use crate::commands::audit::generate::collector::AuditCollector; use crate::commands::audit::generate::slot_toml::{ - merge_slots, render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, - toml_string, + render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, }; use crate::commands::config::init::EXAMPLE_CONFIG; use crate::error::{CliResult, cli_error, report_error}; use analyzer::{analyze_collected_page, extract_gtm_container_id}; +pub(crate) use crawl_plan::CrawlBudget; + /// Writes `contents` to `path` atomically: a same-directory temp file is /// written and fsynced, then renamed over the target, then the directory entry /// is fsynced. @@ -488,70 +489,103 @@ fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) /// Returns an error when the config cannot be read, the page cannot be /// collected, no slots are discovered, or the config has no /// `[creative_opportunities]` section to update. -#[allow(clippy::too_many_arguments, reason = "cohesive one-shot command entry")] +/// Everything one `ts audit ad-templates generate` invocation needs. +pub(crate) struct UpdateSlotsRequest<'a> { + /// Page URL to start from; also bounds the crawl to its origin. + pub(crate) url: &'a str, + /// Operator config to rewrite in place. + pub(crate) config_path: &'a Path, + /// The config's current `[creative_opportunities]`, when it has one. + pub(crate) existing_creative: Option<&'a CreativeOpportunitiesConfig>, + /// Explicit `--page-pattern` values. When non-empty these apply to every + /// slot and pattern inference is skipped entirely. + pub(crate) page_patterns: &'a [String], + /// Replace existing slots rather than merging into them. + pub(crate) replace: bool, + /// Cookies to carry into the crawl. + pub(crate) cookies: &'a [(String, String)], + /// Print the candidate instead of writing it. + pub(crate) dry_run: bool, + /// Crawl bounds. + pub(crate) budget: crawl_plan::CrawlBudget, +} + +/// Share of crawled pages that may yield no slots before the run is refused. +/// +/// A bot-protection challenge serves an interstitial that loads fine and +/// contains no ad stack, so it looks like a page with no slots. Writing a config +/// from a crawl that was mostly challenges would silently narrow the operator's +/// slot set; refusing is the safer failure. +const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; + +/// Runs `ts audit ad-templates generate`: crawl the site's sections, reconcile +/// what each slot looked like across them, infer a `{section}` ad-unit template +/// where the evidence proves one, and rewrite the config's slot array in place. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the root page cannot be +/// collected, no slots are discovered, too many pages came back empty, the +/// pages disagree about the GAM network id, or the resulting config would not +/// load. pub(crate) fn run_update_slots( - url: &str, - config_path: &Path, - existing_creative: Option<&CreativeOpportunitiesConfig>, - page_patterns: &[String], - replace: bool, - cookies: &[(String, String)], - dry_run: bool, + request: &UpdateSlotsRequest<'_>, collector: &dyn AuditCollector, out: &mut dyn Write, ) -> CliResult<()> { - let target_url = parse_audit_url(url)?; - let existing = fs::read_to_string(config_path).map_err(|error| { + let target_url = parse_audit_url(request.url)?; + let existing = fs::read_to_string(request.config_path).map_err(|error| { report_error(format!( "failed to read config {}: {error}", - config_path.display() + request.config_path.display() )) })?; - let collected = collector.collect_page(&target_url, cookies)?; - let artifact = analyze_collected_page(&collected)?; - let page_has_prebid = artifact - .detected_integrations - .iter() - .any(|integration| integration.id == "prebid"); - let discovered = gpt_slots::discover_gpt_slots( - &collected.gpt_slots, - &collected.network_requests, - page_has_prebid, - ); - if discovered.slots.is_empty() { - return cli_error("no ad-template slots were discovered on the page"); - } + let root = collector.collect_page(&target_url, request.cookies)?; + let root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + fold_collected(&mut table, &root_url, &root)?; - // Patterns for slots seen on this run: the `--page-pattern` values, or the - // audited path when none are given (preserving single-page behavior). The - // default uses the recorded post-redirect URL so it matches the page that - // was actually audited, falling back to the requested URL when the - // recorded final URL is invalid. - let run_patterns: Vec = if page_patterns.is_empty() { - let audited_url = collected.final_url().unwrap_or_else(|_| target_url.clone()); - vec![default_page_pattern(&audited_url)] - } else { - page_patterns.to_vec() - }; - // Reject a pattern the runtime cannot compile before it reaches the file: - // a persisted invalid glob either fails the next config load or is silently - // dropped at pattern-compile time, leaving the slot matching fewer pages - // than the config claims. - validate_page_patterns(&run_patterns)?; + // One page per section is enough: ad slots repeat per section, so the crawl + // is sized by the publisher's taxonomy rather than its catalogue. + let plan = crawl_plan::plan_crawl(&root_url, &root.links, &root.sitemap_locs, request.budget); + notes.extend(plan.notes.iter().cloned()); + crawl_sections(collector, &plan, request.cookies, &mut table, &mut notes)?; - let merged = merge_slots(existing_creative, &discovered, &run_patterns, replace); + if table.is_empty() { + return cli_error("no ad-template slots were discovered on any crawled page"); + } + guard_challenge_rate(&table)?; + + let discovered_network_id = table.network_id()?; let network_id = resolve_network_id( - existing_creative, - discovered.gam_network_id.as_deref(), - replace, + request.existing_creative, + discovered_network_id.as_deref(), + request.replace, ); + + // Templating needs a network id to bind `{network_id}` against; without one + // every path stays literal. + let inference = network_id + .as_deref() + .map(|id| unit_template::infer_unit_templates(&table, id)); + if let Some(outcome) = &inference { + notes.extend(outcome.diagnostics.iter().cloned()); + } + let policy = inference + .as_ref() + .and_then(|outcome| outcome.policy.clone()); + + let slots = build_render_slots(&table, inference.as_ref(), policy.as_ref(), request)?; + let merged = slot_toml::merge_render_slots(request.existing_creative, slots, request.replace); let rendered_slots = render_slots(&merged); let updated = splice_creative_slots( &existing, &slot_toml::CreativeSectionKeys { network_id: network_id.as_deref(), - ..slot_toml::CreativeSectionKeys::default() + section_root: policy.as_ref().map(|policy| policy.section_root.as_str()), + section_segment: policy.as_ref().map(|policy| policy.section_segment), }, &rendered_slots, )?; @@ -560,31 +594,165 @@ pub(crate) fn run_update_slots( // candidate has to clear the runtime's own load path before it can replace // the operator's file. This runs on the dry-run path too — otherwise "the // preview looked fine" would not be evidence that the config loads. - for warning in validate::check_candidate(&updated, &existing)? { - writeln!(out, "warning: {warning}") + notes.extend(validate::check_candidate(&updated, &existing)?); + + for note in ¬es { + writeln!(out, "note: {note}") .map_err(|error| report_error(format!("failed to write command output: {error}")))?; } + if policy.is_some() { + writeln!( + out, + "note: this config now uses a {{section}} ad-unit template. Deploy a \ + template-aware binary BEFORE pushing it, and do not roll that binary \ + back while this config is live — an older binary rejects the whole \ + config and serves an error on every route." + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } - if dry_run { + if request.dry_run { writeln!(out, "{updated}") .map_err(|error| report_error(format!("failed to write preview: {error}")))?; return Ok(()); } - write_file_atomically(config_path, &updated).map_err(|error| { + write_file_atomically(request.config_path, &updated).map_err(|error| { report_error(format!( "failed to write config {}: {error}", - config_path.display() + request.config_path.display() )) })?; writeln!( out, - "Wrote {} slot(s) to {} ({} discovered this run)", + "Wrote {} slot(s) to {} ({} slot(s) seen across {} page(s))", merged.len(), - config_path.display(), - discovered.slots.len(), + request.config_path.display(), + table.slot_count(), + table.pages().len(), ) .map_err(|error| report_error(format!("failed to write command output: {error}"))) } + +/// Discovers a collected page's slots and folds them into `table`. +fn fold_collected( + table: &mut evidence::EvidenceTable, + url: &Url, + collected: &collector::CollectedPage, +) -> CliResult<()> { + let artifact = analyze_collected_page(collected)?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + table.fold_page(url.path(), &discovered); + Ok(()) +} + +/// Walks the planned section pages, folding each into `table`. +/// +/// A page that fails to collect is recorded as a note rather than aborting: on a +/// multi-section crawl one blocked or slow page should not discard the sections +/// that did work. The empty-page guard afterwards catches the case where enough +/// of them failed that the result is untrustworthy. +fn crawl_sections( + collector: &dyn AuditCollector, + plan: &crawl_plan::CrawlPlan, + cookies: &[(String, String)], + table: &mut evidence::EvidenceTable, + notes: &mut Vec, +) -> CliResult<()> { + let targets = plan.targets(); + if targets.is_empty() { + notes.push( + "no additional site sections were discovered, so only the requested page was \ + audited; pass explicit --page-pattern values or more URLs to widen coverage" + .to_string(), + ); + return Ok(()); + } + + let mut fold_error = None; + collector.collect_pages(&targets, cookies, &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + if let Err(error) = fold_collected(table, &final_url, &page) { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => notes.push(format!("skipped `{url}`: {error}")), + } + Ok(collector::ControlFlow::Continue) + })?; + match fold_error { + Some(error) => Err(error), + None => Ok(()), + } +} + +/// Refuses a crawl where too many pages produced no slots. +fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { + let total = table.pages().len(); + let empty = table.empty_pages().len(); + if total == 0 || (empty as f64) <= (total as f64) * MAX_EMPTY_PAGE_SHARE { + return Ok(()); + } + let blocked: Vec<&str> = table.empty_pages().iter().map(String::as_str).collect(); + cli_error(format!( + "{empty} of {total} crawled page(s) produced no ad slots ({}), which usually means \ + bot protection served a challenge instead of the real page. Refusing to write a \ + config from partial evidence; re-run with a valid --cookie for the origin", + blocked.join(", ") + )) +} + +/// Turns the evidence table into slots ready to render. +fn build_render_slots( + table: &evidence::EvidenceTable, + inference: Option<&unit_template::InferenceOutcome>, + policy: Option<&unit_template::SectionPolicy>, + request: &UpdateSlotsRequest<'_>, +) -> CliResult> { + // Explicit `--page-pattern` values are an operator override: they apply to + // every slot and disable inference from observed paths entirely. + let explicit = !request.page_patterns.is_empty(); + if explicit { + validate_page_patterns(request.page_patterns)?; + } + let section_segment = policy.map_or(0, |policy| policy.section_segment); + + let mut slots = Vec::with_capacity(table.slot_count()); + for slot in table.slots() { + let patterns = if explicit { + request.page_patterns.to_vec() + } else { + let derived = page_patterns::patterns_for_paths(slot.paths(), section_segment); + validate_page_patterns(&derived)?; + derived + }; + let unit_path = match inference.and_then(|outcome| outcome.decision(&slot.div_id)) { + Some(unit_template::SlotDecision::Template(template)) => Some(template.clone()), + Some(unit_template::SlotDecision::Literal(path)) => Some(path.clone()), + // Refused: write the slot without a path rather than a wrong one. + Some(unit_template::SlotDecision::Refuse { .. }) | None => None, + }; + slots.push(slot_toml::RenderSlot::from_evidence( + &slot.id, + &slot.div_id, + unit_path, + slot.formats.iter().copied(), + patterns, + slot.has_prebid, + )); + } + Ok(slots) +} /// Rejects any page pattern the runtime's glob compiler would not accept. /// /// Uses [`compile_page_pattern`] so the accepted set is exactly what @@ -609,16 +777,6 @@ fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { )) } -/// The default page pattern for a scraped URL: its path, or `/` for the root. -fn default_page_pattern(target_url: &Url) -> String { - let path = target_url.path(); - if path.is_empty() { - "/".to_string() - } else { - path.to_string() - } -} - #[cfg(test)] mod tests { use std::cell::Cell; @@ -657,6 +815,58 @@ mod tests { } } + /// A collector serving a distinct page per URL, recording the crawl order. + struct SiteCollector { + pages: std::collections::HashMap, + visited: std::cell::RefCell>, + } + + impl SiteCollector { + fn new(pages: Vec<(&str, CollectedPage)>) -> Self { + Self { + pages: pages + .into_iter() + .map(|(url, page)| (url.to_string(), page)) + .collect(), + visited: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl AuditCollector for SiteCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.visited.borrow_mut().push(target_url.to_string()); + self.pages + .get(target_url.as_str()) + .cloned() + .ok_or_else(|| report_error(format!("no fake page for {target_url}"))) + } + } + + /// Builds a page carrying one GPT slot plus same-origin nav links. + fn site_page(url: &str, unit_path: &str, nav_paths: &[&str]) -> CollectedPage { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: "ad-header-0".to_string(), + sizes: vec![(728, 90)], + }]; + page.links = nav_paths + .iter() + .map(|path| collector::CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + }) + .collect(); + page + } + fn collected_page() -> CollectedPage { CollectedPage { requested_url: "https://publisher.example/page".to_string(), @@ -1042,13 +1252,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &[], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1056,10 +1269,19 @@ mod tests { let written = fs::read_to_string(&config_path).expect("should read config"); let value = toml::from_str::(&written).expect("valid TOML"); + let patterns: Vec<&str> = value["creative_opportunities"]["slot"][0]["page_patterns"] + .as_array() + .expect("page_patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern string")) + .collect(); + // Patterns come from the post-redirect path: had the requested `/` been + // used, this would be `["/"]`. They now cover the whole section rather + // than only the one article that happened to be scraped. assert_eq!( - value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), - Some("/news/story"), - "default pattern should use the post-redirect path, not the requested one" + patterns, + ["/news", "/news/*"], + "should derive section patterns from the post-redirect path" ); } @@ -1073,13 +1295,16 @@ mod tests { let mut out = Vec::new(); let error = run_update_slots( - "https://publisher.example/", - &config_path, - None, - &["[".to_string()], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["[".to_string()], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1111,13 +1336,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &["/20**".to_string()], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/20**".to_string()], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1144,13 +1372,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &[], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1193,6 +1424,210 @@ mod tests { ) } + #[test] + fn a_crawl_writes_a_section_template_and_per_section_patterns() { + // The end-to-end payoff: crawl sections, reconcile the slot across them, + // infer `{section}`, and write a config the runtime loads. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ), + ), + ( + "https://publisher.example/deals", + site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &collector, + &mut out, + ) + .expect("should crawl and update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "the unvisited-section fallback should come from the root page" + ); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + let slot = &creative["slot"][0]; + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/{network_id}/site/{section}"), + "the varying segment should become a template" + ); + let patterns: Vec<&str> = slot["page_patterns"] + .as_array() + .expect("patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern")) + .collect(); + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "each witnessed section should contribute both halves of its pair" + ); + + // The whole point of the gate: what was written must actually load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + + let report = String::from_utf8(out).expect("utf8 output"); + assert!( + report.contains("Deploy a template-aware binary BEFORE pushing"), + "a templated config must warn about the rollback contract, got:\n{report}" + ); + } + + #[test] + fn a_crawl_refuses_when_most_pages_are_challenged() { + // Bot protection serves an interstitial that loads fine and has no ad + // stack, so it looks like a page with no slots. Writing from that would + // silently narrow the operator's slot set. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let mut blocked_news = site_page("https://publisher.example/news", "/123456789/x", &nav); + blocked_news.gpt_slots.clear(); + let mut blocked_deals = site_page("https://publisher.example/deals", "/123456789/x", &nav); + blocked_deals.gpt_slots.clear(); + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ("https://publisher.example/news", blocked_news), + ("https://publisher.example/deals", blocked_deals), + ]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &collector, + &mut out, + ) + .expect_err("a mostly-challenged crawl should refuse"); + + assert!( + format!("{error:?}").contains("bot protection"), + "the error should name the likely cause, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused run must leave the config untouched" + ); + } + + #[test] + fn max_pages_one_restores_single_page_behavior() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget { + max_sections: 8, + max_pages: 1, + }, + }, + &collector, + &mut out, + ) + .expect("should update from the single page"); + + assert_eq!( + collector.visited.borrow().len(), + 1, + "max_pages = 1 must not crawl beyond the requested page" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"] + .get("section_root") + .is_none(), + "one page cannot witness a section, so no rollback-fatal key may be written" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["gam_unit_path"].as_str(), + Some("/123456789/site/homepage"), + "a single page keeps the literal path" + ); + } + #[test] fn generated_config_loads_through_the_runtime_settings_path() { // The end-to-end contract: whatever `generate` writes must survive the @@ -1208,13 +1643,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &config_path, - None, - &[], - false, - &[], - false, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1301,13 +1739,16 @@ mod tests { let mut out = Vec::new(); run_update_slots( - "https://publisher.example/", - &loaded.app_config_path, - loaded.settings.creative_opportunities.as_ref(), - &[], - false, - &[], - true, + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + budget: CrawlBudget::default(), + }, &collector, &mut out, ) @@ -1329,16 +1770,4 @@ mod tests { }, ); } - - #[test] - fn default_page_pattern_uses_path_or_root() { - assert_eq!( - default_page_pattern(&Url::parse("https://x/news/story").expect("url")), - "/news/story" - ); - assert_eq!( - default_page_pattern(&Url::parse("https://x/").expect("url")), - "/" - ); - } } diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs index 70b18a6ae..5c8c3fd27 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -10,11 +10,6 @@ //! **not** the bare `/news` landing page. Emitting only the star form silently //! drops the landing page from the slot. -#![allow( - dead_code, - reason = "expansion is exercised by tests until run_update_slots orchestrates the crawl" -)] - use std::collections::BTreeSet; /// The root pattern, matching only the site root. diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs index a0fc51da7..9d01d8dc5 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -10,6 +10,7 @@ use trusted_server_core::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunitySlot, }; +#[cfg(test)] use crate::commands::audit::generate::gpt_slots; use crate::error::{CliResult, cli_error, report_error}; @@ -41,6 +42,11 @@ impl RenderSlot { .to_string() } + /// Builds a slot from one page's discovery. + /// + /// Superseded in production by [`RenderSlot::from_evidence`], which reads + /// cross-page evidence; retained as test scaffolding for the merge cases. + #[cfg(test)] fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { Self { id: slot.id.clone(), @@ -59,6 +65,35 @@ impl RenderSlot { } } + /// Builds a slot from cross-page evidence and the inferred unit path. + /// + /// `gam_unit_path` is `None` when inference refused to represent the slot; + /// the slot is still written so its div and formats are not lost, and the + /// runtime falls back to the default `//` path. + pub(super) fn from_evidence( + id: &str, + div_id: &str, + gam_unit_path: Option, + formats: impl IntoIterator, + page_patterns: Vec, + has_prebid: bool, + ) -> Self { + Self { + id: id.to_string(), + div_id: Some(div_id.to_string()), + gam_unit_path, + page_patterns, + formats: formats + .into_iter() + .map(|(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: has_prebid.then(BTreeMap::new), + } + } + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { Self { id: slot.id.clone(), @@ -109,6 +144,7 @@ fn media_type_label(media_type: &MediaType) -> Option<&'static str> { /// - Otherwise existing slots are preserved (covering other pages / hand-tuned /// fields); a slot re-seen this run has `run_patterns` unioned into its /// `page_patterns`; slots seen only this run are appended. +#[cfg(test)] pub(super) fn merge_slots( existing: Option<&CreativeOpportunitiesConfig>, discovered: &gpt_slots::DiscoveredSlots, @@ -120,7 +156,21 @@ pub(super) fn merge_slots( .iter() .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) .collect(); + merge_render_slots(existing, discovered_slots, replace) +} +/// Merges already-built slots into the existing set. +/// +/// Same reconciliation as [`merge_slots`], but the caller supplies the slots — +/// the crawl path builds them from cross-page evidence rather than from one +/// page's discoveries. A slot re-seen this run keeps its configured fields and +/// gains this run's patterns; a genuinely new slot is appended with a +/// non-colliding id. +pub(super) fn merge_render_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> Vec { let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); if replace || existing_slots.is_empty() { return discovered_slots; diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs index 790a89859..d4874b2b2 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -25,11 +25,6 @@ //! [`derive_section`] against every observation. A template that does not //! reproduce what the live page actually requested is downgraded, not written. -#![allow( - dead_code, - reason = "inference is exercised by tests until run_update_slots orchestrates the crawl" -)] - use std::collections::{BTreeMap, BTreeSet}; use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index e024eb8b1..120ac86be 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -137,6 +137,26 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// cookie) so the origin serves the real page instead of a challenge. #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] pub cookies: Vec<(String, String)>, + /// Maximum site sections to sample. Each contributes a landing page and an + /// article, so this bounds how much of the publisher's taxonomy is covered. + #[arg(long, default_value_t = 8)] + pub max_sections: usize, + /// Maximum pages to load in total, including the requested page. + /// + /// Set to 1 to restore single-page behavior: no crawl, no section + /// discovery, and the audited path as the only page pattern. + #[arg(long, default_value_t = 17)] + pub max_pages: usize, +} + +impl AuditAdTemplatesGenerateArgs { + /// The crawl bounds these arguments describe. + pub(crate) fn budget(&self) -> generate::CrawlBudget { + generate::CrawlBudget { + max_sections: self.max_sections, + max_pages: self.max_pages, + } + } } /// Arguments for `ts audit ad-templates verify ...`. @@ -190,13 +210,16 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { let stdout = std::io::stdout(); let mut out = stdout.lock(); generate::run_update_slots( - gen_args.url.as_str(), - &loaded.app_config_path, - loaded.settings.creative_opportunities.as_ref(), - &gen_args.page_patterns, - gen_args.replace, - &gen_args.cookies, - gen_args.dry_run, + &generate::UpdateSlotsRequest { + url: gen_args.url.as_str(), + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &gen_args.page_patterns, + replace: gen_args.replace, + cookies: &gen_args.cookies, + dry_run: gen_args.dry_run, + budget: gen_args.budget(), + }, &collector, &mut out, ) From d9133f2820e0784b083a6108c6eba7e9ae3655f7 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:37:07 +0530 Subject: [PATCH 188/195] Add device-profile cross-checking to ad-template generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishers routinely serve a different GAM ad unit per device (`/network/desktop/news` against `/network/mobile/news`). A single-profile crawl cannot see that: it infers a template that is correct for the profile it used and silently wrong for every other impression, with nothing in the data to say so. This was the one unmitigated risk in the inference design. `--profiles desktop,mobile` walks every planned page once per profile, each with its own viewport and user agent, folding all of it into one evidence table. The user agent matters as much as the viewport here — ad stacks branch on it, so emulating size alone can still return desktop ad units on a phone-sized page. No new refusal logic was needed. Two profiles disagreeing produce two ad-unit paths for a single page, which is already the structural refusal inference applies to a unit that varies by something the request path cannot derive. The slot is still written, with its div and formats intact, but with no `gam_unit_path`: no path at all is better than one that is wrong on mobile, and the runtime falls back to the default unit rather than bidding on a unit that does not exist. Desktop-only stays the default, so the extra crawl is opt-in. --- .../audit/generate/browser_collector.rs | 126 +++++++++++++++-- .../src/commands/audit/generate/mod.rs | 132 ++++++++++++++++-- .../src/commands/audit/mod.rs | 53 ++++++- 3 files changed, 284 insertions(+), 27 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index b1c504cd5..eead2d49a 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -4,6 +4,7 @@ use std::time::Duration; use chromiumoxide::ArcHttpRequest; use chromiumoxide::browser::{Browser, BrowserConfig}; use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; use futures::StreamExt as _; use serde::Deserialize; use tempfile::TempDir; @@ -32,8 +33,98 @@ const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; const RESOURCE_TIMING_BUFFER_WARNING: &str = "browser resource timing buffer reached its default size; some network assets may be missing"; -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; +/// A device the crawl can emulate. +/// +/// Publishers routinely serve different GAM ad units per device +/// (`/network/desktop/news` vs `/network/mobile/news`). A single-profile crawl +/// cannot see that, so it would infer a template that is right for the profile +/// it used and silently wrong for every other impression. Crawling twice makes +/// the disagreement visible: the two profiles produce two ad-unit paths for the +/// same page, which template inference already treats as unrepresentable. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeviceProfile { + /// A desktop viewport with Chrome's own user agent. + Desktop, + /// A phone viewport with touch and a mobile user agent. + Mobile, +} + +impl DeviceProfile { + /// The operator-facing name, matching the `--profiles` value. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Desktop => "desktop", + Self::Mobile => "mobile", + } + } + + /// Parses a `--profiles` value. + /// + /// # Errors + /// + /// Returns an error naming the accepted values when `raw` is not one. + pub(crate) fn parse(raw: &str) -> Result { + match raw.trim().to_ascii_lowercase().as_str() { + "desktop" => Ok(Self::Desktop), + "mobile" => Ok(Self::Mobile), + other => Err(format!( + "unknown device profile `{other}` (expected desktop or mobile)" + )), + } + } + + /// The viewport to emulate. + fn viewport(self) -> Viewport { + match self { + Self::Desktop => Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + Self::Mobile => Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + } + } + + /// The user agent override, or `None` to keep Chrome's own. + /// + /// Ad stacks branch on the user agent as well as the viewport, so emulating + /// the viewport alone can still yield desktop ad units on a phone-sized page. + fn user_agent(self) -> Option<&'static str> { + match self { + Self::Desktop => None, + Self::Mobile => Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + } + } +} + +/// Collects pages through a local Chrome, emulating one device profile. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct BrowserAuditCollector { + profile: Option, +} + +impl BrowserAuditCollector { + /// A collector emulating `profile`. + #[must_use] + pub(crate) fn with_profile(profile: DeviceProfile) -> Self { + Self { + profile: Some(profile), + } + } +} impl AuditCollector for BrowserAuditCollector { fn collect_page( @@ -50,11 +141,13 @@ impl AuditCollector for BrowserAuditCollector { )) })?; + let profile = self.profile; runtime.block_on(async { let mut collected = None; with_browser( std::slice::from_ref(target_url), cookies, + profile, &mut |_, result| { collected = Some(result); Ok(ControlFlow::Stop) @@ -83,7 +176,7 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(with_browser(targets, cookies, on_page)) + runtime.block_on(with_browser(targets, cookies, self.profile, on_page)) } } @@ -98,6 +191,7 @@ impl AuditCollector for BrowserAuditCollector { async fn with_browser( targets: &[Url], cookies: &[(String, String)], + profile: Option, sink: PageSink<'_>, ) -> CliResult<()> { let chrome_executable = find_browser_executable()?; @@ -110,17 +204,27 @@ async fn with_browser( // cookies and writes what it scrapes into the operator's config, so a // certificate-invalid impersonator could both harvest the session and seed // the config with slots of its choosing. Validate certificates. - let config = BrowserConfig::builder() + let mut builder = BrowserConfig::builder() .chrome_executable(chrome_executable) .user_data_dir(user_data_dir.path()) .new_headless_mode() - .respect_https_errors() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; + .respect_https_errors(); + if let Some(profile) = profile { + let viewport = profile.viewport(); + builder = builder + .window_size(viewport.width, viewport.height) + .viewport(viewport); + if let Some(user_agent) = profile.user_agent() { + // Ad stacks branch on the user agent as well as the viewport, so + // emulating size alone can still return desktop ad units. + builder = builder.arg(format!("--user-agent={user_agent}")); + } + } + let config = builder.build().map_err(|error| { + report_error(format!( + "failed to build Chromium configuration for audit: {error}" + )) + })?; let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { report_error(format!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 4584fd9ba..d31030ec7 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -29,6 +29,7 @@ use crate::error::{CliResult, cli_error, report_error}; use analyzer::{analyze_collected_page, extract_gtm_container_id}; +pub(crate) use browser_collector::DeviceProfile; pub(crate) use crawl_plan::CrawlBudget; /// Writes `contents` to `path` atomically: a same-directory temp file is @@ -530,9 +531,12 @@ const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; /// load. pub(crate) fn run_update_slots( request: &UpdateSlotsRequest<'_>, - collector: &dyn AuditCollector, + collectors: &[(&str, &dyn AuditCollector)], out: &mut dyn Write, ) -> CliResult<()> { + let Some((_, first_collector)) = collectors.first() else { + return cli_error("no device profile was selected to audit with"); + }; let target_url = parse_audit_url(request.url)?; let existing = fs::read_to_string(request.config_path).map_err(|error| { report_error(format!( @@ -541,7 +545,7 @@ pub(crate) fn run_update_slots( )) })?; - let root = collector.collect_page(&target_url, request.cookies)?; + let root = first_collector.collect_page(&target_url, request.cookies)?; let root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); let mut table = evidence::EvidenceTable::default(); let mut notes = Vec::new(); @@ -551,7 +555,31 @@ pub(crate) fn run_update_slots( // is sized by the publisher's taxonomy rather than its catalogue. let plan = crawl_plan::plan_crawl(&root_url, &root.links, &root.sitemap_locs, request.budget); notes.extend(plan.notes.iter().cloned()); - crawl_sections(collector, &plan, request.cookies, &mut table, &mut notes)?; + + // Every profile walks the same pages into the same table. When two profiles + // disagree about a slot's ad-unit path, that shows up as two observations of + // one page, which inference already refuses to represent. + for (index, (label, collector)) in collectors.iter().enumerate() { + if index > 0 { + let repeat = first_collector.collect_page(&root_url, request.cookies); + match repeat { + Ok(page) => fold_collected(&mut table, &root_url, &page)?, + Err(error) => notes.push(format!("skipped `{root_url}` on {label}: {error}")), + } + } + crawl_sections(*collector, &plan, request.cookies, &mut table, &mut notes)?; + } + if collectors.len() > 1 { + notes.push(format!( + "audited {} device profile(s): {}", + collectors.len(), + collectors + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", ") + )); + } if table.is_empty() { return cli_error("no ad-template slots were discovered on any crawled page"); @@ -1262,7 +1290,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update slots"); @@ -1305,7 +1333,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect_err("should reject an invalid glob"); @@ -1346,7 +1374,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should accept a runtime-normalisable pattern"); @@ -1382,7 +1410,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update slots"); @@ -1472,7 +1500,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should crawl and update slots"); @@ -1516,6 +1544,86 @@ mod tests { ); } + #[test] + fn disagreeing_device_profiles_refuse_to_write_a_unit_path() { + // Two profiles serving different ad units for the same page is exactly + // the failure a single-profile crawl cannot see. Writing either path + // would be correct for one device and silently wrong for the other. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news"]; + let desktop = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/desktop/news", + &nav, + ), + ), + ]); + let mobile = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/mobile/news", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + ) + .expect("the run should complete and report the conflict"); + + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert!( + creative.get("section_root").is_none(), + "a device split must not produce a section template" + ); + assert!( + creative["slot"][0].get("gam_unit_path").is_none(), + "no ad-unit path is better than one that is wrong on mobile, got:\n{written}" + ); + // What was written must still load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("a slot without an explicit unit path must still load"); + } + #[test] fn a_crawl_refuses_when_most_pages_are_challenged() { // Bot protection serves an interstitial that loads fine and has no ad @@ -1556,7 +1664,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect_err("a mostly-challenged crawl should refuse"); @@ -1603,7 +1711,7 @@ mod tests { max_pages: 1, }, }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update from the single page"); @@ -1653,7 +1761,7 @@ mod tests { dry_run: false, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should update slots"); @@ -1749,7 +1857,7 @@ mod tests { dry_run: true, budget: CrawlBudget::default(), }, - &collector, + &[("desktop", &collector)], &mut out, ) .expect("should render dry-run update"); diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 120ac86be..b526a4088 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -147,6 +147,15 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// discovery, and the audited path as the only page pattern. #[arg(long, default_value_t = 17)] pub max_pages: usize, + /// Device profiles to audit, comma-separated: `desktop`, `mobile`. + /// + /// Defaults to `desktop`. Publishers often serve different GAM ad units per + /// device, which a single-profile crawl cannot see — it would infer a + /// template correct for the profile it used and silently wrong elsewhere. + /// Passing both crawls each page twice and refuses to write an ad-unit path + /// for any slot where the profiles disagree. + #[arg(long, value_delimiter = ',', default_value = "desktop")] + pub profiles: Vec, } impl AuditAdTemplatesGenerateArgs { @@ -157,6 +166,26 @@ impl AuditAdTemplatesGenerateArgs { max_pages: self.max_pages, } } + + /// The device profiles to audit, deduplicated in the order given. + /// + /// # Errors + /// + /// Returns an error when a name is not a known profile, or when none were + /// given. + pub(crate) fn profiles(&self) -> Result, String> { + let mut profiles: Vec = Vec::new(); + for raw in &self.profiles { + let profile = generate::DeviceProfile::parse(raw)?; + if !profiles.contains(&profile) { + profiles.push(profile); + } + } + if profiles.is_empty() { + return Err("--profiles needs at least one of: desktop, mobile".to_string()); + } + Ok(profiles) + } } /// Arguments for `ts audit ad-templates verify ...`. @@ -206,7 +235,23 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { Some(AuditSubcommand::Page(page_args)) => page::run_page(page_args), Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { let loaded = crate::app_config::load_file_settings(&gen_args.config)?; - let collector = generate::browser_collector::BrowserAuditCollector; + let profiles = gen_args.profiles()?; + let collectors: Vec = profiles + .iter() + .map(|profile| { + generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + }) + .collect(); + let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles + .iter() + .zip(collectors.iter()) + .map(|(profile, collector)| { + ( + profile.label(), + collector as &dyn generate::collector::AuditCollector, + ) + }) + .collect(); let stdout = std::io::stdout(); let mut out = stdout.lock(); generate::run_update_slots( @@ -220,7 +265,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { dry_run: gen_args.dry_run, budget: gen_args.budget(), }, - &collector, + &selected, &mut out, ) } @@ -230,7 +275,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { Some(AuditSubcommand::Generate(generate_args)) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); - let collector = generate::browser_collector::BrowserAuditCollector; + let collector = generate::browser_collector::BrowserAuditCollector::default(); generate::run_generate(generate_args, &collector, &mut out) } None => match &args.legacy_url { @@ -239,7 +284,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { .expect("should build generation args when legacy URL is present"); let stdout = std::io::stdout(); let mut out = stdout.lock(); - let collector = generate::browser_collector::BrowserAuditCollector; + let collector = generate::browser_collector::BrowserAuditCollector::default(); generate::run_generate(&generate_args, &collector, &mut out) } None => Err("provide a URL or a subcommand (`page`, `ad-templates`)".to_string()), From 2ea48e097583b4c4c944bfbbdddac4e8756ce088 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 11:41:14 +0530 Subject: [PATCH 189/195] Document ad-template slot generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ts audit ad-templates generate` had no documentation at all. Cover what the crawl does, what it writes, and the two things an operator cannot discover from the output alone. The first is when the command declines to generalize. A wrong ad-unit template makes a publisher bid against inventory that does not exist, so the command prefers a narrow literal path over a plausible guess, and the table says which situations produce which outcome — including the cases that fail the run outright, such as a crawl where bot protection served mostly challenge pages. The second is deploy ordering. A config carrying `section_root` or `section_segment` is not rollback-safe: a binary predating ad-unit templating rejects those keys, and the rejection fails the whole configuration load rather than just the ad-template section, so every route serves an error. Ship the template-aware binary first, push second, and do not roll back while that config is live. --- docs/guide/cli.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/docs/guide/cli.md b/docs/guide/cli.md index e0baac367..b0aeb612b 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -138,6 +138,135 @@ ts audit generate https://publisher.example --force The legacy `ts audit ` form remains a compatibility alias for artifact generation. New automation should use `ts audit generate `. +## Generate ad-template slots from a live site + +`ts audit ad-templates generate ` discovers the publisher's ad slots and +rewrites the `[creative_opportunities]` slot array in `trusted-server.toml` in +place, preserving every other section and comment. + +```bash +ts audit ad-templates generate https://publisher.example/ +``` + +It samples the site rather than a single page. Ad slots repeat per site +section, so the crawl is sized by the publisher's taxonomy — a dozen sections — +not its catalogue: + +1. Load the requested page and read its links and, from `robots.txt`, its + sitemap. +2. Group both into candidate sections, keeping one landing page and one article + per section. +3. Load those pages, recording each slot's div, sizes, and GAM ad-unit path. +4. Reconcile every slot across the pages it appeared on. +5. Infer a `{section}` ad-unit template if the evidence proves one. +6. Verify the result loads, then write it. + +### What it writes + +Given a site whose ad units track the section, the run produces: + +```toml +[creative_opportunities] +gam_network_id = "99999" +section_root = "homepage" +section_segment = 0 + +[[creative_opportunities.slot]] +id = "ad-header-0" +div_id = "ad-header-0" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/deals", "/deals/*", "/news", "/news/*"] +formats = [{ width = 728, height = 90 }] +``` + +Each section contributes **two** patterns. `*` crosses `/` in this glob +dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing +page; emitting only the star form would drop the landing page from the slot. + +Sizes are unioned across pages, so a format that renders only on articles +survives alongside the homepage's. + +### When it keeps literal paths, and when it refuses + +A wrong ad-unit template makes the publisher bid against inventory that does not +exist, so the command prefers a narrow literal path over a plausible guess. + +| Situation | Result | +| --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/car-research` requesting `.../carresearch`) | Literal path; the round-trip check catches it. | +| No root page was seen, so `section_root` is unknown | Literal path rather than a guessed fallback. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | **No `gam_unit_path` at all** for that slot. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | + +Every run checks that the config it produced still loads before replacing the +file, and `--dry-run` runs the same check — a clean preview is evidence the +config loads, not just that it parses. + +### Bounding and steering the crawl + +```bash +# Cover more of a large site. +ts audit ad-templates generate https://publisher.example/ --max-sections 20 --max-pages 41 + +# Audit exactly one page, as earlier releases did. +ts audit ad-templates generate https://publisher.example/ --max-pages 1 + +# Set the patterns yourself; this disables pattern inference entirely. +ts audit ad-templates generate https://publisher.example/ \ + --page-pattern '/' --page-pattern '/news' --page-pattern '/news/*' + +# Preview without writing. +ts audit ad-templates generate https://publisher.example/ --dry-run +``` + +Re-running merges into the existing slots: a slot seen again keeps its +hand-tuned fields and gains this run's patterns, and a hand-written +`gam_unit_path` template is preserved. `--replace` discards existing slots +instead, which also discards any template you wrote by hand. + +Behind bot protection, pass a valid clearance cookie. The crawl reuses one +browser session, so clearance earned on the first page carries to the rest: + +```bash +ts audit ad-templates generate https://publisher.example/ --cookie 'datadome=' +``` + +### Checking for a device split + +Publishers often serve a different ad unit per device +(`/network/desktop/news` against `/network/mobile/news`). A desktop-only crawl +cannot see that — it infers a template correct for desktop and silently wrong +for every mobile impression. + +```bash +ts audit ad-templates generate https://publisher.example/ --profiles desktop,mobile +``` + +Each page is loaded once per profile. Where the profiles disagree, the slot is +written with its div and formats but **no** `gam_unit_path`, so the runtime +falls back to the default unit rather than bidding on one that does not exist. + +### Deploy ordering for templated config + +> **A config containing `section_root` or `section_segment` is not +> rollback-safe.** These keys are rejected outright by a Trusted Server binary +> that predates ad-unit templating, and the rejection fails the _entire_ +> configuration load — not just the ad-template section — so every route serves +> an error. This is a full-site outage, not a degraded ad stack. + +When a run reports that it wrote a `{section}` template: + +1. Deploy the template-aware binary **first**. +2. Then `ts config push`. +3. Do **not** roll that binary back while the config is live. + +A run that did not template writes neither key, and leaves the config exactly as +rollback-safe as it was. + ### Audit safety defaults Every `ts audit` browser session validates TLS certificates. This matters From 90bc61e426c9db7ac9f73de55911926a50ad04c0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 12:20:27 +0530 Subject: [PATCH 190/195] Report why a crawled page yielded no ad slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live run against a bot-protected site refused with "no slots discovered" and nothing else, because the per-page reasons were collected and then thrown away: `fold_collected` discarded each page's collector warnings, and both refusal paths returned before any note was printed. The guards exist for runs that went wrong, so that is exactly when the reasons matter. Notes are now drained as soon as the crawl finishes, ahead of the refusals, and each page's warnings are attributed to its path. Also name the failure that has no warning of its own. Bot protection commonly answers with 200 and a challenge document rather than a 4xx, so the status check passes, the page settles cleanly, and it simply appears to run no ad stack — indistinguishable from a publisher who genuinely has none, though the operator's next move differs completely. A page carrying almost no scripts and no recognised integrations is now called out as a probable challenge, with the advice to supply a current cookie. Verified against a live protected origin: the run previously reported only that no slots were found; it now identifies the interstitial and says what to do. --- .../src/commands/audit/generate/mod.rs | 74 +++++++++++++++++-- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index d31030ec7..d74a22e71 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -549,7 +549,7 @@ pub(crate) fn run_update_slots( let root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); let mut table = evidence::EvidenceTable::default(); let mut notes = Vec::new(); - fold_collected(&mut table, &root_url, &root)?; + fold_collected(&mut table, &root_url, &root, &mut notes)?; // One page per section is enough: ad slots repeat per section, so the crawl // is sized by the publisher's taxonomy rather than its catalogue. @@ -563,7 +563,7 @@ pub(crate) fn run_update_slots( if index > 0 { let repeat = first_collector.collect_page(&root_url, request.cookies); match repeat { - Ok(page) => fold_collected(&mut table, &root_url, &page)?, + Ok(page) => fold_collected(&mut table, &root_url, &page, &mut notes)?, Err(error) => notes.push(format!("skipped `{root_url}` on {label}: {error}")), } } @@ -581,8 +581,17 @@ pub(crate) fn run_update_slots( )); } + // Emit what the crawl learned before any refusal below can return early. + // The guards exist precisely for runs that went wrong, so that is when the + // per-page reasons matter most. + emit_notes(out, &mut notes)?; + if table.is_empty() { - return cli_error("no ad-template slots were discovered on any crawled page"); + return cli_error(format!( + "no ad-template slots were discovered on any of the {} crawled page(s); \ + see the notes above for what each page reported", + table.pages().len() + )); } guard_challenge_rate(&table)?; @@ -624,10 +633,7 @@ pub(crate) fn run_update_slots( // preview looked fine" would not be evidence that the config loads. notes.extend(validate::check_candidate(&updated, &existing)?); - for note in ¬es { - writeln!(out, "note: {note}") - .map_err(|error| report_error(format!("failed to write command output: {error}")))?; - } + emit_notes(out, &mut notes)?; if policy.is_some() { writeln!( out, @@ -661,13 +667,65 @@ pub(crate) fn run_update_slots( .map_err(|error| report_error(format!("failed to write command output: {error}"))) } +/// A page carrying fewer scripts than this is not a real publisher page. +/// +/// A production page runs dozens: the ad stack, analytics, consent, and the +/// site's own bundles. A bot-protection interstitial runs its own challenge +/// script and little else. +const INTERSTITIAL_SCRIPT_CEILING: usize = 3; + +/// Whether a page that loaded successfully is nonetheless not the real page. +/// +/// Bot protection commonly answers with **200** and a challenge document rather +/// than a 4xx, so status-code checks pass and the page simply appears to have no +/// ad stack. Left unexplained, that is indistinguishable from a publisher who +/// genuinely runs no ads on that page — and the operator's next move is entirely +/// different in each case. +fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { + if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING + || !artifact.detected_integrations.is_empty() + { + return None; + } + Some(format!( + "the page returned successfully but carried only {} script(s) and no recognised \ + integrations, which is the shape of a bot-protection challenge rather than the \ + real page. Supply a current --cookie for the origin", + artifact.js_asset_count + )) +} + +/// Writes and clears the pending notes, so each is reported exactly once. +fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { + for note in notes.drain(..) { + writeln!(out, "note: {note}") + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + Ok(()) +} + /// Discovers a collected page's slots and folds them into `table`. +/// +/// Per-page collector warnings are appended to `notes`. They carry the reason a +/// page came back without slots — a non-2xx main document, a navigation that +/// never settled — which is the difference between "this publisher has no ad +/// stack here" and "bot protection served a challenge". Dropping them leaves +/// the operator with a refusal and no way to act on it. fn fold_collected( table: &mut evidence::EvidenceTable, url: &Url, collected: &collector::CollectedPage, + notes: &mut Vec, ) -> CliResult<()> { + // `analyze_collected_page` already carries the collector's warnings forward, + // so this is the complete set, not a second copy. let artifact = analyze_collected_page(collected)?; + for warning in &artifact.warnings { + notes.push(format!("`{}`: {warning}", url.path())); + } + if let Some(reason) = looks_like_an_interstitial(&artifact) { + notes.push(format!("`{}`: {reason}", url.path())); + } let page_has_prebid = artifact .detected_integrations .iter() @@ -709,7 +767,7 @@ fn crawl_sections( match collected { Ok(page) => { let final_url = page.final_url().unwrap_or_else(|_| url.clone()); - if let Err(error) = fold_collected(table, &final_url, &page) { + if let Err(error) = fold_collected(table, &final_url, &page, notes) { fold_error = Some(error); return Ok(collector::ControlFlow::Stop); } From b8a5e5ca450ff2e7fe550dd2c60b773c22eb4dcc Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 12:24:42 +0530 Subject: [PATCH 191/195] Pace the ad-template crawl and allow a headful browser A live crawl of a bot-protected origin returned the real page for the first request and a challenge for the remaining thirteen. A dead cookie fails on the first page, so that shape points at the session being flagged during the run rather than at the credential. Two contributors, both worth correcting regardless of that diagnosis. The crawl issued its navigations back to back. That is discourteous to the origin on its own terms, and request pacing is among the signals bot protection scores, so an unpaced crawl invites the challenge that empties the rest of the run. `--page-delay-ms` now spaces them, defaulting to 750ms. Headless Chrome is trivially detectable, so an origin that serves the real page to a normal browser may answer the same request headless with a challenge. `--headful` runs a visible browser for the cases where that is the difference. Note that `BrowserConfig` defaults to the *old* headless mode, so simply not requesting new-headless yields a more detectable browser rather than a headful one. Both branches are explicit for that reason. --- .../audit/generate/browser_collector.rs | 73 +++++++++++++++++-- .../src/commands/audit/mod.rs | 18 +++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index eead2d49a..b4ef2badd 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -114,6 +114,10 @@ impl DeviceProfile { #[derive(Debug, Clone, Copy, Default)] pub(crate) struct BrowserAuditCollector { profile: Option, + /// Pause between page loads during a crawl. + page_delay: Duration, + /// Run a visible browser instead of a headless one. + headful: bool, } impl BrowserAuditCollector { @@ -122,6 +126,48 @@ impl BrowserAuditCollector { pub(crate) fn with_profile(profile: DeviceProfile) -> Self { Self { profile: Some(profile), + ..Self::default() + } + } + + /// Sets the pause between page loads. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// both discourteous to the origin and self-defeating: request pacing is one + /// of the signals bot protection scores, so an unpaced crawl invites the + /// challenge that empties the rest of the run. + #[must_use] + pub(crate) fn with_page_delay(mut self, delay: Duration) -> Self { + self.page_delay = delay; + self + } + + /// Runs a visible browser rather than a headless one. + /// + /// Headless Chrome is trivially detectable and is scored heavily by bot + /// protection, so an origin that serves a real page to a normal browser may + /// answer the same request headless with a challenge. + #[must_use] + pub(crate) fn headful(mut self, headful: bool) -> Self { + self.headful = headful; + self + } +} + +/// The browser-session knobs one crawl runs under. +#[derive(Debug, Clone, Copy)] +struct SessionSettings { + profile: Option, + page_delay: Duration, + headful: bool, +} + +impl BrowserAuditCollector { + fn session(self) -> SessionSettings { + SessionSettings { + profile: self.profile, + page_delay: self.page_delay, + headful: self.headful, } } } @@ -141,13 +187,13 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - let profile = self.profile; + let settings = self.session(); runtime.block_on(async { let mut collected = None; with_browser( std::slice::from_ref(target_url), cookies, - profile, + settings, &mut |_, result| { collected = Some(result); Ok(ControlFlow::Stop) @@ -176,7 +222,7 @@ impl AuditCollector for BrowserAuditCollector { )) })?; - runtime.block_on(with_browser(targets, cookies, self.profile, on_page)) + runtime.block_on(with_browser(targets, cookies, self.session(), on_page)) } } @@ -191,9 +237,14 @@ impl AuditCollector for BrowserAuditCollector { async fn with_browser( targets: &[Url], cookies: &[(String, String)], - profile: Option, + settings: SessionSettings, sink: PageSink<'_>, ) -> CliResult<()> { + let SessionSettings { + profile, + page_delay, + headful, + } = settings; let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { report_error(format!( @@ -207,8 +258,15 @@ async fn with_browser( let mut builder = BrowserConfig::builder() .chrome_executable(chrome_executable) .user_data_dir(user_data_dir.path()) - .new_headless_mode() .respect_https_errors(); + // `BrowserConfig` defaults to the *old* headless mode, which is both more + // detectable and less faithful than either alternative — so both branches + // must be explicit. Omitting the call is not the same as running headful. + builder = if headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; if let Some(profile) = profile { let viewport = profile.viewport(); builder = builder @@ -243,6 +301,11 @@ async fn with_browser( // Sitemap discovery is a whole-site fact, so only the first target pays for it. let mut result = Ok(()); for (index, target) in targets.iter().enumerate() { + // Pace the crawl. Back-to-back navigations are both discourteous to the + // origin and a signal bot protection scores against the session. + if index > 0 && !page_delay.is_zero() { + sleep(page_delay).await; + } let collected = collect_page_from_browser(&mut browser, target, cookies, index == 0).await; match sink(target, collected) { Ok(ControlFlow::Continue) => {} diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index b526a4088..fb3472c28 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -156,6 +156,22 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// for any slot where the profiles disagree. #[arg(long, value_delimiter = ',', default_value = "desktop")] pub profiles: Vec, + /// Pause in milliseconds between page loads during the crawl. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// discourteous to the origin, and request pacing is one of the signals bot + /// protection scores, so an unpaced crawl can trigger the challenge that + /// empties the rest of the run. + #[arg(long, default_value_t = 750)] + pub page_delay_ms: u64, + /// Run a visible browser instead of a headless one. + /// + /// Headless Chrome is trivially detectable and scored heavily by bot + /// protection, so an origin that serves the real page to a normal browser + /// may answer the same request headless with a challenge. Requires a desktop + /// session; it opens a real window. + #[arg(long)] + pub headful: bool, } impl AuditAdTemplatesGenerateArgs { @@ -240,6 +256,8 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { .iter() .map(|profile| { generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) + .headful(gen_args.headful) }) .collect(); let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles From 142e384de5f475e315ecd89eba51a2381b755ae3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 15:23:34 +0530 Subject: [PATCH 192/195] Answer consent APIs and report GPT state during ad-template generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live audit of a consent-gated publisher reported no ad slots and gave no way to tell why. Two additions, found by debugging exactly that. Publishers gate slot definition behind their consent platform, and a fresh audit profile has no consent cookie, so the crawl never reaches `googletag.defineSlot` and the page looks like it has no ad stack at all. The audit browser now answers the two IAB interfaces every compliant platform exposes, TCF v2 and US Privacy, installed before any page script runs so the real platform finds them already defined. `gdprApplies: false` avoids fabricating a consent string and matches the signal genuinely out-of-scope traffic carries. `--no-assume-consent` observes the un-consented page instead. When the slot registry comes back empty, the run now reports what GPT actually looked like — whether the library reached `apiReady`, how many queued commands never drained, whether `pubads()` exists, and how many scripts the page ran. An empty registry has several very different causes, and the operator's next move differs for each. Against a local proxy this immediately distinguished "GPT never finished loading" from "this page has no ads", which no amount of re-running could have shown before. --- .../audit/generate/browser_collector.rs | 148 +++++++++++++++++- .../src/commands/audit/mod.rs | 10 ++ 2 files changed, 157 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index b4ef2badd..beb5da93d 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -118,6 +118,8 @@ pub(crate) struct BrowserAuditCollector { page_delay: Duration, /// Run a visible browser instead of a headless one. headful: bool, + /// Answer the consent APIs as a consenting reader. + assume_consent: bool, } impl BrowserAuditCollector { @@ -152,6 +154,15 @@ impl BrowserAuditCollector { self.headful = headful; self } + + /// Answers the IAB consent APIs so a gated ad stack initialises. + /// + /// See [`CONSENT_STUB_SCRIPT`] for what is answered and why. + #[must_use] + pub(crate) fn assume_consent(mut self, assume_consent: bool) -> Self { + self.assume_consent = assume_consent; + self + } } /// The browser-session knobs one crawl runs under. @@ -160,6 +171,7 @@ struct SessionSettings { profile: Option, page_delay: Duration, headful: bool, + assume_consent: bool, } impl BrowserAuditCollector { @@ -168,10 +180,97 @@ impl BrowserAuditCollector { profile: self.profile, page_delay: self.page_delay, headful: self.headful, + assume_consent: self.assume_consent, } } } +/// Answers the consent APIs as a consenting, non-GDPR reader. +/// +/// Publishers gate slot definition behind their consent platform, so a browser +/// with no consent cookie never reaches `googletag.defineSlot` and the audit +/// sees a page with no ad stack. That is indistinguishable from a page that +/// genuinely has none, and it is the state every fresh audit profile starts in. +/// +/// Rather than special-casing each vendor, this answers the two IAB interfaces +/// every compliant platform exposes — TCF v2 (`__tcfapi`) and US Privacy +/// (`__uspapi`) — installed before any page script runs so the real platform +/// finds them already defined. `gdprApplies: false` is used deliberately: it +/// needs no fabricated consent string, and it is the same signal the ad stack +/// receives for genuinely out-of-scope traffic. +/// +/// This makes the audit behave like a consenting reader; it does not alter what +/// the publisher's own readers experience. +const CONSENT_STUB_SCRIPT: &str = r#"(() => { + const tcData = { + tcString: '', + tcfPolicyVersion: 2, + cmpId: 0, + cmpVersion: 1, + gdprApplies: false, + eventStatus: 'tcloaded', + cmpStatus: 'loaded', + listenerId: 1, + isServiceSpecific: true, + useNonStandardTexts: false, + purposeOneTreatment: false, + publisherCC: 'US', + purpose: { consents: {}, legitimateInterests: {} }, + vendor: { consents: {}, legitimateInterests: {} }, + specialFeatureOptins: {}, + }; + for (let index = 1; index <= 10; index += 1) { + tcData.purpose.consents[index] = true; + tcData.purpose.legitimateInterests[index] = true; + } + + const tcfapi = (command, version, callback, parameter) => { + if (typeof callback !== 'function') return; + switch (command) { + case 'ping': + callback({ + gdprApplies: false, + cmpLoaded: true, + cmpStatus: 'loaded', + displayStatus: 'hidden', + apiVersion: '2.0', + cmpId: 0, + }, true); + break; + case 'addEventListener': + case 'getTCData': + callback(tcData, true); + break; + case 'removeEventListener': + callback(true, true); + break; + default: + callback(tcData, true); + } + }; + + const uspapi = (command, version, callback) => { + if (typeof callback !== 'function') return; + callback({ version: 1, uspString: '1---' }, true); + }; + + // Non-writable so the real platform cannot replace these and re-gate the + // page; a failed assignment is the intended outcome. + const pin = (name, value) => { + try { + Object.defineProperty(window, name, { + value, + writable: false, + configurable: false, + }); + } catch (error) { + /* already pinned */ + } + }; + pin('__tcfapi', tcfapi); + pin('__uspapi', uspapi); +})();"#; + impl AuditCollector for BrowserAuditCollector { fn collect_page( &self, @@ -244,6 +343,7 @@ async fn with_browser( profile, page_delay, headful, + assume_consent, } = settings; let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { @@ -306,7 +406,9 @@ async fn with_browser( if index > 0 && !page_delay.is_zero() { sleep(page_delay).await; } - let collected = collect_page_from_browser(&mut browser, target, cookies, index == 0).await; + let collected = + collect_page_from_browser(&mut browser, target, cookies, index == 0, assume_consent) + .await; match sink(target, collected) { Ok(ControlFlow::Continue) => {} Ok(ControlFlow::Stop) => break, @@ -346,11 +448,22 @@ async fn collect_page_from_browser( target_url: &Url, cookies: &[(String, String)], discover_sitemap: bool, + assume_consent: bool, ) -> CliResult { let page = browser.new_page("about:blank").await.map_err(|error| { report_error(format!("failed to create browser page for audit: {error}")) })?; + // Must run before any page script, so the consent platform finds the APIs + // already answered rather than installing its own gate. + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| { + report_error(format!("failed to install the consent stub: {error}")) + })?; + } + // Set operator-supplied cookies before navigating so the origin sees an // authenticated session on the first request. Scoping each to the target URL // lets Chrome infer domain/path. @@ -470,6 +583,19 @@ async fn collect_page_from_browser( // page keeps its link graph in the framework payload, so parsing the raw // HTML finds only a fraction of the site's sections. Best-effort — an empty // list just means crawl planning falls back to other sources. + // When the registry is empty, report what GPT actually looked like. An + // empty registry has several very different causes — the library never + // loaded, it loaded but the command queue never drained, or slots really + // are absent — and the operator's next move differs for each. + if gpt_slots.is_empty() + && let Ok(result) = page.evaluate(GPT_DIAGNOSTIC_SCRIPT).await + && let Ok(state) = result.into_value::() + { + warnings.push(format!( + "no GPT slots in the registry; googletag state: {state}" + )); + } + let links: Vec = match page.evaluate(LINKS_SCRIPT).await { Ok(result) => result.into_value().unwrap_or_default(), Err(_) => Vec::new(), @@ -626,6 +752,26 @@ const SITEMAP_SCRIPT: &str = r#"async () => { return pages.slice(0, 5000); }"#; +/// Reports the observable state of GPT, for pages whose registry came back empty. +const GPT_DIAGNOSTIC_SCRIPT: &str = r#"() => { + const tag = window.googletag; + const count = (() => { + try { return tag.pubads().getSlots().length } catch (error) { return -1 } + })(); + return { + googletag: typeof tag, + api_ready: !!(tag && tag.apiReady), + cmd_pending: tag && tag.cmd && typeof tag.cmd.length === 'number' ? tag.cmd.length : -1, + has_pubads: !!(tag && typeof tag.pubads === 'function'), + slots: count, + tcfapi: typeof window.__tcfapi, + scripts: document.scripts.length, + ts_ad_slots: (() => { + try { return (window.tsjs && window.tsjs.adSlots || []).length } catch (error) { return -1 } + })(), + }; +}"#; + /// Reads the live GPT slot registry into `{gam_unit_path, div_id, sizes}` rows. /// /// Mirrors the ad-template verifier's `getSlots()` scrape: it defends against a diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index fb3472c28..777b09a2e 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -172,6 +172,15 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// session; it opens a real window. #[arg(long)] pub headful: bool, + /// Do not answer the IAB consent APIs on behalf of the audit browser. + /// + /// Publishers gate slot definition behind their consent platform, and a + /// fresh audit profile has no consent cookie, so by default the crawl + /// answers the standard TCF v2 and US Privacy interfaces as a consenting, + /// out-of-scope reader. Without that, such a site reports no ad slots at + /// all. Pass this to observe the un-consented page instead. + #[arg(long)] + pub no_assume_consent: bool, } impl AuditAdTemplatesGenerateArgs { @@ -258,6 +267,7 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { generate::browser_collector::BrowserAuditCollector::with_profile(*profile) .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) .headful(gen_args.headful) + .assume_consent(!gen_args.no_assume_consent) }) .collect(); let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles From f5c861620c0daf3fefd74fd422a88a5eddb72221 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 15:55:59 +0530 Subject: [PATCH 193/195] Audit through a proxy and collapse lowercase React div-id tokens Verified against a live publisher served by `ts dev proxy`, which surfaced two defects that no fixture could. `normalize_div_stem` matched only the uppercase React `_R_` marker. React also emits the lowercase `_r_0_` form client-side, and the token changes on every render, so a slot arrived as `ad-header-0-_r_0_` on one page and `ad-header-0-_r_8_` on the next. One logical slot fragmented into a new key per page: the written `div_id` would never match at runtime, and template inference saw no slot twice, so it had no variation to reason about and kept every path literal. Collapsing the lowercase form is what lets the crawl rediscover `/{network_id}/autoblog/{section}` from live evidence. Add `--browser-proxy` so the audit can run against a production hostname served locally, which keeps the page's origin, cookie scope, and any origin checks in the ad stack matching production rather than `localhost`. `--danger-accept- invalid-certs` covers a MITM certificate whose CA the throwaway browser profile does not trust. Note that chromiumoxide builds each Chrome flag by prefixing `--` to the arg key, so a pre-formatted `--flag=value` string becomes `----flag=value` and is silently dropped. Both the new proxy flags and the existing mobile user-agent override were written that way; the user-agent override had therefore never taken effect. Both now pass `(key, value)` pairs. --- .../audit/generate/browser_collector.rs | 65 +++++++++++++++++-- .../src/commands/audit/generate/gpt_slots.rs | 52 ++++++++++++++- .../src/commands/audit/mod.rs | 19 ++++++ 3 files changed, 127 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs index beb5da93d..148bbf2aa 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs @@ -111,7 +111,7 @@ impl DeviceProfile { } /// Collects pages through a local Chrome, emulating one device profile. -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub(crate) struct BrowserAuditCollector { profile: Option, /// Pause between page loads during a crawl. @@ -120,6 +120,10 @@ pub(crate) struct BrowserAuditCollector { headful: bool, /// Answer the consent APIs as a consenting reader. assume_consent: bool, + /// Route the browser through this proxy, as `host:port`. + proxy: Option, + /// Accept TLS certificates that do not validate. + accept_invalid_certs: bool, } impl BrowserAuditCollector { @@ -163,24 +167,50 @@ impl BrowserAuditCollector { self.assume_consent = assume_consent; self } + + /// Routes the browser through `proxy` (`host:port`). + /// + /// Lets the audit run against a production hostname served by a local + /// MITM proxy, so the page's origin, cookie scope, and any origin checks in + /// the ad stack match production rather than `localhost`. + #[must_use] + pub(crate) fn with_proxy(mut self, proxy: Option) -> Self { + self.proxy = proxy; + self + } + + /// Accepts TLS certificates that do not validate. + /// + /// Needed when a MITM proxy presents a certificate from a CA the browser + /// profile does not trust. Dangerous against a real origin: see the flag + /// documentation. + #[must_use] + pub(crate) fn accept_invalid_certs(mut self, accept: bool) -> Self { + self.accept_invalid_certs = accept; + self + } } /// The browser-session knobs one crawl runs under. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct SessionSettings { profile: Option, page_delay: Duration, headful: bool, assume_consent: bool, + proxy: Option, + accept_invalid_certs: bool, } impl BrowserAuditCollector { - fn session(self) -> SessionSettings { + fn session(&self) -> SessionSettings { SessionSettings { profile: self.profile, page_delay: self.page_delay, headful: self.headful, assume_consent: self.assume_consent, + proxy: self.proxy.clone(), + accept_invalid_certs: self.accept_invalid_certs, } } } @@ -344,6 +374,8 @@ async fn with_browser( page_delay, headful, assume_consent, + proxy, + accept_invalid_certs, } = settings; let chrome_executable = find_browser_executable()?; let user_data_dir = TempDir::new().map_err(|error| { @@ -357,8 +389,26 @@ async fn with_browser( // the config with slots of its choosing. Validate certificates. let mut builder = BrowserConfig::builder() .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .respect_https_errors(); + .user_data_dir(user_data_dir.path()); + if !accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = &proxy { + // Chrome ignores a scheme-less `--proxy-server` value, silently sending + // traffic direct instead, so normalise it. `<-loopback>` keeps Chrome + // from bypassing the proxy for loopback hosts, which is exactly the case + // a local MITM proxy serves. + let endpoint = if proxy.contains("://") { + proxy.clone() + } else { + format!("http://{proxy}") + }; + // Keys carry no `--`: chromiumoxide adds it, so a pre-formatted + // `--flag=value` string becomes `----flag=value` and is ignored. + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } // `BrowserConfig` defaults to the *old* headless mode, which is both more // detectable and less faithful than either alternative — so both branches // must be explicit. Omitting the call is not the same as running headful. @@ -375,7 +425,10 @@ async fn with_browser( if let Some(user_agent) = profile.user_agent() { // Ad stacks branch on the user agent as well as the viewport, so // emulating size alone can still return desktop ad units. - builder = builder.arg(format!("--user-agent={user_agent}")); + // Key without the `--`: chromiumoxide prefixes it, so passing a + // pre-formatted `--flag=value` string yields `----flag=value`, + // which Chrome silently ignores. + builder = builder.arg(("user-agent", user_agent)); } } let config = builder.build().map_err(|error| { diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs index 365a5b696..d7b0b6bfe 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -33,6 +33,21 @@ use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedReq static HEX_HASH_SEGMENT: LazyLock = LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); +/// Matches a React `useId` token, which changes on every render. +/// +/// React emits these in both cases — `_R_3f_` from a server render and `_r_0_` +/// from a client one — so matching only the uppercase form leaves the lowercase +/// variant in the stem. That is not merely untidy: the suffix differs per +/// render, so one logical slot fragments into a new key on every page, which +/// both breaks runtime div matching and starves template inference of the +/// repeated observations it needs. +/// +/// The uppercase form is distinctive enough to match bare. The lowercase one is +/// anchored (`_r_`, a short alphanumeric run, `_`) so an ordinary id that merely +/// contains `_r_` keeps its full stem. +static REACT_USE_ID: LazyLock = + LazyLock::new(|| Regex::new(r"_R_|_r_[0-9a-z]{1,8}_").expect("should compile react id regex")); + /// Hosts that serve GPT `gampad/ads` requests. const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; @@ -185,12 +200,13 @@ fn is_usable_unit_path(path: &str) -> bool { /// a valid **prefix** of the live div id, which is how verify matches slots. /// /// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` -/// → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` → `ad-in_content`. +/// and `ad-header-0-_r_8_` → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` +/// → `ad-in_content`. fn normalize_div_stem(div_id: &str) -> String { let stem = div_id.strip_suffix("-container").unwrap_or(div_id); let mut cut = stem.len(); - if let Some(pos) = stem.find("_R_") { - cut = cut.min(pos); + if let Some(matched) = REACT_USE_ID.find(stem) { + cut = cut.min(matched.start()); } if let Some(matched) = HEX_HASH_SEGMENT.find(stem) { cut = cut.min(matched.start()); @@ -496,6 +512,36 @@ mod tests { } } + #[test] + fn lowercase_react_use_id_suffixes_collapse_to_one_slot() { + // React emits `_r_0_` client-side and `_R_3f_` server-side, and the + // token changes per render. Leaving it in the stem fragments one slot + // into a new key on every page, which starves template inference. + for volatile in [ + "ad-header-0-_r_0_", + "ad-header-0-_r_8_", + "ad-header-0-_r_a_", + "ad-header-0-_R_3f_", + ] { + let registry = vec![registry_slot("/123/site/news", volatile, &[(728, 90)])]; + let discovered = discover_gpt_slots(®istry, &[], false); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "`{volatile}` should normalize to a stable stem" + ); + } + } + + #[test] + fn an_ordinary_id_containing_r_is_left_alone() { + // The React shape is anchored, so a legitimate id keeps its full stem. + let registry = vec![registry_slot("/123/site/news", "ad_r_rail", &[(300, 250)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].div_id, "ad_r_rail"); + } + #[test] fn registry_slot_with_brace_in_unit_path_is_skipped() { // `gam_unit_path` is a template and there is no escape syntax, so a diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 777b09a2e..d5787665e 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -181,6 +181,23 @@ pub(crate) struct AuditAdTemplatesGenerateArgs { /// all. Pass this to observe the un-consented page instead. #[arg(long)] pub no_assume_consent: bool, + /// Route the audit browser through a proxy, as `host:port`. + /// + /// Pairs with `ts dev proxy`, which serves a production hostname from a + /// local Trusted Server. Auditing through it means the page's origin, + /// cookie scope, and any origin checks in the ad stack match production + /// rather than `localhost`. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Accept TLS certificates that do not validate. + /// + /// DANGEROUS against a real origin: the audit sends any `--cookie` session + /// upstream and treats the response as evidence, so an invalid certificate + /// could mean an impersonator is harvesting the session and fabricating the + /// result. Intended for a local MITM proxy whose CA the browser profile does + /// not trust; prefer installing that CA (`ts dev proxy ca`) over this flag. + #[arg(long)] + pub danger_accept_invalid_certs: bool, } impl AuditAdTemplatesGenerateArgs { @@ -268,6 +285,8 @@ pub(crate) fn run_audit(args: &AuditArgs) -> Result<(), String> { .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) .headful(gen_args.headful) .assume_consent(!gen_args.no_assume_consent) + .with_proxy(gen_args.browser_proxy.clone()) + .accept_invalid_certs(gen_args.danger_accept_invalid_certs) }) .collect(); let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles From 230958b6aaeb51880cb7da4b53a4d3db0e2706e2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 16:09:59 +0530 Subject: [PATCH 194/195] Refuse ad-template slots that are one placement under per-render div ids A live crawl produced fourteen slots where four were real. Ten were two placements repeated: an ad stack built its div ids from a per-render token, so the same placement arrived under a new key on every page. Written verbatim those ids match nothing at runtime, and the fragmentation also starves template inference, which needs to observe a slot more than once. Detect it from evidence rather than by pattern-matching token shapes, since each stack invents its own and the previous two forms already needed separate handling. Candidates share an identical ad-unit path and identical formats; what separates a fragmented placement from two legitimate siblings on one unit is co-occurrence. Real siblings appear together on a page, while fragments never do, because each page yields exactly one of them. Fragments are reported and skipped rather than written. The report names the observed ids and the stable prefix they share, so the operator can add the placement once with a prefix they know survives a render. That prefix is deliberately not written as a `div_id`: it reaches only as far as the observed tokens happen to agree, so it would match this crawl's ids and miss the next render's. Verified live: the run that previously wrote fourteen slots now writes the four real ones and explains the two it declined. --- .../src/commands/audit/generate/evidence.rs | 200 ++++++++++++++++++ .../src/commands/audit/generate/mod.rs | 35 ++- 2 files changed, 234 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs index bfa004b94..b2404da54 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -61,6 +61,62 @@ impl SlotEvidence { } } +/// Slots grouped by the shape that would make them one placement: an identical +/// ad-unit path and an identical format set. +type SlotsByShape<'a> = BTreeMap<(String, Vec<(u32, u32)>), Vec<&'a SlotEvidence>>; + +/// Several observed slots that are really one placement under volatile div ids. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FragmentGroup { + /// The volatile div ids observed, in evidence order. + pub(super) div_ids: Vec, + /// The ad-unit path every fragment shared. + pub(super) unit_path: String, + /// The stable prefix the ids share, when they share a useful one. + /// + /// Offered to the operator as a starting point only. It is deliberately not + /// written as a `div_id`: the shared prefix reaches only as far as the + /// *observed* tokens happen to agree, so it would keep matching this crawl's + /// ids and stop matching the next render's. + pub(super) suggested_prefix: Option, +} + +/// Whether no two slots were ever seen on the same page. +fn pages_are_disjoint(slots: &[&SlotEvidence]) -> bool { + for (index, slot) in slots.iter().enumerate() { + let pages = slot.paths(); + if slots[index + 1..] + .iter() + .any(|other| other.paths().intersection(&pages).next().is_some()) + { + return false; + } + } + true +} + +/// The longest prefix the div ids share, trimmed back to a separator. +/// +/// Trimming matters: the raw common prefix usually ends mid-token (the leading +/// digits of a timestamp two fragments happen to share), which is worse than +/// useless as a suggestion. Cutting at the last `-` or `_` yields the part a +/// human would recognise as the placement's name. +fn shared_div_prefix(slots: &[&SlotEvidence]) -> Option { + let mut prefix: &str = slots.first()?.div_id.as_str(); + for slot in &slots[1..] { + let shared = slot + .div_id + .char_indices() + .zip(prefix.chars()) + .take_while(|((_, left), right)| left == right) + .count(); + prefix = &prefix[..shared]; + } + let trimmed = prefix.trim_end_matches(|ch: char| ch != '-' && ch != '_'); + let candidate = trimmed.trim_end_matches(['-', '_']); + (!candidate.is_empty()).then(|| candidate.to_string()) +} + /// Slot evidence accumulated across every collected page. #[derive(Debug, Clone, Default)] pub(super) struct EvidenceTable { @@ -144,6 +200,46 @@ impl EvidenceTable { self.slots.is_empty() } + /// Groups of slots that are one slot wearing a different div id per page. + /// + /// Some ad stacks build div ids from a per-render token — a timestamp, a + /// framework id — so the same placement arrives under a new key on every + /// page. Written verbatim those ids never match at runtime, and the + /// fragmentation also starves template inference, which needs to see one + /// slot more than once. + /// + /// Detection is by evidence rather than by guessing at token shapes, because + /// each stack invents its own. Candidates share an identical ad-unit path and + /// identical formats; what separates a fragmented slot from two legitimate + /// siblings on the same unit is **co-occurrence**. Real siblings appear + /// together on a page; fragments of one slot never do, because each page + /// produces exactly one of them. + pub(super) fn fragmented_slots(&self) -> Vec { + let mut by_shape: SlotsByShape<'_> = BTreeMap::new(); + for slot in self.slots() { + // Only slots pinned to exactly one unit path can be compared this + // way; a slot whose unit varies is inference's problem, not this one. + let units = slot.unit_paths(); + if units.len() != 1 { + continue; + } + let unit = (*units.iter().next().expect("one unit path")).to_string(); + let formats: Vec<(u32, u32)> = slot.formats.iter().copied().collect(); + by_shape.entry((unit, formats)).or_default().push(slot); + } + + by_shape + .into_iter() + .filter(|(_, slots)| slots.len() > 1) + .filter(|(_, slots)| pages_are_disjoint(slots)) + .map(|((unit_path, _), slots)| FragmentGroup { + div_ids: slots.iter().map(|slot| slot.div_id.clone()).collect(), + unit_path, + suggested_prefix: shared_div_prefix(&slots), + }) + .collect() + } + /// The single GAM network id observed across the crawl. /// /// # Errors @@ -291,6 +387,110 @@ mod tests { ); } + #[test] + fn one_placement_under_per_render_div_ids_is_detected() { + // The live shape: a timestamped token means each page yields a new key + // for the same placement. Same unit, same formats, never co-occurring. + let mut table = EvidenceTable::default(); + for (path, div) in [ + ( + "/features/a", + "rh-gam-kso_26329268ce6Bj0uc8sL0_ei_overlay_1", + ), + ("/news/b", "rh-gam-kso_26329269aoYmv4RQyN3n_ei_overlay_1"), + ("/deals/c", "rh-gam-kso_26329270mYPDB3tz8cpB_ei_overlay_1"), + ] { + table.fold_page( + path, + &page(&[("/99/site_Overlay", div, &[(300, 250)])], false), + ); + } + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1, "the three fragments should form one group"); + assert_eq!(groups[0].div_ids.len(), 3); + assert_eq!(groups[0].unit_path, "/99/site_Overlay"); + assert_eq!( + groups[0].suggested_prefix.as_deref(), + Some("rh-gam-kso"), + "the suggestion should be trimmed back off the volatile token" + ); + } + + #[test] + fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { + // Two real in-content positions can share a unit path and formats. What + // distinguishes them from fragments is that they appear *together* on a + // page, so refusing to write them would lose real inventory. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ("/99/site/news", "ad-in_content-1", &[(300, 250)]), + ("/99/site/news", "ad-in_content-2", &[(300, 250)]), + ], + false, + ), + ); + + assert!( + table.fragmented_slots().is_empty(), + "co-occurring slots are siblings, not fragments" + ); + } + + #[test] + fn slots_differing_in_formats_are_not_fragments() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "slot-aaaa", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "slot-bbbb", &[(728, 90)])], false), + ); + + assert!( + table.fragmented_slots().is_empty(), + "a differing format set means these are different placements" + ); + } + + #[test] + fn a_slot_seen_alone_is_never_a_fragment() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "only-slot", &[(300, 250)])], false), + ); + + assert!(table.fragmented_slots().is_empty()); + } + + #[test] + fn fragments_with_no_shared_prefix_report_none() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "alpha-1111", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "beta-2222", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1); + assert_eq!( + groups[0].suggested_prefix, None, + "unrelated ids should not produce a misleading suggestion" + ); + } + #[test] fn conflicting_network_ids_are_a_hard_error() { let mut table = EvidenceTable::default(); diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index d74a22e71..2a2d4b11b 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -614,7 +614,32 @@ pub(crate) fn run_update_slots( .as_ref() .and_then(|outcome| outcome.policy.clone()); - let slots = build_render_slots(&table, inference.as_ref(), policy.as_ref(), request)?; + // Slots that are one placement wearing a per-render div id cannot be + // written: the ids never match at runtime. Report them so the operator can + // add the placement once with a prefix they know is stable. + let fragmented = table.fragmented_slots(); + for group in &fragmented { + let suggestion = group.suggested_prefix.as_deref().map_or_else( + || "no stable prefix was shared".to_string(), + |prefix| format!("they share the prefix `{prefix}`"), + ); + notes.push(format!( + "skipped {} slot(s) that look like one placement under a per-render div id on \ + `{}` ({}); {suggestion}. Add it once by hand with a div_id prefix that is \ + stable across renders", + group.div_ids.len(), + group.unit_path, + group.div_ids.join(", "), + )); + } + + let slots = build_render_slots( + &table, + inference.as_ref(), + policy.as_ref(), + request, + &fragmented, + )?; let merged = slot_toml::merge_render_slots(request.existing_creative, slots, request.replace); let rendered_slots = render_slots(&merged); let updated = splice_creative_slots( @@ -804,7 +829,12 @@ fn build_render_slots( inference: Option<&unit_template::InferenceOutcome>, policy: Option<&unit_template::SectionPolicy>, request: &UpdateSlotsRequest<'_>, + fragmented: &[evidence::FragmentGroup], ) -> CliResult> { + let skip: std::collections::BTreeSet<&str> = fragmented + .iter() + .flat_map(|group| group.div_ids.iter().map(String::as_str)) + .collect(); // Explicit `--page-pattern` values are an operator override: they apply to // every slot and disable inference from observed paths entirely. let explicit = !request.page_patterns.is_empty(); @@ -815,6 +845,9 @@ fn build_render_slots( let mut slots = Vec::with_capacity(table.slot_count()); for slot in table.slots() { + if skip.contains(slot.div_id.as_str()) { + continue; + } let patterns = if explicit { request.page_patterns.to_vec() } else { From 64b941478197fddd15d3e8eee636c24394931cf8 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 17 Aug 2026 16:39:06 +0530 Subject: [PATCH 195/195] Document the ad-template crawl options added after the first draft MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four options landed after the command was first documented and were never written up: request pacing, a headful browser, the consent answer, and auditing through a local proxy. Each exists because a live audit of a protected publisher failed without it, so the reason belongs alongside the flag. Consent gets its own section because the failure is silent. A publisher gates slot definition behind its consent platform, the audit runs in a throwaway profile with no consent cookie, and the result is a page that appears to have no ad stack at all — indistinguishable from one that genuinely has none. Also record that an empty slot registry now reports GPT's observable state, which is what separates "the library never loaded" from "this page has no ads". Proxy auditing gets a section because `ts dev proxy` is how a production hostname is served locally, and matching the production origin matters for cookie scope and for origin checks inside the ad stack. Note the caveat that a local Trusted Server injects its own configured slots, so a run through the proxy can rediscover config it already has. Finally, describe how per-render div ids are detected and reported, including why the suggested prefix is offered but never written. --- docs/guide/cli.md | 76 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/guide/cli.md b/docs/guide/cli.md index b0aeb612b..9c31fb56d 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -229,12 +229,84 @@ hand-tuned fields and gains this run's patterns, and a hand-written instead, which also discards any template you wrote by hand. Behind bot protection, pass a valid clearance cookie. The crawl reuses one -browser session, so clearance earned on the first page carries to the rest: +browser session, so clearance earned on the first page carries to the rest, and +`--page-delay-ms` spaces the requests — an unpaced crawl is both discourteous to +the origin and likelier to be challenged partway through: ```bash -ts audit ad-templates generate https://publisher.example/ --cookie 'datadome=' +ts audit ad-templates generate https://publisher.example/ \ + --cookie '=' --page-delay-ms 1500 +``` + +Some origins refuse a headless browser outright regardless of the cookie. +`--headful` runs a visible one, which is also the quickest way to _see_ whether +a challenge is being shown: + +```bash +ts audit ad-templates generate https://publisher.example/ --headful +``` + +### Sites behind a consent platform + +Publishers gate slot definition behind their consent platform, and the audit +runs in a throwaway browser profile with no consent cookie. Left alone, such a +site defines no slots at all and looks identical to a site with no ad stack. + +The crawl therefore answers the two IAB interfaces every compliant platform +exposes — TCF v2 and US Privacy — as a consenting, out-of-scope reader, before +any page script runs. This changes only what the audit browser sees; it does not +affect the publisher's own readers. Pass `--no-assume-consent` to observe the +un-consented page instead. + +When a page still yields no slots, the run reports GPT's observable state — +whether the library reached `apiReady`, how many queued commands never drained, +how many scripts ran. An empty slot registry has several very different causes, +and that line distinguishes them. + +### Auditing a production hostname served locally + +`ts dev proxy` serves a production hostname from a local Trusted Server. +Auditing through it keeps the page's origin, cookie scope, and any origin checks +in the ad stack matching production rather than `localhost`: + +```bash +ts dev proxy --map www.publisher.example=127.0.0.1:7676 --upstream-plaintext --rewrite-host + +ts audit ad-templates generate https://www.publisher.example/ \ + --browser-proxy 127.0.0.1:18080 --danger-accept-invalid-certs ``` +`--danger-accept-invalid-certs` covers the proxy's MITM certificate when the +throwaway browser profile does not trust its CA; installing that CA +(`ts dev proxy ca`) is preferable. Against a real origin the flag is dangerous — +the audit sends any `--cookie` session upstream and treats the response as +evidence, so an invalid certificate could mean an impersonator is both +harvesting the session and fabricating the result. + +Note that a local Trusted Server injects its own configured slots into the page, +so a run through the proxy can rediscover config it already has. Slot ids that +are absent from the current config are the publisher's own. + +### Slots that change div id on every render + +Some ad stacks build div ids from a per-render token, so one placement arrives +under a new id on every page. Those ids match nothing at runtime, so the run +declines to write them and reports the group instead: + +```text +note: skipped 3 slot(s) that look like one placement under a per-render div id + on `/12345678/example.com_Overlay` (kso_2632930aBc_overlay_1, …); + they share the prefix `kso`. Add it once by hand with a div_id prefix + that is stable across renders +``` + +The detection is by evidence, not by recognising token shapes: candidates share +an ad-unit path and formats, and what separates a fragmented placement from two +legitimate siblings on one unit is co-occurrence — real siblings appear together +on a page, fragments never do. The suggested prefix is a starting point only, not +written as a `div_id`, because it reaches only as far as the observed tokens +happen to agree. + ### Checking for a device split Publishers often serve a different ad unit per device