diff --git a/changelog.d/pr4-stage8-dns-cutover-canonical-baseline.added.md b/changelog.d/pr4-stage8-dns-cutover-canonical-baseline.added.md new file mode 100644 index 000000000..f6d66b7e2 --- /dev/null +++ b/changelog.d/pr4-stage8-dns-cutover-canonical-baseline.added.md @@ -0,0 +1,4 @@ +- Canonical ramp baseline (GAE-via-LB at 100/0, captured at the Stage 8 DNS cutover) + committed to `docs/migration/baselines/`; ramp runbook holds cut to ~4h with a + peak-window carve-out; rollback runbook records the at-cutover GAE certificate + expiry and the verified 5-minute DNS TTL. diff --git a/docs/migration/baselines/canonical-gae-via-lb.json b/docs/migration/baselines/canonical-gae-via-lb.json new file mode 100644 index 000000000..8b511d7f1 --- /dev/null +++ b/docs/migration/baselines/canonical-gae-via-lb.json @@ -0,0 +1,346 @@ +{ + "completion_count": 0, + "completion_failure_count": 0, + "error_rate": 0.0, + "failure_count": 0, + "p50_completion_ms": null, + "p50_latency_ms": 322.55, + "p95_completion_ms": null, + "p95_latency_ms": 4096.66, + "probes": [ + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 510.59, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 304.92, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3973.77, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 329.59, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 363.52, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3853.12, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 306.72, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 287.0, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3990.98, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 305.86, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 292.69, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3894.08, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 304.44, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 306.8, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3743.18, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 346.08, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 279.73, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3990.85, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 291.54, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 313.12, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3409.96, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 293.86, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 315.52, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 4096.66, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 253.89, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 314.87, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 8645.23, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 350.95, + "method": "GET", + "name": "liveness", + "ok": true, + "path": "/liveness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 274.4, + "method": "GET", + "name": "readiness", + "ok": true, + "path": "/readiness-check", + "status_code": 200 + }, + { + "completed": null, + "completion_ms": null, + "error": null, + "latency_ms": 3929.02, + "method": "GET", + "name": "us_metadata", + "ok": true, + "path": "/us/metadata", + "status_code": 200 + } + ], + "request_count": 30, + "status_counts": { + "200": 30 + } +} diff --git a/docs/migration/pr4-app-engine-rollback-runbook.md b/docs/migration/pr4-app-engine-rollback-runbook.md index a30a3c67c..a14ea3bb8 100644 --- a/docs/migration/pr4-app-engine-rollback-runbook.md +++ b/docs/migration/pr4-app-engine-rollback-runbook.md @@ -77,12 +77,14 @@ are deliberately NOT embedded here — get them from, in order: Two clocks limit this option: -1. **TTL**: propagation is bounded by the TTL recorded at Stage 7 (Squarespace floor; - expect ≥1h). Clients resolve back gradually, not at once. +1. **TTL**: propagation is bounded by the record TTL — set to **300s (5 min)** at + Stage 7 and verified on public resolvers. Clients resolve back gradually, not at + once. 2. **Certificate decay**: once DNS moves to the LB at Stage 8, App Engine's managed - certificate for `api.policyengine.org` stops renewing. Record its expiry at cutover - (typically ≤90 days out). **After that expiry, class (c) serves invalid TLS and is - dead** — the only path is fixing the LB forward and using class (a). + certificate for `api.policyengine.org` stops renewing. **Recorded at cutover + (2026-07-10): cert `38894749` expires 2026-08-22T19:09:24Z.** After that expiry, + class (c) serves invalid TLS and is dead — the only path is fixing the LB forward + and using class (a). After any (c) execution: the LB keeps running (no teardown under pressure); re-cutover follows the Stage 8 procedure once the failure is understood. diff --git a/docs/migration/pr4-ramp-runbook.md b/docs/migration/pr4-ramp-runbook.md index 35e55edeb..71f7f5bd8 100644 --- a/docs/migration/pr4-ramp-runbook.md +++ b/docs/migration/pr4-ramp-runbook.md @@ -6,12 +6,17 @@ > rollback in [`pr4-app-engine-rollback-runbook.md`](pr4-app-engine-rollback-runbook.md). > Gates are **logs-first**: both backend services log at `sampleRate: 1.0` (verified > 2026-07-09), so `gcloud logging read` sweeps are exact counts, not samples. Cloud -> Monitoring dashboards/alerts are optional (deferred per plan doc); **there is no -> automatic failover** — serverless NEGs have no LB health checks, so a failed gate is -> acted on by a human running the rollback runbook. - -Ramp ladder: `bs-cloud-run` weight **1 → 5 → 25 → 50 → 100**, holding **~24h** per step. -Advance only when every gate below is green for the full hold. +> Monitoring dashboards/alerts were dropped (user decision 2026-07-09; re-open +> condition below); **there is no automatic failover** — serverless NEGs have no LB +> health checks, so a failed gate is acted on by a human running the rollback runbook. + +Ramp ladder: `bs-cloud-run` weight **1 → 5 → 25 → 50 → 100**, holding **~4h** per step +(cut from 24h, user decision 2026-07-09: at this service's volume 4h is statistically +sufficient for the gates, rollback is minutes, and short attended holds beat long +unattended ones given the no-alerts decision). Advance only when every gate below is +green for the full hold. **Carve-out: at least one of the 25% / 50% holds must span the +daily traffic peak window** (mined from 7 days of hourly request counts before the 1% +step) — schedule those two steps so one covers it. ## 0. Shell variables @@ -128,19 +133,21 @@ E2E suite against production per its own repo instructions. ## 6. Advance, hold, or roll back -- All gates green for the full ~24h hold → next step on the ladder. +- All gates green for the full ~4h hold → next step on the ladder (respecting the + peak-window carve-out for the 25%/50% steps). - Gate failure → [`pr4-app-engine-rollback-runbook.md`](pr4-app-engine-rollback-runbook.md) class (a) (weights back to `app_engine=100`), then diagnose. - Record every step (weights, `$STEP_START`, gate outputs, scale-out count, anomalies) in `docs/migration/history/` when the ramp campaign completes. -## Accepted risk while alerts are deferred +## Accepted risk with alerts dropped -With Stage 6 item 4 deferred there is no unattended detection: a regression that starts +With Stage 6 item 4 dropped there is no unattended detection: a regression that starts mid-hold goes unnoticed until the next manual sweep. During holds, run the step-3 5xx -sweep at least every few waking hours. If this cadence is not sustainable, implement -the deferred alerts (three policies, ~$1.50/mo once Google's alerting billing starts, -$0 before then) rather than lengthening the sweep interval. +sweep at least every few waking hours; the 4h holds are sized to be attended +end-to-end. **Re-open condition:** if this cadence proves unsustainable, implement the +dropped alerts (three policies, ~$1.50/mo once Google's alerting billing starts, $0 +before then) rather than lengthening the sweep interval. ## Log-volume caveat (post-100%) diff --git a/docs/migration/urlmap/20260710T185147Z-lb-api-99-1-stage9.yaml b/docs/migration/urlmap/20260710T185147Z-lb-api-99-1-stage9.yaml new file mode 100644 index 000000000..938ebfa67 --- /dev/null +++ b/docs/migration/urlmap/20260710T185147Z-lb-api-99-1-stage9.yaml @@ -0,0 +1,12 @@ +creationTimestamp: '2026-07-08T11:10:19.533-07:00' +defaultRouteAction: + weightedBackendServices: + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-app-engine + weight: 99 + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-cloud-run + weight: 1 +fingerprint: O2KiL1CKhu0= +id: 1171555047161670757 +kind: compute#urlMap +name: lb-api +selfLink: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/urlMaps/lb-api diff --git a/docs/migration/urlmap/20260710T203710Z-lb-api-rehearsal-100-0-stage9.yaml b/docs/migration/urlmap/20260710T203710Z-lb-api-rehearsal-100-0-stage9.yaml new file mode 100644 index 000000000..481c6b787 --- /dev/null +++ b/docs/migration/urlmap/20260710T203710Z-lb-api-rehearsal-100-0-stage9.yaml @@ -0,0 +1,12 @@ +creationTimestamp: '2026-07-08T11:10:19.533-07:00' +defaultRouteAction: + weightedBackendServices: + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-app-engine + weight: 100 + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-cloud-run + weight: 0 +fingerprint: QsBhxBnDkxI= +id: 1171555047161670757 +kind: compute#urlMap +name: lb-api +selfLink: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/urlMaps/lb-api diff --git a/docs/migration/urlmap/20260710T204214Z-lb-api-restored-99-1-stage9.yaml b/docs/migration/urlmap/20260710T204214Z-lb-api-restored-99-1-stage9.yaml new file mode 100644 index 000000000..938ebfa67 --- /dev/null +++ b/docs/migration/urlmap/20260710T204214Z-lb-api-restored-99-1-stage9.yaml @@ -0,0 +1,12 @@ +creationTimestamp: '2026-07-08T11:10:19.533-07:00' +defaultRouteAction: + weightedBackendServices: + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-app-engine + weight: 99 + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-cloud-run + weight: 1 +fingerprint: O2KiL1CKhu0= +id: 1171555047161670757 +kind: compute#urlMap +name: lb-api +selfLink: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/urlMaps/lb-api diff --git a/docs/migration/urlmap/20260711T123622Z-lb-api-95-5-stage10-step1.yaml b/docs/migration/urlmap/20260711T123622Z-lb-api-95-5-stage10-step1.yaml new file mode 100644 index 000000000..40d4800b9 --- /dev/null +++ b/docs/migration/urlmap/20260711T123622Z-lb-api-95-5-stage10-step1.yaml @@ -0,0 +1,12 @@ +creationTimestamp: '2026-07-08T11:10:19.533-07:00' +defaultRouteAction: + weightedBackendServices: + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-app-engine + weight: 95 + - backendService: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/backendServices/bs-cloud-run + weight: 5 +fingerprint: mywYQi8PhPA= +id: 1171555047161670757 +kind: compute#urlMap +name: lb-api +selfLink: https://www.googleapis.com/compute/v1/projects/policyengine-api/global/urlMaps/lb-api