From fc23880fc89c4927bd6f3504c53e88664d017805 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 17:22:01 -0400 Subject: [PATCH 01/10] docs: reconcile desktop and remote substrate evidence --- README.md | 29 ++++++++++++++++++++--------- claims.yaml | 31 +++++++++++++++++++++++-------- docs/LIMITS.md | 10 ++++++---- docs/PRODUCT_STATUS.md | 12 ++++++------ docs/VERIFICATION.md | 14 ++++++++------ docs/backends/RDP.md | 5 +++-- docs/desktop/CITRIX_PIXEL.md | 2 +- docs/verification.json | 32 +++++++++++++++++++++++++++----- public-artifacts.json | 4 ++-- scripts/validate_claims.py | 6 +++--- 10 files changed, 99 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index c2a196ca..1ac70e59 100644 --- a/README.md +++ b/README.md @@ -106,15 +106,23 @@ openadapt-flow record --backend macos --macos-app TextEdit --out rec openadapt-flow record --backend linux --linux-app gedit \ --linux-window-title "Untitled Document 1" --out rec -# Remote display / Citrix / VDI (pixel-only vision ladder over RDP). +# RDP remote display (pixel-only vision ladder over the network session). openadapt-flow record --backend rdp --rdp-host 10.0.0.5 --out rec + +# Citrix / VDI (one exact local Citrix Workspace window). +openadapt-flow record --backend citrix \ + --window "Citrix Viewer" \ + --rdp-window "Citrix Viewer" \ + --rdp-window-title "Ward A" \ + --rdp-readiness-text "Appointments" \ + --out rec ``` `--backend web` is browser-first (the app is a `--url`). For -`windows`, `macos`, `linux`, and `rdp` the capture has no field identity, so the +`windows`, `macos`, `linux`, `rdp`, and `citrix` the capture has no field identity, so the target is the app window or host: `--agent-url` for Windows, `--macos-app`, `--linux-app` plus `--linux-window-title`, and `--rdp-host` (or a configured -`rdp_window` for a local Citrix / Parallels window). Pass the same `--backend` +exact `rdp_window` / `rdp_window_title` for Citrix Workspace). Pass the same `--backend` plus target flags to `replay`, or drive a real deployment with `openadapt-flow run bundle --config deploy.yaml`, which reads the backend, effects, actuation, durable, and policy sections from one config. Recorded @@ -208,10 +216,10 @@ runtime; none is a second-class add-on. Maturity is reported honestly per the |---|---|---|---| | Web / browser | `--backend web` | Validated | Full lifecycle on every CI build, plus third-party OpenEMR evidence | | Native macOS (AX) | `--backend macos` | Validated | 3/3 fixed TextEdit trials with exact file-byte effects; refused a two-window ambiguity without changing either file | -| Native Windows (UIA) | `--backend windows` | Early | 3/3 fixed WinForms trials with independently confirmed SQLite effects; 3/3 refusal for both stale and ambiguous targets | -| Native Linux (AT-SPI) | `--backend linux` | Early | Required CI drives a real GTK3 workflow through an isolated X11 / session-D-Bus environment: three verified effects, plus three ambiguous-target and three stale-target refusals | -| RDP (remote display) | `--backend rdp` | Early | Real-network Aardwolf RDP into Windows 11 passed 3/3 fixed remote-input trials with independent guest-tools file verification | -| Citrix / VDI (pixel ladder) | `--backend rdp` + `rdp_window` | Exploratory | Pixel-only vision ladder; no validated ICA / HDX environment qualified yet. Enters design-partner qualification in the customer's exact published application | +| Native Windows (UIA) | `--backend windows` | Available | 3/3 fixed WinForms trials with independently confirmed SQLite effects; 3/3 refusal for both stale and ambiguous targets | +| Native Linux (AT-SPI) | `--backend linux` | Available | Required CI drives a real GTK3 workflow through an isolated X11 / session-D-Bus environment: three verified effects, plus three ambiguous-target and three stale-target refusals | +| RDP (remote display) | `--backend rdp` | Available | Real-network Aardwolf RDP into Windows 11 passed 3/3 fixed remote-input trials with independent guest-tools file verification; a separate real-FreeRDP batch covers record → compile → governed replay and refusal | +| Citrix / VDI (pixel ladder) | `--backend citrix` | Code-qualified | Dedicated exact-Workspace-window driver, readiness gate, durable resume, and 3 healthy + 3 drift-halt no-DOM trials; the retained artifact records `ica_hdx_accepted=false` until a counted ICA/HDX run is attached | These are accepted scoped qualifications. A customer application is qualified against its own controls, session/display policy, identity evidence, and effect @@ -422,8 +430,11 @@ submission format: [`benchmark/effectbench/LEADERBOARD.md`](benchmark/effectbenc The reference browser path runs record, compile, policy-check, deterministic replay, refusal, and report generation in CI. Windows UIA, native macOS, native Linux, and RDP each have retained 3/3 accepted task evidence with independent -effects or oracles. Citrix and each new third-party application are qualified -with a design partner in the exact deployment environment. The workflow-program +effects or oracles. Citrix has a dedicated exact-window backend and a retained +3+3 code-readiness record; an exact ICA/HDX deployment receives its own counted +qualification record rather than inheriting RDP or stand-in evidence. Each new +third-party application is similarly qualified against its controls and effect +oracle. The workflow-program IR adds parameters, branches, loops, effect verification, and governed recovery on the same runtime. `DESIGN.md` has the module contracts; [`docs/design/WORKFLOW_PROGRAM_IR.md`](docs/design/WORKFLOW_PROGRAM_IR.md) diff --git a/claims.yaml b/claims.yaml index fa8478b9..ce0e3e27 100644 --- a/claims.yaml +++ b/claims.yaml @@ -68,7 +68,7 @@ claims: - >- "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped - acceptance and design-partner claims below. + acceptance and code-qualified claims below. - >- The full record->compile->replay e2e runs in the post-merge/nightly full suite; the fast required PR gate covers the replayer + demo units. @@ -488,9 +488,10 @@ claims: # ------------------------------- Citrix pixel-only (validating, opt-in gated) - id: citrix-pixel-validating claim: >- - Citrix uses the governed pixel-only remote-display contract and enters - design-partner qualification in the customer's exact ICA/HDX application, - with identity, ambiguity refusal, and independent effects above the driver. + Citrix uses a dedicated exact-Workspace-window backend beneath the governed + pixel-only remote-display contract. The backend, readiness gate, durable + resume, identity/refusal ladder, and independent effects are code-qualified; + live ICA/HDX acceptance remains separately bound to the exact deployment. surfaces: [README.md, docs, website] tier: validating evidence: @@ -503,11 +504,24 @@ claims: - path: tests/test_pixel_identity_probe.py proves: >- The pixel-substrate identity probe behind the on-pixels identity gate. + - path: tests/e2e/test_citrix_workspace_standin_e2e.py + proves: >- + The dedicated Citrix backend passes three healthy effect-confirmed + record->compile->replay trials and three severe-drift safe-halts over + a no-DOM surface, with zero model calls, false completion, or silent + incorrect success. + - path: benchmark/citrix_workspace/results.json + kind: artifact + proves: >- + The retained six-trial record reports code_readiness_accepted=true + and ica_hdx_accepted=false, keeping driver readiness distinct from a + counted live ICA/HDX qualification. - path: docs/desktop/CITRIX_PIXEL.md kind: doc proves: >- - The Citrix driver model, reusable evidence, design-partner acceptance - contract, independent-effect boundary, and customer-controlled posture. + The Citrix driver model, reusable evidence, exact-deployment + acceptance contract, independent-effect boundary, and + customer-controlled posture. caveats: - >- PIXEL-ONLY: no UIA/DOM crosses to the driver. On-screen OCR read-back is @@ -516,7 +530,8 @@ claims: A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); - never runs on default CI, and does not replace design-partner validation. + never runs on default CI, and does not replace a counted live ICA/HDX + acceptance record. # ------------------------ Windows-host window replay (mock-tested, roadmap) - id: win32-window-replay-roadmap @@ -540,7 +555,7 @@ claims: proves: >- The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, - then design-partner Citrix). + then a counted live Citrix host run). caveats: - >- NOT qualified: no counted end-to-end batch on a real Windows host or diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 2095158f..78cd50b8 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -23,9 +23,11 @@ For the current evidence behind each maturity claim, see | Deterministic re-resolution | **Beta** | Bounded visual or structural drift can be resolved through non-model evidence and recorded as a reviewable change. | It is not general adaptation to a redesigned workflow, changed business rules, or missing evidence. | | AI-assisted repair | **Experimental** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. | | Human teaching and resume | **Experimental** | A halt can produce evidence for an operator correction, guarded promotion, and durable resume. | Field recovery time, broad authoring UX, enterprise identity integration, and non-repudiation are not established. | -| Windows UI Automation | **Experimental** | The backend contract is tested and a gated Windows environment has exercised structural resolution. | Windows is not the default supported substrate and is not broadly validated across third-party desktop applications. | -| Native macOS automation | **Research** | The browser path runs on macOS. | There is no production-candidate native macOS accessibility backend in this repository. | -| RDP and Citrix-style pixel-only automation | **Research** | Adapter and pixel-only analog tests exercise the backend seam and refusal behavior. | Real RDP session diversity, ICA/HDX, DPI changes, lock screens, latency, credentials, and clinical applications are not validated. This is not a validated Citrix integration. | +| Windows UI Automation | **Scoped acceptance** | The native UIA backend completed 3/3 exact WinForms effects and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. | +| Native macOS automation | **Scoped acceptance** | The AX/AppKit backend completed 3/3 exact TextEdit file effects and refused an ambiguous two-window target; structural record/locate/refuse behavior is covered separately. | Acceptance is bound to the tested host and application. Accessibility exposure and supported native actions vary by application. | +| Native Linux automation | **Scoped acceptance** | Required current-main CI exercises AT-SPI against GTK3 with 3 exact effects, 3 ambiguity refusals, and 3 stale-target refusals. | The built-in path is X11. Wayland needs an operator-approved portal session, and each application retains its own qualification. | +| RDP automation | **Scoped acceptance** | A real-network Aardwolf/Windows batch passed 3/3 with an independent guest-tools oracle; a separate real-FreeRDP batch covers record, compile, governed replay, and drift refusal. | The two records do not by themselves cover arbitrary applications, WAN conditions, display policy, or Citrix ICA/HDX. | +| Citrix / VDI pixel-window automation | **Code-qualified** | The dedicated exact-Workspace-window backend, readiness gate, governed run, and durable resume passed 3 healthy effect-confirmed no-DOM trials plus 3 drift safe-halts. | The retained record explicitly has `ica_hdx_accepted=false`; live ICA/HDX acceptance stays bound to the exact deployment instead of being inferred from a stand-in or RDP. | | Managed browser execution | **Beta** | The hosted lane admits attested browser bundles; production mode requires a configured real runner and refuses silent mock fallback. | It does not extend the supported claim to Windows, RDP, Citrix, PHI-bearing shared-cloud execution, an SLA, or a regulated certification. | | On-premises / customer-managed deployment | **Experimental** | Deployment configuration, local run gates, egress checks, and audit primitives are supplied. | OpenAdapt does not configure the customer's firewall, KMS, storage, identity provider, backups, retention, incident response, or legal compliance program. | @@ -145,7 +147,7 @@ from every different record — glyph-collapse siblings and wrong MRNs alike false-accept across the different-record trials with margin**. That evidence is **synthetic**; a pixel false-accept is a silent wrong record, the worst possible outcome, and no real RDP/Citrix/HDX identifier corpus has been captured yet -(that substrate is Research). The exact bar to flip the default on: reproduce +(so the shipped pixel-verify default remains off). The exact bar to flip the default on: reproduce `false_accept == 0` with a comparable gap on a **real captured remote-display identifier corpus**. Until then, verify is reachable only when a caller opts in per risk class (`verify_pixel_identity(..., enable_verify=True)`); mismatch and diff --git a/docs/PRODUCT_STATUS.md b/docs/PRODUCT_STATUS.md index 70730c56..38fb1ba9 100644 --- a/docs/PRODUCT_STATUS.md +++ b/docs/PRODUCT_STATUS.md @@ -11,8 +11,8 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). | --- | --- | | **Beta** | Runs end to end on the default CI or clean-machine lifecycle and is available for evaluation. No production SLA or general enterprise certification is implied. | | **Scoped acceptance** | A fixed task, environment, run count, oracle, and failure taxonomy have passed an accepted qualification. This is stronger than a protocol spike and narrower than arbitrary-application support. | -| **Design-partner qualification** | The product surface is available for qualification in a customer's exact application and environment; acceptance is bound to that deployment's evidence. | -| **Experimental** | Real implementation exists, but evidence is opt-in, infrastructure-gated, mocked at an external boundary, or limited to design-partner validation. | +| **Code-qualified** | The integrated product path and its refusal contract passed a counted end-to-end stand-in or protocol qualification. A transport- or application-specific acceptance record remains separately bound to the exact deployment. | +| **Experimental** | Real implementation exists, but evidence is opt-in, infrastructure-gated, mocked at an external boundary, or otherwise below a counted acceptance record. | | **Research** | Protocol spike, design, or synthetic proof. Not a supported deployment surface. | | **Deprecated** | Superseded path retained only for migration. Do not start new work on it. | | **Archived** | Historical code with no active product role. | @@ -31,16 +31,16 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). | Windows UIA replay | **Scoped acceptance** | Candidate `20260717-candidate-56759c8-v2` completed 3/3 exact WinForms trials with independently confirmed SQLite effects and 12 native UIA delivery receipts. Stale and ambiguous targets each refused 3/3; silent incorrect successes, over-halts, and model calls were zero. See [`benchmark/windows_uia/results.json`](../benchmark/windows_uia/results.json). | Acceptance covers the in-tree WinForms workflow and exact Windows VM. Each third-party application is qualified against its own controls, versions, identity rules, and effect oracle. | | Desktop recording (`windows` / `rdp`) | **Beta, capture-assisted** | `openadapt-capture` conversion, compile, and replay orchestration run in CI, and the native substrate qualifications below prove desktop actuation paths. | Structural UIA evidence is collected by the live Windows observer rather than reconstructed from an offline pixel recording. Regulated profiles require declared secret handling and fail-closed privacy configuration. | | Native macOS desktop actuation | **Scoped acceptance** | Candidate `b1b61a5` completed 3/3 TextEdit replace-and-save trials with exact file-byte effects and refused a two-window ambiguous selector without changing either file. See the [accepted evidence adjudication](../benchmark/macos_native/textedit_counted_3plus1_b1b61a5_20260717.adjudication.json). | Acceptance covers TextEdit on one macOS 15.7.3 Apple Silicon host and active user session. Customer applications require workflow-specific qualification. | -| Native macOS AX structured identity | **Experimental** | The macOS backend implements the same structured-layer contract as the browser DOM, Windows UIA, and Linux AT-SPI backends: it records a stable AX locator, re-finds the UNIQUE element at replay, refuses ambiguous / truncated / scope-escaping enumeration instead of guessing, and returns the real structured text under a point, so the same-name/same-DOB sibling adversary is caught structurally rather than falling through to OCR. Headless unit CI covers record/locate/refuse; a live-AX TextEdit run produced real evidence ([AX identity adjudication](../benchmark/macos_native/ax_identity_20260720.adjudication.json)); the record→compile→replay conformance test asserts zero model calls on a healthy replay. See [`tests/test_macos_structural.py`](../tests/test_macos_structural.py) and the [capability matrix](../tests/test_backend_capability_matrix.py). | The backend does NOT claim native AXPress actuation; a structurally resolved element is acted on through the fully gated point-bound physical click. AX exposure varies by application; where a control has no durable AX identity, resolution falls back to the visual ladder. Not a general per-application support claim. | -| Native Linux desktop actuation | **Scoped acceptance candidate** | The required `linux-atspi-x11` job runs a real GTK3 application against AT-SPI inside an isolated Xvfb/session-D-Bus environment: 3 clean exact-file-effect trials, 3 ambiguous-target refusals, and 3 stale-target refusals. Unit CI covers the remaining window, traversal, capture, physical-input, and portal boundaries. | Acceptance is bounded to the in-tree GTK3 workflow and CI image. Each customer app and environment requires its own qualification. Wayland requires a live operator-approved XDG portal session; the built-in client refuses without one. | +| Native macOS AX structured identity | **Scoped acceptance** | The macOS backend implements the same structured-layer contract as the browser DOM, Windows UIA, and Linux AT-SPI backends: it records a stable AX locator, re-finds the UNIQUE element at replay, refuses ambiguous / truncated / scope-escaping enumeration instead of guessing, and returns structured text under a point. Headless unit CI covers record/locate/refuse; a live-AX TextEdit run produced real evidence ([AX identity adjudication](../benchmark/macos_native/ax_identity_20260720.adjudication.json)); the record→compile→replay conformance test asserts zero model calls on healthy replay. See [`tests/test_macos_structural.py`](../tests/test_macos_structural.py) and the [capability matrix](../tests/test_backend_capability_matrix.py). | The backend uses gated point-bound physical click after structural resolution rather than claiming AXPress everywhere. AX exposure varies by application; controls without durable AX identity use the visual ladder. | +| Native Linux desktop actuation | **Scoped acceptance** | The required `linux-atspi-x11` job runs a real GTK3 application against AT-SPI inside an isolated Xvfb/session-D-Bus environment: 3 clean exact-file-effect trials, 3 ambiguous-target refusals, and 3 stale-target refusals. Unit CI covers the remaining window, traversal, capture, physical-input, and portal boundaries. | Acceptance is bounded to the in-tree GTK3 workflow and CI image. Each application and environment retains its own qualification. The built-in driver uses X11; Wayland requires a live operator-approved XDG portal session and refuses without one. | | RDP | **Scoped acceptance** | Candidate `82a658a` completed 3/3 real-network Aardwolf RDP trials into Windows 11, with a guest-tools file oracle, zero failures, zero silent incorrect successes, zero over-halts, and zero model calls. See the [accepted batch](../benchmark/rdp/ACCEPTED_BATCH_82A658A.md). | Acceptance covers the tested 1280×800 transport/input task. Target applications, identity/effect rules, session policies, and display conditions are qualified per deployment. | -| Citrix / pixel-only remote display | **Design-partner qualification** | The pixel backend seam, refusal behavior, and remote-display workflow contract are implemented and exercised through CI and the qualified RDP path. | Citrix ICA/HDX acceptance requires access to the customer's exact published application, session policy, display configuration, and independent effect oracle; RDP evidence is not represented as Citrix evidence. | +| Citrix / pixel-only remote display | **Code-qualified** | `--backend citrix` binds an exact Citrix Workspace window, readiness marker, pixel-only ladder, governed run, durable resume, and report. The retained no-DOM qualification passed 3 healthy effect-confirmed trials and 3 drift safe-halts with zero model calls, silent incorrect successes, or false completion; it records `code_readiness_accepted=true`. | The same artifact records `ica_hdx_accepted=false`: a counted live ICA/HDX batch remains bound to the exact Workspace/server/application matrix and independent effect oracle rather than being inferred from RDP or the stand-in. | | Identity verification | **Experimental, armed steps only** | Wrong-entity refusal and adversarial corpora run in CI. | Unarmed clicks have no identity check. Real compiled bundles currently arm only a subset of clicks. | | System-of-record effect verification | **Experimental** | REST, FHIR, and document-hash verifier contracts catch fault classes that screen-only verification misses. | Effects are not generally inferred; both authored effects and a configured verifier are required. | | Lint and certification policies | **Beta** | The CLI reports coverage gaps and refuses bundles that violate a selected policy. | Certification is opt-in; `replay` remains the permissive tutorial path. Use fail-closed `run` for a deployment. | | Durable pause, approval, and resume | **Experimental** | Checkpoint, bundle-version binding, approval, stale-pause, and resume semantics are tested. | Operator identity is recorded, not integrated with an enterprise IdP; field operation is unmeasured. | | On-prem / air-gapped deployment | **Beta foundation** | Local queue, fail-closed run gate, egress attestation, audit-chain verification, signed release verification, fresh-environment smoke/air-gap checks, atomic blue/green update, and rollback ship. | Site firewall, storage, keys, OS hardening, identity/effect integrations, and acceptance in the customer's environment remain deployment responsibilities. | -| Desktop GUI and tray | **Experimental, separate repositories** | Component-level engines and rewiring branches exist. | No integrated, supported installer proves record through governed production operation today. | +| Desktop GUI and tray | **Beta, separate repository** | Desktop `v0.9.0` ships installable Windows, macOS, and Linux artifacts with checksums; its frozen engine lifecycle and install/launch/uninstall contracts run in release CI. | The published installer evidence covers the embedded browser lifecycle. Native and remote substrate selection is independently qualified through Flow and remains bound to the selected target configuration. | | Hosted dashboard / control plane | **Live beta, separate repository** | The deployed service uses live Supabase, Stripe, and Modal dependencies for account and organization onboarding, checkout, exact-hash artifact ingest, attested browser workflow versions, structural reports, replacement activation, scheduling, entitlements, and metering. The reversible pre-payment contract passed 3/3 production trials. | The first genuine customer payment remains the acceptance event for the paid post-payment lifecycle. SLA, BAA, and compliance commitments apply only when included in reviewed written terms. | | Hosted execution | **Live beta — browser** | Production mode admits exact attested browser bundles, dispatches the configured runner, authenticates callbacks, and refuses mock fallback; development mock mode remains visibly synthetic. | Desktop, RDP, Citrix, and customer-controlled regulated execution are separately scoped deployment lanes rather than capabilities implied by the browser subscription. | | Offline update and rollback | **Beta** | The operator-pulled path verifies signed archives, installs into a fresh blue/green environment, runs smoke and air-gap checks, atomically swaps the active release, and records rollback state. | Signer trust, artifact transport, OS/container policy, backup, disaster recovery, and a customer-site rehearsal remain deployment responsibilities. | diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 199c9f4c..c982d8f4 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -6,7 +6,7 @@ - Green-check against a junit artifact: **not run** (no `--junit` artifact supplied) - Gate: `python scripts/validate_claims.py --check` (a claim whose tier outranks its strongest backing evidence fails CI). -**What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test that actually runs on the default (non-opt-in) CI suite; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does NOT replace the human half — third-party design-partner validation of the `validating` surfaces (Windows, Citrix) is exactly the evidence this repository cannot self-generate. +**What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test that actually runs on the default (non-opt-in) CI suite; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. ## What is CI-proven today vs. being validated @@ -31,7 +31,7 @@ **Caveats (honest limits):** -- "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and design-partner claims below. +- "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below. - The full record->compile->replay e2e runs in the post-merge/nightly full suite; the fast required PR gate covers the replayer + demo units. ### `deterministic-zero-model-replay` — supported — CI-proven today @@ -252,7 +252,7 @@ ### `citrix-pixel-validating` — validating — opt-in / infra-gated or field test -> Citrix uses the governed pixel-only remote-display contract and enters design-partner qualification in the customer's exact ICA/HDX application, with identity, ambiguity refusal, and independent effects above the driver. +> Citrix uses a dedicated exact-Workspace-window backend beneath the governed pixel-only remote-display contract. The backend, readiness gate, durable resume, identity/refusal ladder, and independent effects are code-qualified; live ICA/HDX acceptance remains separately bound to the exact deployment. - Surfaces: README.md, docs, website - Strongest evidence strength: **supported** (tier is `validating`) @@ -261,12 +261,14 @@ |---|---|---|---|---| | `tests/e2e/test_citrix_pixel_e2e.py` | test | opt-in (OAFLOW_CITRIX_PIXEL_E2E) | validating | OPT-IN pixel-only proof: structural_armed_coverage == 0, replay resolves on template/ocr/geometry only, on-screen OCR read-back verifies the write, the identity gate HALTs on a look-alike patient, and render drift triggers halt-on-ambiguity. | | `tests/test_pixel_identity_probe.py` | test | ci (required PR gate (test)) | supported | The pixel-substrate identity probe behind the on-pixels identity gate. | -| `docs/desktop/CITRIX_PIXEL.md` | doc | artifact (doc/benchmark) | roadmap | The Citrix driver model, reusable evidence, design-partner acceptance contract, independent-effect boundary, and customer-controlled posture. | +| `tests/e2e/test_citrix_workspace_standin_e2e.py` | test | ci (post-merge/nightly full suite) | supported | The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success. | +| `benchmark/citrix_workspace/results.json` | artifact | artifact (doc/benchmark) | roadmap | The retained six-trial record reports code_readiness_accepted=true and ica_hdx_accepted=false, keeping driver readiness distinct from a counted live ICA/HDX qualification. | +| `docs/desktop/CITRIX_PIXEL.md` | doc | artifact (doc/benchmark) | roadmap | The Citrix driver model, reusable evidence, exact-deployment acceptance contract, independent-effect boundary, and customer-controlled posture. | **Caveats (honest limits):** - PIXEL-ONLY: no UIA/DOM crosses to the driver. On-screen OCR read-back is SAME-SURFACE, not an independent system-of-record check. -- A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); never runs on default CI, and does not replace design-partner validation. +- A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); never runs on default CI, and does not replace a counted live ICA/HDX acceptance record. ### `win32-window-replay-roadmap` — roadmap — designed, not yet proven @@ -278,7 +280,7 @@ | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| | `tests/test_win32_window_client.py` | test | ci (required PR gate (test)) | supported | MOCK-ONLY conformance: the win32 WindowClient implements the remote-display WindowClient seam (exact process/title/class selection with ambiguity halt, client-area coordinate mapping under per-monitor DPI awareness, PrintWindow-then-BitBlt capture fallback, focus-verification refusal, and the UIPI elevation guard) against a scripted Win32 API — never against a real Windows host. | -| `docs/desktop/CITRIX_PIXEL.md` | doc | artifact (doc/benchmark) | roadmap | The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, then design-partner Citrix). | +| `docs/desktop/CITRIX_PIXEL.md` | doc | artifact (doc/benchmark) | roadmap | The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, then a counted live Citrix host run). | **Caveats (honest limits):** diff --git a/docs/backends/RDP.md b/docs/backends/RDP.md index d0e1c718..17a7f74e 100644 --- a/docs/backends/RDP.md +++ b/docs/backends/RDP.md @@ -169,5 +169,6 @@ statistical reliability claim for every Windows application. For a production workflow, record and qualify the customer's exact application under its real account/session policy, DPI and scaling, disconnect/reconnect behavior, latency envelope, identity evidence, and independent effect oracle. -Citrix ICA/HDX is a separate design-partner qualification; the RDP batch is not -used as Citrix acceptance evidence. +Citrix ICA/HDX receives a separate counted qualification on its exact +Workspace/server/application matrix; the RDP batch is not used as Citrix +acceptance evidence. diff --git a/docs/desktop/CITRIX_PIXEL.md b/docs/desktop/CITRIX_PIXEL.md index a8435db2..945c5c73 100644 --- a/docs/desktop/CITRIX_PIXEL.md +++ b/docs/desktop/CITRIX_PIXEL.md @@ -66,7 +66,7 @@ environment because the published application, Workspace policy, rendering, session behavior, and available effect oracle determine the workflow's accepted scope. -## Design-partner qualification +## Exact-deployment qualification A Citrix deployment starts with one repeated workflow and one independently observable business result. The acceptance record names: diff --git a/docs/verification.json b/docs/verification.json index baa9a3b8..e748efc3 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -15,7 +15,7 @@ ], "strongest_evidence": "supported", "caveats": [ - "\"Supported\" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and design-partner claims below.", + "\"Supported\" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below.", "The full record->compile->replay e2e runs in the post-merge/nightly full suite; the fast required PR gate covers the replayer + demo units." ], "evidence": [ @@ -720,7 +720,7 @@ }, { "id": "citrix-pixel-validating", - "claim": "Citrix uses the governed pixel-only remote-display contract and enters design-partner qualification in the customer's exact ICA/HDX application, with identity, ambiguity refusal, and independent effects above the driver.", + "claim": "Citrix uses a dedicated exact-Workspace-window backend beneath the governed pixel-only remote-display contract. The backend, readiness gate, durable resume, identity/refusal ladder, and independent effects are code-qualified; live ICA/HDX acceptance remains separately bound to the exact deployment.", "tier": "validating", "reproducibility": null, "surfaces": [ @@ -731,7 +731,7 @@ "strongest_evidence": "supported", "caveats": [ "PIXEL-ONLY: no UIA/DOM crosses to the driver. On-screen OCR read-back is SAME-SURFACE, not an independent system-of-record check.", - "A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); never runs on default CI, and does not replace design-partner validation." + "A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); never runs on default CI, and does not replace a counted live ICA/HDX acceptance record." ], "evidence": [ { @@ -756,6 +756,28 @@ "junit_status": null, "proves": "The pixel-substrate identity probe behind the on-pixels identity gate." }, + { + "path": "tests/e2e/test_citrix_workspace_standin_e2e.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (post-merge/nightly full suite)", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success." + }, + { + "path": "benchmark/citrix_workspace/results.json", + "kind": "artifact", + "exists": true, + "strength": "roadmap", + "gating": "artifact (doc/benchmark)", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "The retained six-trial record reports code_readiness_accepted=true and ica_hdx_accepted=false, keeping driver readiness distinct from a counted live ICA/HDX qualification." + }, { "path": "docs/desktop/CITRIX_PIXEL.md", "kind": "doc", @@ -765,7 +787,7 @@ "node": null, "node_found": null, "junit_status": null, - "proves": "The Citrix driver model, reusable evidence, design-partner acceptance contract, independent-effect boundary, and customer-controlled posture." + "proves": "The Citrix driver model, reusable evidence, exact-deployment acceptance contract, independent-effect boundary, and customer-controlled posture." } ], "errors": [] @@ -803,7 +825,7 @@ "node": null, "node_found": null, "junit_status": null, - "proves": "The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, then design-partner Citrix)." + "proves": "The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, then a counted live Citrix host run)." } ], "errors": [] diff --git a/public-artifacts.json b/public-artifacts.json index ed4ff101..0d05a6db 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -556,7 +556,7 @@ }, { "path": "claims.yaml", - "sha256": "4b5fe58ec8dc2bb2e380c815f737210b5f023f87ebad5b0677963deaf091a4d0" + "sha256": "82877909e7f14167b7076f5516e6654b1d8ab7f9218e9e8869a84173c1005a9a" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1764,7 +1764,7 @@ }, { "path": "docs/verification.json", - "sha256": "89bf037a6d975ee98e15a44757a4e4716d91dd2b3fc7840806efc4bd0f5bd4dc" + "sha256": "debbcdd83db5670295d6878b97a52f7c5350ec9be683effd60799fb34588125d" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/scripts/validate_claims.py b/scripts/validate_claims.py index 5e0970e4..3a8a8e9c 100644 --- a/scripts/validate_claims.py +++ b/scripts/validate_claims.py @@ -385,9 +385,9 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s "must be backed by a test that actually runs on the default (non-opt-in) " "CI suite; a `validating` claim must be grounded in a REAL opt-in / " "infra-gated proof or a field test, and is never presented as " - "supported. It does NOT replace the human half — third-party " - "design-partner validation of the `validating` surfaces (Windows, " - "Citrix) is exactly the evidence this repository cannot self-generate." + "supported. It does not replace workflow- and deployment-specific " + "acceptance: application controls, identity rules, effect oracles, and " + "live transport conditions remain bound to their counted evidence." ) lines.append("") From 5323b9cf1abb3d0d83200882c266350b3e097204 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 18:04:22 -0400 Subject: [PATCH 02/10] qualify desktop substrate claims --- README.md | 18 +- benchmark/vision_hardening/README.md | 6 +- claims.yaml | 83 +- docs/PRODUCT_STATUS.md | 4 +- docs/SANITIZED_ARTIFACTS.md | 32 +- openadapt_flow/__main__.py | 37 +- openadapt_flow/ir.py | 9 + openadapt_flow/runtime/durable/resume.py | 9 +- openadapt_flow/runtime/replayer.py | 6 + openadapt_flow/runtime_validation.py | 92 +- .../runtime-validation-attestation-v2.json | 139 +++ scripts/qualify_macos_ax_release.py | 849 ++++++++++++++++++ scripts/validate_claims.py | 87 ++ .../runtime-validation-canonical-vector.json | 7 +- tests/test_backend_factory.py | 2 + tests/test_cli_new_commands.py | 20 +- tests/test_hosted.py | 3 + tests/test_macos_ax_release_qualification.py | 146 +++ tests/test_run_gate.py | 11 +- tests/test_runtime_validation.py | 134 ++- tests/test_sanitized_artifact.py | 1 + tests/test_validate_claims.py | 3 + 22 files changed, 1636 insertions(+), 62 deletions(-) create mode 100644 schemas/runtime-validation-attestation-v2.json create mode 100644 scripts/qualify_macos_ax_release.py create mode 100644 tests/test_macos_ax_release_qualification.py diff --git a/README.md b/README.md index 1ac70e59..8b5d50bc 100644 --- a/README.md +++ b/README.md @@ -221,9 +221,11 @@ runtime; none is a second-class add-on. Maturity is reported honestly per the | RDP (remote display) | `--backend rdp` | Available | Real-network Aardwolf RDP into Windows 11 passed 3/3 fixed remote-input trials with independent guest-tools file verification; a separate real-FreeRDP batch covers record → compile → governed replay and refusal | | Citrix / VDI (pixel ladder) | `--backend citrix` | Code-qualified | Dedicated exact-Workspace-window driver, readiness gate, durable resume, and 3 healthy + 3 drift-halt no-DOM trials; the retained artifact records `ica_hdx_accepted=false` until a counted ICA/HDX run is attached | -These are accepted scoped qualifications. A customer application is qualified -against its own controls, session/display policy, identity evidence, and effect -oracle. Details: +Every row is bounded to its stated evidence. Accepted application workflows are +qualified against their own controls, session/display policy, identity evidence, +and effect oracle; code-qualified Citrix deployments additionally attach the +counted ICA/HDX record for their exact Workspace/server/application matrix. +Details: [`docs/backends/RDP.md`](docs/backends/RDP.md), [`docs/desktop/LINUX_NATIVE.md`](docs/desktop/LINUX_NATIVE.md), [`docs/desktop/CITRIX_PIXEL.md`](docs/desktop/CITRIX_PIXEL.md). @@ -514,7 +516,7 @@ openadapt-flow push ./triage.sanitized --kind recording openadapt-flow compile ./triage.sanitized --out ./triage.bundle --name triage openadapt-flow lint ./triage.bundle --strict openadapt-flow certify ./triage.bundle --policy permissive -openadapt-flow replay ./triage.bundle --url https://example.internal/login \ +openadapt-flow replay ./triage.bundle --url https://app.example.com/login \ --run-dir ./triage.run --param patient_id=example # Privacy-review the executable bytes. A changed executable is refused. @@ -527,11 +529,17 @@ openadapt-flow approve-sanitized ./triage.bundle.sanitized \ openadapt-flow validate-hosted \ --recording ./triage.sanitized --bundle ./triage.bundle.sanitized \ --run-dir ./triage.run --policy permissive --risk-class low \ - --environment staging-v1 --target-url https://example.internal/login \ + --environment staging-v1 --target-kind web \ + --target-url https://app.example.com/login \ --out triage.validation.json openadapt-flow push ./triage.bundle.sanitized --kind bundle \ --validation-attestation triage.validation.json +# Desktop/RDP/Citrix use the same validation command, deriving the signed +# target kind from report.json. Their app/window/host details stay local: +# --target-kind citrix --environment clinic-citrix-qualified-v1 +# (omit --target-url and --allowed-host outside the web substrate). + # To activate this as a new version of an existing workflow, add: # --workflow-id 00000000-0000-0000-0000-000000000000 # To bind that replacement to the exact halted run it repairs, also add: diff --git a/benchmark/vision_hardening/README.md b/benchmark/vision_hardening/README.md index 937066e2..273c3871 100644 --- a/benchmark/vision_hardening/README.md +++ b/benchmark/vision_hardening/README.md @@ -187,9 +187,9 @@ pytest -q tests/test_hardening_flywheel.py - These are **synthetic fixtures**, not a live RDP/Citrix/HDX frame. The sweep measures the *resolution ladder's* silent-mis-resolution behaviour under a - controlled metamorphic perturbation space. It **does not** raise any - substrate's maturity claim; RDP/Citrix pixel-only stays **Research** - (`docs/LIMITS.md`). + controlled metamorphic perturbation space. It **does not independently + qualify** a substrate; the separate counted RDP and Citrix records and their + exact scope remain authoritative (`docs/LIMITS.md`). - It **adds no capability and weakens no guard** — it only measures the existing resolver and locks in that the silent-wrong rate can only fall. - The over-halt vs safe-halt split uses a documented per-perturbation severity diff --git a/claims.yaml b/claims.yaml index ce0e3e27..7c0f2bc2 100644 --- a/claims.yaml +++ b/claims.yaml @@ -443,11 +443,46 @@ claims: environment, not arbitrary applications, identity policies, or Citrix. # ------------------------------ Desktop recording via the CLI (validating) + - id: substrate-runtime-validation-supported + claim: >- + Runtime-validation v2 binds the successful run's resolved target kind + (`web|windows|macos|linux|rdp|citrix`) into the signed operator + attestation while preserving the same exact artifact, compiler, + parameter-schema, strict-lint, policy, risk, report, and opaque + environment bindings. Web retains its exact HTTPS/host boundary; + native and remote execution emits no app, window, host, readiness, or + backend-hint values to Cloud. + surfaces: [README.md, docs, website] + tier: supported + evidence: + - path: tests/test_runtime_validation.py + proves: >- + Required CI exercises all six target kinds, validates v2 against its + closed JSON Schema, refuses target-kind mismatch or missing report + provenance, preserves web URL/host checks, keeps native/remote + execution empty, verifies the canonical HMAC vector, and retains + signed v1 web compatibility. + - path: tests/test_cli_new_commands.py + proves: >- + Required CI carries the resolved web, Windows, and Citrix backend + token through durable resume into the new run report rather than + trusting an upload-time relabel. + - path: schemas/runtime-validation-attestation-v2.json + kind: doc + proves: >- + The public closed envelope requires target_kind and conditionally + permits the browser execution fields only for web. + caveats: + - >- + The HMAC proves operator-token possession and envelope integrity; it is + not independent observation of the local replay or certification of an + arbitrary target application. + - id: desktop-recording-validating claim: >- The product CLI records desktop workflows through openadapt-capture and - `record --backend windows|rdp` captures the operator's real demonstration - and emits the compile-ready recording format, so + `record --backend windows|macos|linux|rdp|citrix` captures the operator's + real demonstration and emits the compile-ready recording format, so record -> compile -> replay closes on the desktop substrate, not just the browser. surfaces: [README.md, docs, website] @@ -456,10 +491,10 @@ claims: - path: tests/test_desktop_record.py proves: >- The live-capture orchestration + CLI wiring (record --backend - windows|rdp -> openadapt-capture -> convert_capture), and that a - desktop-shaped recording COMPILES into a bundle and REPLAYS to - completion through the desktop backend path, resolving each click to - its recorded target (runs on default CI). + windows|macos|linux|rdp|citrix -> openadapt-capture -> + convert_capture), and that a desktop-shaped recording COMPILES into a + bundle and REPLAYS to completion through the desktop backend path, + resolving each click to its recorded target (runs on default CI). - path: tests/test_capture_adapter.py proves: >- The capture->recording bridge over a REAL openadapt-capture session @@ -489,9 +524,11 @@ claims: - id: citrix-pixel-validating claim: >- Citrix uses a dedicated exact-Workspace-window backend beneath the governed - pixel-only remote-display contract. The backend, readiness gate, durable - resume, identity/refusal ladder, and independent effects are code-qualified; - live ICA/HDX acceptance remains separately bound to the exact deployment. + pixel-only remote-display contract. Required CI covers its readiness gate, + governed execution, durable resume, report binding, and refusal contracts; + the counted no-DOM qualification covers the driver, visual ladder, + independent effects, and drift halts. Live ICA/HDX acceptance remains + separately bound to the exact deployment. surfaces: [README.md, docs, website] tier: validating evidence: @@ -504,6 +541,27 @@ claims: - path: tests/test_pixel_identity_probe.py proves: >- The pixel-substrate identity probe behind the on-pixels identity gate. + - path: tests/test_citrix_workspace_backend.py + proves: >- + Required CI covers the dedicated backend preset, exact owner/title + overrides, readiness probe construction, pixel-only capability + boundary, backend factory, and product CLI selectors. + - path: tests/test_run_gate.py + proves: >- + Required CI refuses governed Citrix execution before action when its + readiness binding is absent or blank, admits a sealed bundle carrying + a recorded readiness binding, and reports only the resolved backend + token rather than sensitive target strings. + - path: tests/test_cli_new_commands.py + proves: >- + Required CI restores the recorded Citrix owner, exact title, and + readiness binding through durable approve/resume, and refuses before + backend construction when resumed configuration is incomplete. + - path: tests/test_hosted.py + proves: >- + Required CI binds a completed local Citrix report to the closed + `citrix` execution token without copying target owner, title, or + readiness values into the hosted summary. - path: tests/e2e/test_citrix_workspace_standin_e2e.py proves: >- The dedicated Citrix backend passes three healthy effect-confirmed @@ -529,9 +587,10 @@ claims: - >- A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated - (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); - never runs on default CI, and does not replace a counted live ICA/HDX - acceptance record. + (`OAFLOW_CITRIX_PIXEL_E2E=1` or + `OAFLOW_CITRIX_STANDIN_E2E=1`); neither live-window proof runs on + default CI, and neither replaces a counted live ICA/HDX acceptance + record. # ------------------------ Windows-host window replay (mock-tested, roadmap) - id: win32-window-replay-roadmap diff --git a/docs/PRODUCT_STATUS.md b/docs/PRODUCT_STATUS.md index 38fb1ba9..62043074 100644 --- a/docs/PRODUCT_STATUS.md +++ b/docs/PRODUCT_STATUS.md @@ -29,12 +29,12 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). | AI-assisted repair | **Experimental** | Local/remote VLM contracts, egress gates, refusal behavior, and retention boundaries are tested. | Off by default; model accuracy is not a safety guarantee and real deployment quality is unmeasured. | | Human teaching (`teach`) | **Experimental** | Halt-to-correction-to-guarded-promotion and regression refusal run in default CI. | Evidence is controlled/synthetic; broad authoring UX and field recovery time are not established. | | Windows UIA replay | **Scoped acceptance** | Candidate `20260717-candidate-56759c8-v2` completed 3/3 exact WinForms trials with independently confirmed SQLite effects and 12 native UIA delivery receipts. Stale and ambiguous targets each refused 3/3; silent incorrect successes, over-halts, and model calls were zero. See [`benchmark/windows_uia/results.json`](../benchmark/windows_uia/results.json). | Acceptance covers the in-tree WinForms workflow and exact Windows VM. Each third-party application is qualified against its own controls, versions, identity rules, and effect oracle. | -| Desktop recording (`windows` / `rdp`) | **Beta, capture-assisted** | `openadapt-capture` conversion, compile, and replay orchestration run in CI, and the native substrate qualifications below prove desktop actuation paths. | Structural UIA evidence is collected by the live Windows observer rather than reconstructed from an offline pixel recording. Regulated profiles require declared secret handling and fail-closed privacy configuration. | +| Desktop recording (`windows` / `macos` / `linux` / `rdp` / `citrix`) | **Beta, capture-assisted** | `openadapt-capture` conversion, compile, and replay orchestration run in CI for every desktop selector, and the native substrate qualifications below prove the corresponding actuation paths. | Offline pixel capture cannot reconstruct structural accessibility evidence. Workflows that require UIA, AX, or AT-SPI identity use a live structural observer or are re-armed against the qualified application before release. Regulated profiles require declared secret handling and fail-closed privacy configuration. | | Native macOS desktop actuation | **Scoped acceptance** | Candidate `b1b61a5` completed 3/3 TextEdit replace-and-save trials with exact file-byte effects and refused a two-window ambiguous selector without changing either file. See the [accepted evidence adjudication](../benchmark/macos_native/textedit_counted_3plus1_b1b61a5_20260717.adjudication.json). | Acceptance covers TextEdit on one macOS 15.7.3 Apple Silicon host and active user session. Customer applications require workflow-specific qualification. | | Native macOS AX structured identity | **Scoped acceptance** | The macOS backend implements the same structured-layer contract as the browser DOM, Windows UIA, and Linux AT-SPI backends: it records a stable AX locator, re-finds the UNIQUE element at replay, refuses ambiguous / truncated / scope-escaping enumeration instead of guessing, and returns structured text under a point. Headless unit CI covers record/locate/refuse; a live-AX TextEdit run produced real evidence ([AX identity adjudication](../benchmark/macos_native/ax_identity_20260720.adjudication.json)); the record→compile→replay conformance test asserts zero model calls on healthy replay. See [`tests/test_macos_structural.py`](../tests/test_macos_structural.py) and the [capability matrix](../tests/test_backend_capability_matrix.py). | The backend uses gated point-bound physical click after structural resolution rather than claiming AXPress everywhere. AX exposure varies by application; controls without durable AX identity use the visual ladder. | | Native Linux desktop actuation | **Scoped acceptance** | The required `linux-atspi-x11` job runs a real GTK3 application against AT-SPI inside an isolated Xvfb/session-D-Bus environment: 3 clean exact-file-effect trials, 3 ambiguous-target refusals, and 3 stale-target refusals. Unit CI covers the remaining window, traversal, capture, physical-input, and portal boundaries. | Acceptance is bounded to the in-tree GTK3 workflow and CI image. Each application and environment retains its own qualification. The built-in driver uses X11; Wayland requires a live operator-approved XDG portal session and refuses without one. | | RDP | **Scoped acceptance** | Candidate `82a658a` completed 3/3 real-network Aardwolf RDP trials into Windows 11, with a guest-tools file oracle, zero failures, zero silent incorrect successes, zero over-halts, and zero model calls. See the [accepted batch](../benchmark/rdp/ACCEPTED_BATCH_82A658A.md). | Acceptance covers the tested 1280×800 transport/input task. Target applications, identity/effect rules, session policies, and display conditions are qualified per deployment. | -| Citrix / pixel-only remote display | **Code-qualified** | `--backend citrix` binds an exact Citrix Workspace window, readiness marker, pixel-only ladder, governed run, durable resume, and report. The retained no-DOM qualification passed 3 healthy effect-confirmed trials and 3 drift safe-halts with zero model calls, silent incorrect successes, or false completion; it records `code_readiness_accepted=true`. | The same artifact records `ica_hdx_accepted=false`: a counted live ICA/HDX batch remains bound to the exact Workspace/server/application matrix and independent effect oracle rather than being inferred from RDP or the stand-in. | +| Citrix / pixel-only remote display | **Code-qualified** | `--backend citrix` binds an exact Citrix Workspace window, readiness marker, pixel-only ladder, governed run, durable resume, and report; required CI covers those orchestration and refusal contracts. Separately, the retained no-DOM driver qualification passed 3 healthy effect-confirmed trials and 3 drift safe-halts with zero model calls, silent incorrect successes, or false completion, and records `code_readiness_accepted=true`. | The counted stand-in does not claim that its ordinary replay exercised every governed/durable CLI path. The same artifact records `ica_hdx_accepted=false`: live ICA/HDX acceptance remains bound to the exact Workspace/server/application matrix and independent effect oracle rather than being inferred from RDP or the stand-in. | | Identity verification | **Experimental, armed steps only** | Wrong-entity refusal and adversarial corpora run in CI. | Unarmed clicks have no identity check. Real compiled bundles currently arm only a subset of clicks. | | System-of-record effect verification | **Experimental** | REST, FHIR, and document-hash verifier contracts catch fault classes that screen-only verification misses. | Effects are not generally inferred; both authored effects and a configured verifier are required. | | Lint and certification policies | **Beta** | The CLI reports coverage gaps and refuses bundles that violate a selected policy. | Certification is opt-in; `replay` remains the permissive tutorial path. Use fail-closed `run` for a deployment. | diff --git a/docs/SANITIZED_ARTIFACTS.md b/docs/SANITIZED_ARTIFACTS.md index cfb8932c..a679893a 100644 --- a/docs/SANITIZED_ARTIFACTS.md +++ b/docs/SANITIZED_ARTIFACTS.md @@ -19,7 +19,7 @@ openadapt-flow push triage.sanitized/ --kind recording openadapt-flow compile triage.sanitized/ --out triage.bundle/ --name triage openadapt-flow lint triage.bundle/ --strict openadapt-flow certify triage.bundle/ --policy permissive -openadapt-flow replay triage.bundle/ --url https://example.internal/login \ +openadapt-flow replay triage.bundle/ --url https://app.example.com/login \ --run-dir triage.run/ --param patient_id=example openadapt-flow sanitize triage.bundle/ --kind bundle \ @@ -31,7 +31,8 @@ openadapt-flow approve-sanitized triage.bundle.sanitized/ \ openadapt-flow validate-hosted --recording triage.sanitized/ \ --bundle triage.bundle.sanitized/ --run-dir triage.run/ \ --policy permissive --risk-class low --environment staging-v1 \ - --target-url https://example.internal/login --out triage.validation.json + --target-kind web --target-url https://app.example.com/login \ + --out triage.validation.json openadapt-flow push triage.bundle.sanitized/ --kind bundle \ --validation-attestation triage.validation.json ``` @@ -154,15 +155,22 @@ remain `not-preserved` and are refused by `push`. `validate-hosted` is deliberately later than privacy approval. It recomputes strict lint and policy certification, requires a successful non-halted report bound to the same bundle/source recording/parameter schema, derives the -bundle's `low` or `consequential` risk class, and signs the exact HTTPS target -origin and host allowlist against a short-lived one-time Cloud challenge. The +bundle's `low` or `consequential` risk class, and signs the report's resolved +target kind (`web`, `windows`, `macos`, `linux`, `rdp`, or `citrix`) against a +short-lived one-time Cloud challenge. A supplied `--target-kind` is only a +cross-check; it cannot relabel the run. + +For `web`, v2 retains the exact HTTPS target origin and host allowlist. Managed +browser targets must use public DNS names; literal IP, loopback, +private/link-local resolution, wildcard, and special-use hostnames are refused. +For native and remote targets, `execution` is deliberately empty: app names, +window titles, remote hosts, readiness text, and backend hints can identify a +patient or customer environment and never cross this boundary. +`--environment` names the exact qualified runner boundary, and only its SHA-256 +is included. Cloud binds that digest during activation; a descriptive label +that was never qualified is not evidence by configuration alone. + Cloud admission also requires exact membership in its policy, risk-class, and -deployed compiler-version allowlists. Managed hosted targets must use public DNS -names; literal IP, loopback, private/link-local resolution, wildcard, and -special-use hostnames are refused. The HMAC is operator evidence: it proves +deployed compiler-version allowlists. The HMAC is operator evidence: it proves token possession and envelope integrity, but does not mean Cloud or an -independent auditor witnessed the local replay. `--environment` must name the -exact runner boundary qualified by the deployment operator. Cloud hashes that -identifier, binds it during activation, and the runner refuses a job whose -boundary ID/hash differs; a descriptive label that was never qualified is not -accepted as evidence by configuration alone. +independent auditor witnessed the local replay. diff --git a/openadapt_flow/__main__.py b/openadapt_flow/__main__.py index 1a66cf65..db2dc702 100644 --- a/openadapt_flow/__main__.py +++ b/openadapt_flow/__main__.py @@ -54,12 +54,12 @@ import sys from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Any, Iterator, Optional, Sequence +from typing import TYPE_CHECKING, Any, Iterator, Optional, Sequence, cast from urllib.parse import urlsplit if TYPE_CHECKING: # pragma: no cover from openadapt_flow.backend import Backend - from openadapt_flow.ir import RunReport + from openadapt_flow.ir import ExecutionTargetKind, RunReport _VIEWPORT = {"width": 1280, "height": 800} @@ -345,7 +345,7 @@ def _resolve_backend_config(args: argparse.Namespace, cfg, workflow=None): return backend -def _report_backend_kind(kind: object) -> str: +def _report_backend_kind(kind: object) -> "ExecutionTargetKind": """Return the closed hosted-summary token for a resolved backend kind. Execution accepts ``remote-display`` / ``remote_display`` as RDP aliases. @@ -355,7 +355,9 @@ def _report_backend_kind(kind: object) -> str: token = str(kind).strip().lower() if token in ("remote-display", "remote_display"): return "rdp" - return token + if token not in {"web", "windows", "macos", "linux", "rdp", "citrix"}: + raise ValueError(f"unsupported execution target kind: {kind!r}") + return cast("ExecutionTargetKind", token) def _refuse_missing_citrix_readiness(backend_cfg: object, *, operation: str) -> bool: @@ -431,6 +433,7 @@ def _build_and_run_replayer( pixel_verify_enabled: bool = False, governed_authorization=None, runtime_config=None, + execution_target_kind: Optional["ExecutionTargetKind"] = None, execution_origin: Optional[str] = None, execution_entry_url: Optional[str] = None, ): @@ -452,6 +455,7 @@ def _build_and_run_replayer( bundle_dir=bundle, run_dir=run_dir, save_healed_to=save_healed_to, + execution_target_kind=execution_target_kind, execution_origin=execution_origin, execution_entry_url=execution_entry_url, ) @@ -535,6 +539,7 @@ def _replay_desktop( pixel_verify_enabled=pixel_verify_enabled, governed_authorization=governed_authorization, runtime_config=runtime_config, + execution_target_kind=_report_backend_kind(backend_cfg.kind), ) finally: close = getattr(backend, "close", None) @@ -947,6 +952,7 @@ def _cmd_replay(args: argparse.Namespace) -> int: args, "_governed_run_authorization", None ), runtime_config=cfg.runtime, + execution_target_kind="web", execution_origin=( f"{urlsplit(page.url).scheme}://{urlsplit(page.url).netloc}" ), @@ -1150,7 +1156,12 @@ def _resume_with(backend: "Backend") -> "RunReport": checkpoint_key=ckpt_key, allow_model_grounding=allow_egress, ) - return resume(run_dir, replayer, key=ckpt_key) + return resume( + run_dir, + replayer, + key=ckpt_key, + execution_target_kind=_report_backend_kind(backend_cfg.kind), + ) try: if _normalize_kind(backend_cfg.kind) == "web": @@ -1634,6 +1645,7 @@ def _cmd_validate_hosted(args: argparse.Namespace) -> int: policy_source=args.policy, risk_class=args.risk_class, environment=args.environment, + target_kind=args.target_kind, target_url=args.target_url, allowed_hosts=args.allowed_host, compiler_config=compiler_config, @@ -3008,12 +3020,21 @@ def build_parser() -> argparse.ArgumentParser: "Non-PHI validation environment identifier; only its SHA-256 is uploaded" ), ) + p.add_argument( + "--target-kind", + choices=["web", "windows", "macos", "linux", "rdp", "citrix"], + default=None, + help=( + "Optional expected replay substrate. The signed value is derived " + "from report.json; a supplied value must match it exactly." + ), + ) p.add_argument( "--target-url", - required=True, + default=None, help=( - "Exact non-PHI HTTPS entry URL used by the validated browser workflow; " - "query strings, fragments, and credentials are refused" + "Web only: exact non-PHI HTTPS entry URL used by the validated " + "browser workflow; required for web and refused for native/remote" ), ) p.add_argument( diff --git a/openadapt_flow/ir.py b/openadapt_flow/ir.py index 02b5c18a..dbc4be2c 100644 --- a/openadapt_flow/ir.py +++ b/openadapt_flow/ir.py @@ -40,6 +40,7 @@ Region = tuple[int, int, int, int] Point = tuple[int, int] +ExecutionTargetKind = Literal["web", "windows", "macos", "linux", "rdp", "citrix"] #: Current bundle schema version. v2 adds the bundle manifest (per-asset #: hashes, a whole-bundle content digest, and compiler/certification @@ -1812,6 +1813,14 @@ class HaltObservation(BaseModel): class RunReport(BaseModel): workflow_name: str started_at: str + execution_target_kind: Optional[ExecutionTargetKind] = Field( + default=None, + description=( + "Resolved backend/substrate that produced this report. The CLI " + "sets this from the resolved backend configuration; runtime " + "validation refuses a native/remote attestation without it." + ), + ) execution_origin: Optional[str] = Field( default=None, description=( diff --git a/openadapt_flow/runtime/durable/resume.py b/openadapt_flow/runtime/durable/resume.py index a3dc2320..5a448df4 100644 --- a/openadapt_flow/runtime/durable/resume.py +++ b/openadapt_flow/runtime/durable/resume.py @@ -34,7 +34,7 @@ from pathlib import Path from typing import Any, Optional -from openadapt_flow.ir import RunReport, Workflow +from openadapt_flow.ir import ExecutionTargetKind, RunReport, Workflow from openadapt_flow.runtime.durable.approval import ( ApprovalRecord, BundleMismatch, @@ -70,6 +70,7 @@ def resume( params: Optional[dict[str, str]] = None, worklists: Optional[dict[str, list[dict[str, str]]]] = None, save_healed_to: Optional[Path | str] = None, + execution_target_kind: Optional[ExecutionTargetKind] = None, now: Optional[datetime] = None, key: Optional[str] = None, ) -> RunReport: @@ -94,6 +95,8 @@ def resume( worklists: Override the worklists recorded in the manifest; defaults to the original frozen worklists so program loops resume identically. save_healed_to: Override the manifest's healed-bundle path. + execution_target_kind: Resolved backend token for the resumed leg's + report and substrate-aware runtime attestation. now: Injectable clock for the stale-pause check (defaults to UTC now). key: At-rest passphrase for an ENCRYPTED run (its checkpoints and/or its bundle). Resolved from ``key`` or ``OPENADAPT_BUNDLE_KEY``. Used to @@ -177,6 +180,7 @@ def resume( save_healed_to=resolved_healed, live_bundle_version=live_bundle_version, run_id=(manifest.run_id if manifest is not None else None), + execution_target_kind=execution_target_kind, manifest=manifest, pending=pending, ) @@ -193,6 +197,7 @@ def resume( save_healed_to=(Path(resolved_healed) if resolved_healed else None), resume_from=start_index, run_id=(manifest.run_id if manifest is not None else None), + execution_target_kind=execution_target_kind, ) @@ -208,6 +213,7 @@ def _resume_program( save_healed_to: Optional[Path | str], live_bundle_version: str, run_id: Optional[str], + execution_target_kind: Optional[ExecutionTargetKind], manifest: Any, pending: Any, ) -> RunReport: @@ -269,4 +275,5 @@ def _resume_program( save_healed_to=(Path(save_healed_to) if save_healed_to else None), resume_program=checkpoint, run_id=run_id, + execution_target_kind=execution_target_kind, ) diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index f7ab08d2..e19822f0 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -48,6 +48,7 @@ from openadapt_flow.ir import ( ActionKind, Anchor, + ExecutionTargetKind, HaltObservation, IdentityCheck, Interstitial, @@ -496,6 +497,7 @@ def run( resume_from: Optional[int] = None, resume_program: Optional[ProgramCheckpoint] = None, run_id: Optional[str] = None, + execution_target_kind: Optional[ExecutionTargetKind] = None, execution_origin: Optional[str] = None, execution_entry_url: Optional[str] = None, ) -> RunReport: @@ -544,6 +546,9 @@ def run( New runs generate one; resumed legs preserve the manifest value so attended capabilities and effect idempotency remain bound to the same logical run. + execution_target_kind: Resolved backend token (web, windows, macos, + linux, rdp, or citrix). Runtime validation binds this report + field into the signed substrate-aware attestation. execution_origin: Actual browser origin loaded before replay. It is evidence metadata only; hosted validation requires an exact match to its signed target origin. @@ -601,6 +606,7 @@ def run( report = RunReport( workflow_name=workflow.name, started_at=datetime.now(timezone.utc).isoformat(), + execution_target_kind=execution_target_kind, execution_origin=execution_origin, execution_entry_url=execution_entry_url, bundle_content_digest=( diff --git a/openadapt_flow/runtime_validation.py b/openadapt_flow/runtime_validation.py index 5a179480..a9f6cc1d 100644 --- a/openadapt_flow/runtime_validation.py +++ b/openadapt_flow/runtime_validation.py @@ -47,7 +47,9 @@ load_valid_approval, ) -SCHEMA = "openadapt.runtime-validation/v1" +SCHEMA = "openadapt.runtime-validation/v2" +LEGACY_SCHEMA = "openadapt.runtime-validation/v1" +TARGET_KINDS = ("web", "windows", "macos", "linux", "rdp", "citrix") RISK_CLASSES = ("low", "consequential") _DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") _NON_PUBLIC_SUFFIXES = ( @@ -186,6 +188,40 @@ def normalize_execution_scope( } +def normalize_target_execution( + target_kind: str, + target_url: Optional[str], + allowed_hosts: Optional[list[str]], +) -> dict[str, Any]: + """Return the PHI-safe v2 execution contract for one resolved substrate. + + Browser execution retains the exact v1 HTTPS/public-DNS boundary. Native + and remote targets deliberately emit an empty object: app names, window + titles, hostnames, readiness text, and backend hints can identify a patient + or customer environment and must not cross into Cloud. Their exact + environment remains bound opaquely by ``replay.environment_sha256``. + """ + if target_kind not in TARGET_KINDS: + raise RuntimeValidationError( + "Target kind must be one of: " + ", ".join(TARGET_KINDS) + ) + if target_kind == "web": + if not isinstance(target_url, str) or not target_url: + raise RuntimeValidationError( + "Web runtime validation requires the exact HTTPS target URL" + ) + return normalize_execution_scope(target_url, allowed_hosts) + if target_url is not None: + raise RuntimeValidationError( + f"Target URL is web-only and must be omitted for {target_kind}" + ) + if allowed_hosts: + raise RuntimeValidationError( + f"Allowed hosts are web-only and must be omitted for {target_kind}" + ) + return {} + + def _canonical_bytes(value: Any) -> bytes: return json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False @@ -298,7 +334,8 @@ def create_runtime_validation_attestation( policy_source: str, risk_class: str, environment: str, - target_url: str, + target_kind: Optional[str] = None, + target_url: Optional[str] = None, allowed_hosts: Optional[list[str]] = None, compiler_config: Optional[dict[str, Any]] = None, host: Optional[str] = None, @@ -328,7 +365,10 @@ def create_runtime_validation_attestation( raise RuntimeValidationError( "Risk class must be one of: " + ", ".join(RISK_CLASSES) ) - execution = normalize_execution_scope(target_url, allowed_hosts) + if target_kind is not None and target_kind not in TARGET_KINDS: + raise RuntimeValidationError( + "Target kind must be one of: " + ", ".join(TARGET_KINDS) + ) resolved_host = resolve_host(host) resolve_destination_policy( resolved_host, @@ -468,13 +508,31 @@ def create_runtime_validation_attestation( raise RuntimeValidationError( "Run report parameter schema does not match the bundle" ) - if report.execution_origin != execution["target_origin"]: + report_target_kind = report.execution_target_kind + if report_target_kind not in TARGET_KINDS: raise RuntimeValidationError( - "Run report browser origin does not match the attested target origin" + "Run report has no valid resolved execution target kind; rerun the " + "workflow with the current substrate-aware Flow runtime" ) - if report.execution_entry_url != execution["entry_url"]: + if target_kind is not None and target_kind != report_target_kind: raise RuntimeValidationError( - "Run report browser entry URL does not match the attested target URL" + f"Requested target kind {target_kind!r} does not match the resolved " + f"run report target kind {report_target_kind!r}" + ) + target_kind = report_target_kind + execution = normalize_target_execution(target_kind, target_url, allowed_hosts) + if target_kind == "web": + if report.execution_origin != execution["target_origin"]: + raise RuntimeValidationError( + "Run report browser origin does not match the attested target origin" + ) + if report.execution_entry_url != execution["entry_url"]: + raise RuntimeValidationError( + "Run report browser entry URL does not match the attested target URL" + ) + elif report.execution_origin is not None or report.execution_entry_url is not None: + raise RuntimeValidationError( + "Native/remote run report must not carry browser origin or entry URL" ) compiler_version = provenance.compiler_version or __version__ @@ -483,6 +541,7 @@ def create_runtime_validation_attestation( payload: dict[str, Any] = { "schema": SCHEMA, + "target_kind": target_kind, "challenge_id": challenge["challenge_id"], "nonce": challenge["nonce"], "source_recording_sha256": recording_approval["approved_derivative_sha256"], @@ -521,8 +580,25 @@ def verify_runtime_validation_attestation( attestation: dict[str, Any], *, bundle_sha256: str, token: str ) -> None: """Verify local signature and exact-bundle binding before upload.""" - if attestation.get("schema") != SCHEMA: + schema = attestation.get("schema") + if schema not in (SCHEMA, LEGACY_SCHEMA): raise RuntimeValidationError("Unsupported runtime-validation schema") + if schema == SCHEMA: + target_kind = attestation.get("target_kind") + execution = attestation.get("execution") + if target_kind not in TARGET_KINDS: + raise RuntimeValidationError("Runtime validation target kind is invalid") + if not isinstance(execution, dict): + raise RuntimeValidationError("Runtime validation execution is invalid") + if target_kind == "web": + if set(execution) != {"entry_url", "target_origin", "allowed_hosts"}: + raise RuntimeValidationError( + "Web runtime validation execution boundary is invalid" + ) + elif execution: + raise RuntimeValidationError( + "Native/remote runtime validation execution must be PHI-safe and empty" + ) if attestation.get("bundle_sha256") != bundle_sha256: raise RuntimeValidationError( "Runtime validation is bound to a different approved bundle" diff --git a/schemas/runtime-validation-attestation-v2.json b/schemas/runtime-validation-attestation-v2.json new file mode 100644 index 00000000..083a308a --- /dev/null +++ b/schemas/runtime-validation-attestation-v2.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openadapt.ai/schemas/runtime-validation-attestation-v2.json", + "title": "OpenAdapt substrate-aware runtime validation operator attestation v2", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "target_kind", + "challenge_id", + "nonce", + "source_recording_sha256", + "bundle_sha256", + "compiler", + "parameter_schema_sha256", + "parameters", + "execution", + "lint", + "certification", + "replay", + "signature" + ], + "properties": { + "schema": { "const": "openadapt.runtime-validation/v2" }, + "target_kind": { + "enum": ["web", "windows", "macos", "linux", "rdp", "citrix"] + }, + "challenge_id": { "type": "string", "minLength": 1, "maxLength": 200 }, + "nonce": { "type": "string", "minLength": 16, "maxLength": 500 }, + "source_recording_sha256": { "$ref": "#/$defs/sha256" }, + "bundle_sha256": { "$ref": "#/$defs/sha256" }, + "compiler": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "config_sha256"], + "properties": { + "name": { "const": "openadapt-flow" }, + "version": { "type": "string", "minLength": 1 }, + "config_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "parameter_schema_sha256": { "$ref": "#/$defs/sha256" }, + "parameters": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "type", "required", "secret", "choices"], + "properties": { + "name": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,63}$" }, + "type": { "enum": ["string", "date", "enum", "number", "entity_ref"] }, + "required": { "type": "boolean" }, + "secret": { "type": "boolean" }, + "choices": { + "type": "array", + "maxItems": 100, + "items": { "type": "string", "maxLength": 500 } + } + } + } + }, + "execution": { + "type": "object", + "additionalProperties": false, + "properties": { + "entry_url": { "type": "string", "format": "uri", "pattern": "^https://" }, + "target_origin": { "type": "string", "format": "uri", "pattern": "^https://" }, + "allowed_hosts": { + "type": "array", + "minItems": 1, + "maxItems": 20, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1, "maxLength": 253 } + } + } + }, + "lint": { + "type": "object", + "additionalProperties": false, + "required": ["strict", "passed", "evidence_sha256"], + "properties": { + "strict": { "const": true }, + "passed": { "const": true }, + "evidence_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "certification": { + "type": "object", + "additionalProperties": false, + "required": ["policy", "risk_class", "passed", "evidence_sha256"], + "properties": { + "policy": { "type": "string", "minLength": 1 }, + "risk_class": { "enum": ["low", "consequential"] }, + "passed": { "const": true }, + "evidence_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "replay": { + "type": "object", + "additionalProperties": false, + "required": ["success", "report_sha256", "environment_sha256", "validated_at"], + "properties": { + "success": { "const": true }, + "report_sha256": { "$ref": "#/$defs/sha256" }, + "environment_sha256": { "$ref": "#/$defs/sha256" }, + "validated_at": { "type": "string", "format": "date-time" } + } + }, + "signature": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { + "properties": { "target_kind": { "const": "web" } }, + "required": ["target_kind"] + }, + "then": { + "properties": { + "execution": { + "required": ["entry_url", "target_origin", "allowed_hosts"], + "minProperties": 3, + "maxProperties": 3 + } + } + }, + "else": { + "properties": { + "execution": { + "maxProperties": 0 + } + } + } + } + ], + "$defs": { + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } +} diff --git a/scripts/qualify_macos_ax_release.py b/scripts/qualify_macos_ax_release.py new file mode 100644 index 00000000..940ec737 --- /dev/null +++ b/scripts/qualify_macos_ax_release.py @@ -0,0 +1,849 @@ +#!/usr/bin/env python3 +"""Release-lane macOS AX record/compile/replay qualification. + +The fixed matrix uses one synthetic TextEdit workflow and the unmodified +Recorder -> compiler -> Replayer stack: + +* 3 healthy trials must resolve the recorded AX target structurally, verify its + structured identity, make zero model calls, and produce exact target-file + bytes. +* 3 one-glyph wrong-identity trials must halt before input and preserve the + exact original file bytes. +* 3 ambiguous-window trials must refuse before input and preserve both exact + original files. + +There are no automatic retries. Every TextEdit process is launched under a +temporary qualification root and terminated by exact PID; unrelated TextEdit +processes are audited and preserved. The run refuses before opening TextEdit +unless the candidate is a clean, exact Git commit and both macOS permissions +are available. + +Usage: + python scripts/qualify_macos_ax_release.py \ + --output benchmark/macos_native/ax_release__.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import shutil +import subprocess +import sys +import tempfile +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from openadapt_flow import __version__ # noqa: E402 +from openadapt_flow.backends.macos_backend import ( # noqa: E402 + MacOSBackend, + MacOSBackendError, + QuartzMacAXClient, +) +from openadapt_flow.backends.remote_display import ( # noqa: E402 + MacWindowClient, + WindowInfo, +) +from openadapt_flow.compiler import compile_recording # noqa: E402 +from openadapt_flow.ir import StructuralLocator, Workflow # noqa: E402 +from openadapt_flow.recorder import Recorder # noqa: E402 +from openadapt_flow.runtime.replayer import Replayer # noqa: E402 +from scripts.qualify_macos_textedit import ( # noqa: E402 + _cleanup_failures, + _close_target, + _open_isolated, + _textedit_pids, + _wait_for_matches, + file_oracle, +) + +TEXTEDIT_APP = "TextEdit" +HEALTHY_TRIALS = 3 +WRONG_IDENTITY_TRIALS = 3 +AMBIGUITY_TRIALS = 3 +IDENTITY = "MG4408 Okafor, Philip DOB 1966-01-17" +WRONG_IDENTITY = "MG44O8 Okafor, Philip DOB 1966-01-17" +TEXT_VIEW_LOCATOR = StructuralLocator(automation_id="First Text View", role="textbox") + + +def _git(args: list[str]) -> str: + result = subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + return result.stdout.strip() + + +def _candidate_state() -> dict[str, Any]: + """Bind the run to a clean, exact commit and tree.""" + try: + sha = _git(["rev-parse", "HEAD"]) + tree = _git(["rev-parse", "HEAD^{tree}"]) + dirty = _git(["status", "--porcelain", "--untracked-files=all"]) + except Exception as error: # noqa: BLE001 + return { + "git_sha": None, + "git_tree": None, + "git_dirty": None, + "git_state_error": str(error), + "flow_version": __version__, + } + return { + "git_sha": sha, + "git_tree": tree, + "git_dirty": bool(dirty), + "dirty_paths": [line for line in dirty.splitlines() if line], + "flow_version": __version__, + } + + +def _permissions(client: MacWindowClient) -> dict[str, bool]: + return { + "screen_recording": client.capture_trusted(), + "accessibility": client.input_trusted(), + } + + +def _wait_for_ax_body( + client: MacWindowClient, + title: str, + expected_identity: str, + *, + timeout_s: float = 8.0, +) -> tuple[WindowInfo, MacOSBackend, tuple[int, int]]: + """Return only after one live AX text view exposes the exact file identity.""" + window = _wait_for_matches(client, title, count=1, timeout_s=20.0)[0] + backend = MacOSBackend( + client, + app=TEXTEDIT_APP, + window_title=title, + settle_s=0.01, + foreground_settle_s=0.05, + ax_client=QuartzMacAXClient(), + ) + backend.screenshot() + ax = QuartzMacAXClient() + deadline = time.monotonic() + timeout_s + last_observed: str | None = None + while time.monotonic() < deadline: + candidates = ax.find_candidates( + window.pid, + window.title, + TEXT_VIEW_LOCATOR, + limit=4000, + ) + if len(candidates.candidates) == 1 and not candidates.truncated: + element = candidates.candidates[0] + ex, ey, ew, eh = element.bounds + wx, wy, _ww, _wh = window.bounds + point = ( + int((ex + ew / 2 - wx) * backend._scale_x), + int((ey + eh / 2 - wy) * backend._scale_y), + ) + last_observed = backend.structured_text_at(*point) + if last_observed == expected_identity: + return window, backend, point + time.sleep(0.15) + raise RuntimeError( + "live AX text view did not settle to the exact expected identity " + f"(observed={last_observed!r})" + ) + + +def _identity_summary(report: Any) -> dict[str, Any]: + """Extract the first identity-armed result without retaining PHI.""" + for result in report.results: + if result.identity is not None: + identity = result.identity + return { + "status": str(identity.status), + "mode": identity.mode, + "safety_halt": result.safety_halt, + } + return {"status": None, "mode": None, "safety_halt": False} + + +def _resolution_rungs(report: Any) -> list[str]: + return [ + result.resolution.rung + for result in report.results + if result.resolution is not None + ] + + +def _record_and_compile( + client: MacWindowClient, + root: Path, + title: str, + cleanup_warnings: list[str], + cleanup_receipts: list[dict[str, Any]], +) -> tuple[Workflow, Path, dict[str, Any]]: + demo_dir = root / "demonstration" + demo_dir.mkdir() + path = demo_dir / f"{title}.txt" + path.write_text(IDENTITY + "\n", encoding="utf-8") + pid: int | None = None + try: + _open_isolated(path) + window, backend, point = _wait_for_ax_body(client, title, IDENTITY) + pid = window.pid + recording_dir = root / "recording" + bundle_dir = root / "bundle" + recorder = Recorder( + backend, + recording_dir, + settle_interval_s=0.05, + settle_timeout_s=3.0, + ) + recorder.click(*point) + recorder.press("ControlOrMeta+a") + recorder.type_text("OpenAdapt AX demonstration\n", param="replacement") + recorder.press("ControlOrMeta+s") + recorder.finish() + workflow = compile_recording( + recording_dir, + bundle_dir, + name="macos-ax-release", + ) + first = workflow.steps[0] + if first.anchor is None or first.anchor.structural is None: + raise RuntimeError("compiled click lost the recorded AX locator") + if not ( + first.anchor.structured_identity is not None + or ( + first.anchor.identity_template is not None + and first.anchor.identity_template.structured + ) + ): + raise RuntimeError("compiled click lost the recorded AX identity") + return ( + workflow, + bundle_dir, + { + "step_count": len(workflow.steps), + "click_structural_locator_recorded": True, + "click_structured_identity_recorded": True, + "recording_parameter": "replacement", + "model_calls": 0, + }, + ) + finally: + if pid is not None: + _close_target( + client, + title, + pid, + warnings=cleanup_warnings, + receipts=cleanup_receipts, + ) + + +def _run_healthy_trial( + client: MacWindowClient, + root: Path, + title: str, + workflow: Workflow, + bundle_dir: Path, + trial: int, + cleanup_warnings: list[str], + cleanup_receipts: list[dict[str, Any]], +) -> dict[str, Any]: + case_dir = root / f"healthy-{trial}" + case_dir.mkdir() + path = case_dir / f"{title}.txt" + baseline = (IDENTITY + "\n").encode() + expected = f"OpenAdapt AX healthy trial {trial}\n".encode() + path.write_bytes(baseline) + pid: int | None = None + started = time.monotonic() + try: + _open_isolated(path) + window, backend, _point = _wait_for_ax_body(client, title, IDENTITY) + pid = window.pid + report = Replayer(backend, poll_interval_s=0.05).run( + workflow, + params={"replacement": expected.decode()}, + bundle_dir=bundle_dir, + run_dir=root / "runs" / f"healthy-{trial}", + ) + oracle = file_oracle(path, expected) + identity = _identity_summary(report) + rungs = _resolution_rungs(report) + passed = ( + report.success + and oracle["status"] == "confirmed" + and "structural" in rungs + and identity["status"] == "verified" + and identity["mode"] == "structured" + and report.model_calls == 0 + ) + return { + "trial": trial, + "status": "passed" if passed else "failed", + "report_success": report.success, + "all_steps_ok": all(result.ok for result in report.results), + "oracle": oracle, + "resolution_rungs": rungs, + "identity": identity, + "model_calls": report.model_calls, + "duration_s": round(time.monotonic() - started, 3), + } + except Exception as error: # noqa: BLE001 + return { + "trial": trial, + "status": "failed", + "report_success": False, + "oracle": file_oracle(path, expected), + "failure_type": type(error).__name__, + "error": str(error), + "model_calls": 0, + "duration_s": round(time.monotonic() - started, 3), + } + finally: + if pid is not None: + _close_target( + client, + title, + pid, + warnings=cleanup_warnings, + receipts=cleanup_receipts, + ) + + +def _run_wrong_identity_trial( + client: MacWindowClient, + root: Path, + title: str, + workflow: Workflow, + bundle_dir: Path, + trial: int, + cleanup_warnings: list[str], + cleanup_receipts: list[dict[str, Any]], +) -> dict[str, Any]: + case_dir = root / f"wrong-identity-{trial}" + case_dir.mkdir() + path = case_dir / f"{title}.txt" + baseline = (WRONG_IDENTITY + "\n").encode() + attempted = f"MUST NOT WRITE {trial}\n" + path.write_bytes(baseline) + pid: int | None = None + started = time.monotonic() + try: + _open_isolated(path) + window, backend, _point = _wait_for_ax_body( + client, + title, + WRONG_IDENTITY, + ) + pid = window.pid + report = Replayer(backend, poll_interval_s=0.05).run( + workflow, + params={"replacement": attempted}, + bundle_dir=bundle_dir, + run_dir=root / "runs" / f"wrong-identity-{trial}", + ) + oracle = file_oracle(path, baseline) + identity = _identity_summary(report) + passed = ( + not report.success + and oracle["status"] == "confirmed" + and identity["status"] == "mismatch" + and identity["mode"] == "structured" + and identity["safety_halt"] + and report.model_calls == 0 + ) + return { + "trial": trial, + "status": "passed" if passed else "failed", + "report_success": report.success, + "pre_write_halt": identity["safety_halt"], + "oracle": oracle, + "identity": identity, + "model_calls": report.model_calls, + "duration_s": round(time.monotonic() - started, 3), + } + except Exception as error: # noqa: BLE001 + return { + "trial": trial, + "status": "failed", + "report_success": False, + "pre_write_halt": False, + "oracle": file_oracle(path, baseline), + "failure_type": type(error).__name__, + "error": str(error), + "model_calls": 0, + "duration_s": round(time.monotonic() - started, 3), + } + finally: + if pid is not None: + _close_target( + client, + title, + pid, + warnings=cleanup_warnings, + receipts=cleanup_receipts, + ) + + +def _run_ambiguity_trial( + client: MacWindowClient, + root: Path, + title: str, + workflow: Workflow, + bundle_dir: Path, + trial: int, + cleanup_warnings: list[str], + cleanup_receipts: list[dict[str, Any]], +) -> dict[str, Any]: + baseline = (IDENTITY + "\n").encode() + paths: list[Path] = [] + pids: dict[int, str] = {} + started = time.monotonic() + try: + for suffix in ("a", "b"): + case_dir = root / f"ambiguity-{trial}-{suffix}" + case_dir.mkdir() + path = case_dir / f"{title}.txt" + path.write_bytes(baseline) + paths.append(path) + _open_isolated(path) + _wait_for_matches(client, title, count=len(paths)) + + matches = _wait_for_matches(client, title, count=2) + pids.update({window.pid: window.title for window in matches}) + backend = MacOSBackend( + client, + app=TEXTEDIT_APP, + window_title=title, + settle_s=0.01, + foreground_settle_s=0.05, + ax_client=QuartzMacAXClient(), + ) + refused = False + refusal_type: str | None = None + report_success = False + model_calls = 0 + try: + report = Replayer(backend, poll_interval_s=0.05).run( + workflow, + params={"replacement": f"MUST NOT WRITE ambiguity {trial}\n"}, + bundle_dir=bundle_dir, + run_dir=root / "runs" / f"ambiguity-{trial}", + ) + report_success = report.success + model_calls = report.model_calls + errors = [result.error or "" for result in report.results] + refused = not report.success and any( + "ambiguous native macOS target" in error for error in errors + ) + refusal_type = "run_report_halt" if refused else None + except MacOSBackendError as error: + refused = "ambiguous native macOS target" in str(error) + refusal_type = type(error).__name__ + oracles = [file_oracle(path, baseline) for path in paths] + passed = ( + refused + and not report_success + and len(matches) == 2 + and all(oracle["status"] == "confirmed" for oracle in oracles) + and model_calls == 0 + ) + return { + "trial": trial, + "status": "passed" if passed else "failed", + "matched_windows": len(matches), + "refused": refused, + "refusal_type": refusal_type, + "report_success": report_success, + "oracles": oracles, + "model_calls": model_calls, + "duration_s": round(time.monotonic() - started, 3), + } + except Exception as error: # noqa: BLE001 + return { + "trial": trial, + "status": "failed", + "refused": False, + "report_success": False, + "oracles": [file_oracle(path, baseline) for path in paths], + "failure_type": type(error).__name__, + "error": str(error), + "model_calls": 0, + "duration_s": round(time.monotonic() - started, 3), + } + finally: + for pid, window_title in pids.items(): + _close_target( + client, + window_title, + pid, + warnings=cleanup_warnings, + receipts=cleanup_receipts, + ) + + +def evaluate_report(report: dict[str, Any]) -> dict[str, Any]: + """Apply the release acceptance contract to a bounded aggregate report.""" + healthy = report.get("healthy_trials", []) + wrong = report.get("wrong_identity_trials", []) + ambiguous = report.get("ambiguity_trials", []) + candidate = report.get("environment", {}).get("candidate", {}) + + silent_incorrect = sum( + trial.get("report_success") is True + and trial.get("oracle", {}).get("status") != "confirmed" + for trial in healthy + ) + over_halts = sum(trial.get("report_success") is not True for trial in healthy) + false_completions = sum( + trial.get("report_success") is True for trial in [*wrong, *ambiguous] + ) + refusal_writes = sum( + trial.get("oracle", {}).get("status") != "confirmed" for trial in wrong + ) + sum( + any(item.get("status") != "confirmed" for item in trial.get("oracles", [])) + for trial in ambiguous + ) + model_calls = sum( + int(trial.get("model_calls", 0)) for trial in [*healthy, *wrong, *ambiguous] + ) + healthy_semantics = all( + trial.get("status") == "passed" + and trial.get("report_success") is True + and trial.get("all_steps_ok") is True + and trial.get("oracle", {}).get("status") == "confirmed" + and "structural" in trial.get("resolution_rungs", []) + and trial.get("identity", {}).get("status") == "verified" + and trial.get("identity", {}).get("mode") == "structured" + and trial.get("model_calls") == 0 + for trial in healthy + ) + wrong_identity_semantics = all( + trial.get("status") == "passed" + and trial.get("report_success") is False + and trial.get("pre_write_halt") is True + and trial.get("oracle", {}).get("status") == "confirmed" + and trial.get("identity", {}).get("status") == "mismatch" + and trial.get("identity", {}).get("mode") == "structured" + and trial.get("model_calls") == 0 + for trial in wrong + ) + ambiguity_semantics = all( + trial.get("status") == "passed" + and trial.get("report_success") is False + and trial.get("refused") is True + and len(trial.get("oracles", [])) == 2 + and all(item.get("status") == "confirmed" for item in trial.get("oracles", [])) + and trial.get("model_calls") == 0 + for trial in ambiguous + ) + exact_matrix = ( + len(healthy) == HEALTHY_TRIALS + and len(wrong) == WRONG_IDENTITY_TRIALS + and len(ambiguous) == AMBIGUITY_TRIALS + ) + accepted = ( + report.get("lane") == "release" + and report.get("automatic_retry") is False + and candidate.get("git_dirty") is False + and isinstance(candidate.get("git_sha"), str) + and len(candidate["git_sha"]) == 40 + and isinstance(candidate.get("git_tree"), str) + and len(candidate["git_tree"]) == 40 + and exact_matrix + and healthy_semantics + and wrong_identity_semantics + and ambiguity_semantics + and silent_incorrect == 0 + and over_halts == 0 + and false_completions == 0 + and refusal_writes == 0 + and model_calls == 0 + and not report.get("cleanup_errors") + ) + return { + "decision": "accepted" if accepted else "rejected", + "accepted": accepted, + "candidate": candidate, + "matrix": { + "healthy_exact_effect_trials": len(healthy), + "one_glyph_wrong_identity_pre_write_halts": len(wrong), + "ambiguous_window_pre_write_halts": len(ambiguous), + "exact_fixed_matrix": exact_matrix, + "healthy_semantics": healthy_semantics, + "wrong_identity_semantics": wrong_identity_semantics, + "ambiguity_semantics": ambiguity_semantics, + }, + "metrics": { + "silent_incorrect_successes": silent_incorrect, + "over_halts": over_halts, + "false_completions": false_completions, + "writes_after_refusal": refusal_writes, + "model_calls": model_calls, + }, + "oracle": report.get("oracle"), + "failure_taxonomy": report.get("failure_taxonomy"), + "cleanup_errors": report.get("cleanup_errors", []), + } + + +def qualify() -> tuple[int, dict[str, Any]]: + client = MacWindowClient() + candidate = _candidate_state() + permissions = _permissions(client) + base: dict[str, Any] = { + "schema_version": 1, + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "task": "macOS AX record-compile-replay with exact effects and refusal", + "lane": "release", + "evidence_classification": "acceptance_candidate", + "environment": { + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + "permissions": permissions, + "candidate": candidate, + }, + "run_count": { + "healthy_exact_effect_trials": HEALTHY_TRIALS, + "one_glyph_wrong_identity_trials": WRONG_IDENTITY_TRIALS, + "ambiguous_window_trials": AMBIGUITY_TRIALS, + }, + "automatic_retry": False, + "oracle": ( + "exact target-file bytes read independently after replay or refusal" + ), + "failure_taxonomy": [ + "permission_refusal", + "dirty_or_unbound_candidate", + "record_or_compile_failure", + "target_absent", + "target_ambiguous", + "structured_resolution_failure", + "identity_false_accept", + "identity_false_reject", + "input_delivery_failure", + "file_effect_refuted", + "file_effect_unverifiable", + "false_completion", + "cleanup_failure", + ], + "caveats": [ + "TextEdit workflow on one named macOS host and active user session", + "synthetic local identity/file values; no customer data", + "resolved AX element is acted through gated point-bound physical input", + ], + } + missing = [name for name, granted in permissions.items() if not granted] + if missing: + base.update( + { + "status": "blocked", + "missing_permissions": missing, + "healthy_trials": [], + "wrong_identity_trials": [], + "ambiguity_trials": [], + } + ) + return 2, base + if ( + candidate.get("git_dirty") is not False + or not isinstance(candidate.get("git_sha"), str) + or len(candidate["git_sha"]) != 40 + or not isinstance(candidate.get("git_tree"), str) + or len(candidate["git_tree"]) != 40 + ): + base.update( + { + "status": "blocked", + "candidate_gate": "release qualification requires a clean exact SHA", + "healthy_trials": [], + "wrong_identity_trials": [], + "ambiguity_trials": [], + } + ) + return 2, base + + run_id = uuid.uuid4().hex[:10] + title = f"oa-ax-release-{run_id}" + root = Path(tempfile.mkdtemp(prefix=f"openadapt-ax-release-{run_id}-")) + original_frontmost = client.frontmost_pid() + textedit_pids_before = _textedit_pids() + cleanup_warnings: list[str] = [] + cleanup_receipts: list[dict[str, Any]] = [] + healthy: list[dict[str, Any]] = [] + wrong: list[dict[str, Any]] = [] + ambiguous: list[dict[str, Any]] = [] + workflow_summary: dict[str, Any] = {} + harness_error: dict[str, str] | None = None + try: + workflow, bundle_dir, workflow_summary = _record_and_compile( + client, + root, + title, + cleanup_warnings, + cleanup_receipts, + ) + healthy = [ + _run_healthy_trial( + client, + root, + title, + workflow, + bundle_dir, + trial, + cleanup_warnings, + cleanup_receipts, + ) + for trial in range(1, HEALTHY_TRIALS + 1) + ] + wrong = [ + _run_wrong_identity_trial( + client, + root, + title, + workflow, + bundle_dir, + trial, + cleanup_warnings, + cleanup_receipts, + ) + for trial in range(1, WRONG_IDENTITY_TRIALS + 1) + ] + ambiguous = [ + _run_ambiguity_trial( + client, + root, + title, + workflow, + bundle_dir, + trial, + cleanup_warnings, + cleanup_receipts, + ) + for trial in range(1, AMBIGUITY_TRIALS + 1) + ] + except Exception as error: # noqa: BLE001 + harness_error = {"type": type(error).__name__, "error": str(error)} + finally: + if original_frontmost is not None: + client.activate(original_frontmost) + try: + shutil.rmtree(root) + except Exception as error: # noqa: BLE001 + cleanup_warnings.append(f"temporary directory cleanup {root}: {error}") + + textedit_pids_after = _textedit_pids() + cleanup_errors = _cleanup_failures( + cleanup_receipts, + root_exists=root.exists(), + textedit_pids_before=textedit_pids_before, + textedit_pids_after=textedit_pids_after, + ) + if harness_error is not None: + cleanup_errors.append( + f"harness failed: {harness_error['type']}: {harness_error['error']}" + ) + base.update( + { + "workflow": workflow_summary, + "healthy_trials": healthy, + "wrong_identity_trials": wrong, + "ambiguity_trials": ambiguous, + "cleanup_errors": cleanup_errors, + "cleanup_warnings": cleanup_warnings, + "cleanup_receipts": cleanup_receipts, + "cleanup_audit": { + "temporary_root_removed": not root.exists(), + "textedit_pids_before": sorted(textedit_pids_before), + "textedit_pids_after": sorted(textedit_pids_after), + "unrelated_textedit_pids_preserved": ( + textedit_pids_before == textedit_pids_after + ), + }, + } + ) + evaluation = evaluate_report(base) + base["metrics"] = evaluation["metrics"] + base["status"] = "passed" if evaluation["accepted"] else "failed" + return (0 if evaluation["accepted"] else 1), base + + +def write_evidence(report: dict[str, Any], output: Path) -> dict[str, Any]: + """Write the aggregate and a byte/hash-bound adjudication beside it.""" + output.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(report, indent=2, sort_keys=True) + "\n").encode() + output.write_bytes(payload) + evaluation = evaluate_report(report) + adjudication = { + **evaluation, + "task": report["task"], + "lane": report["lane"], + "evidence_classification": ( + "release_lane_scoped_acceptance" + if evaluation["accepted"] + else "release_lane_rejected" + ), + "original_evidence": { + "path": str(output.relative_to(REPO_ROOT)), + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + "status": report["status"], + "preserved_byte_for_byte": True, + }, + "scope": ( + "TextEdit on the recorded macOS host/session: real AX locator and " + "structured identity through Recorder -> compiler -> Replayer, " + "with exact file effects and pre-write identity/ambiguity refusal." + ), + } + adjudication_path = output.with_suffix(".adjudication.json") + adjudication_path.write_text( + json.dumps(adjudication, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return { + "evidence": str(output), + "adjudication": str(adjudication_path), + "sha256": adjudication["original_evidence"]["sha256"], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + parser.add_argument("--request-permissions", action="store_true") + args = parser.parse_args() + client = MacWindowClient() + if args.request_permissions: + client.request_capture_access() + client.request_input_access() + print( + "macOS permission prompts requested. Approve Screen Recording and " + "Accessibility for the launching terminal, restart it, then rerun." + ) + return 2 + + code, report = qualify() + print(json.dumps(report, indent=2, sort_keys=True)) + if args.output is not None and report["status"] != "blocked": + written = write_evidence(report, args.output.resolve()) + print(json.dumps(written, indent=2, sort_keys=True)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_claims.py b/scripts/validate_claims.py index 3a8a8e9c..5cbf2acb 100644 --- a/scripts/validate_claims.py +++ b/scripts/validate_claims.py @@ -36,6 +36,7 @@ from __future__ import annotations import argparse +import ast import json import os import re @@ -84,6 +85,92 @@ def detect_optin_env(source: str) -> Optional[str]: """ if not _PYTESTMARK.search(source): return None + + # Prefer syntax-aware detection so both of the supported module-level + # forms are classified correctly: + # + # RUN = os.environ.get("FLAG") == "1" + # pytestmark = pytest.mark.skipif(not RUN, ...) + # + # and: + # + # pytestmark = pytest.mark.skipif( + # os.environ.get("FLAG") != "1", ... + # ) + # + # Restrict the search to the value assigned to a module-level + # ``pytestmark``. An environment-gated test function elsewhere in the file + # must not make the whole module look opt-in. + try: + tree = ast.parse(source) + except SyntaxError: + tree = None + if tree is not None: + env_flags: dict[str, str] = {} + pytestmark_values: list[ast.AST] = [] + + def env_from_call(node: ast.AST) -> Optional[str]: + if not isinstance(node, ast.Call) or not node.args: + return None + func = node.func + is_environ_get = ( + isinstance(func, ast.Attribute) + and func.attr == "get" + and isinstance(func.value, ast.Attribute) + and func.value.attr == "environ" + and isinstance(func.value.value, ast.Name) + and func.value.value.id == "os" + ) + is_getenv = ( + isinstance(func, ast.Attribute) + and func.attr == "getenv" + and isinstance(func.value, ast.Name) + and func.value.id == "os" + ) + first = node.args[0] + if (is_environ_get or is_getenv) and isinstance(first, ast.Constant): + return first.value if isinstance(first.value, str) else None + return None + + for statement in tree.body: + value: Optional[ast.AST] = None + targets: list[ast.expr] = [] + if isinstance(statement, ast.Assign): + value = statement.value + targets = list(statement.targets) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets = [statement.target] + if value is None: + continue + names = [target.id for target in targets if isinstance(target, ast.Name)] + if "pytestmark" in names: + pytestmark_values.append(value) + for name in names: + for node in ast.walk(value): + if (env := env_from_call(node)) is not None: + env_flags[name] = env + break + + for value in pytestmark_values: + for call in ( + node + for node in ast.walk(value) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "skipif" + ): + if not call.args: + continue + condition = call.args[0] + for node in ast.walk(condition): + if (env := env_from_call(node)) is not None: + return env + if isinstance(node, ast.Name) and node.id in env_flags: + return env_flags[node.id] + + # Keep the deliberately narrow legacy pattern as a fail-safe for source + # syntax that the running Python cannot parse. flags = {m.group("flag"): m.group("env") for m in _ENV_FLAG.finditer(source)} for flag, env in flags.items(): # `pytestmark = [pytest.mark.skipif(not FLAG, ...)]` diff --git a/tests/fixtures/runtime-validation-canonical-vector.json b/tests/fixtures/runtime-validation-canonical-vector.json index c8be9134..a1f093c9 100644 --- a/tests/fixtures/runtime-validation-canonical-vector.json +++ b/tests/fixtures/runtime-validation-canonical-vector.json @@ -23,8 +23,9 @@ "type": "entity_ref" } ], - "schema": "openadapt.runtime-validation/v1" + "schema": "openadapt.runtime-validation/v2", + "target_kind": "web" }, - "canonical": "{\"certification\":{\"evidence_sha256\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"passed\":true,\"policy\":\"clinical-write\",\"risk_class\":\"consequential\"},\"challenge_id\":\"challenge-α\",\"execution\":{\"allowed_hosts\":[\"app.example\",\"cdn.example\"],\"entry_url\":\"https://app.example/login\",\"target_origin\":\"https://app.example\"},\"parameter_schema_sha256\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"parameters\":[{\"choices\":[],\"name\":\"patient_id\",\"required\":true,\"secret\":false,\"type\":\"entity_ref\"}],\"schema\":\"openadapt.runtime-validation/v1\"}", - "signature": "bdb491013fa4eb5fc8e7ed2b92ae0ebadd853da1a44b136d96dad0c31f52c403" + "canonical": "{\"certification\":{\"evidence_sha256\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"passed\":true,\"policy\":\"clinical-write\",\"risk_class\":\"consequential\"},\"challenge_id\":\"challenge-α\",\"execution\":{\"allowed_hosts\":[\"app.example\",\"cdn.example\"],\"entry_url\":\"https://app.example/login\",\"target_origin\":\"https://app.example\"},\"parameter_schema_sha256\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"parameters\":[{\"choices\":[],\"name\":\"patient_id\",\"required\":true,\"secret\":false,\"type\":\"entity_ref\"}],\"schema\":\"openadapt.runtime-validation/v2\",\"target_kind\":\"web\"}", + "signature": "a6001dbcb4f4521e4899672ad80abcc8f29d9c2a434d3c7201dc78663e1ca8a1" } diff --git a/tests/test_backend_factory.py b/tests/test_backend_factory.py index 2e8c809b..b85740e6 100644 --- a/tests/test_backend_factory.py +++ b/tests/test_backend_factory.py @@ -481,6 +481,7 @@ def test_replay_windows_constructs_windows_backend(monkeypatch, tmp_path) -> Non def fake_run(backend, **kwargs): captured["backend"] = backend + captured.update(kwargs) return _FakeReport() monkeypatch.setattr(m, "_build_and_run_replayer", fake_run) @@ -506,6 +507,7 @@ def fake_run(backend, **kwargs): assert rc == 0 assert type(captured["backend"]).__name__ == "WindowsBackend" assert captured["backend"].server_url == "http://localhost:5001" + assert captured["execution_target_kind"] == "windows" def test_replay_desktop_refuses_drift(monkeypatch, tmp_path) -> None: diff --git a/tests/test_cli_new_commands.py b/tests/test_cli_new_commands.py index c4013560..0f585949 100644 --- a/tests/test_cli_new_commands.py +++ b/tests/test_cli_new_commands.py @@ -499,8 +499,9 @@ def test_resume_web_default_builds_playwright_backend( seen: dict = {} - def _fake_resume(run_dir, replayer, key=None): + def _fake_resume(run_dir, replayer, key=None, execution_target_kind=None): seen["called"] = True + seen["execution_target_kind"] = execution_target_kind return _FakeReport() monkeypatch.setattr(durable_mod, "resume", _fake_resume) @@ -516,6 +517,7 @@ def _fake_resume(run_dir, replayer, key=None): ) assert rc == 0 assert seen["called"] is True + assert seen["execution_target_kind"] == "web" # The factory built the browser-backed backend (the fake returns "backend"). assert captured["ctor"] # Replayer was constructed for the web path @@ -540,7 +542,12 @@ def __init__(self, backend, **kwargs): monkeypatch.setattr(runtime_mod, "Replayer", _CapturingReplayer) monkeypatch.setattr( - durable_mod, "resume", lambda run_dir, replayer, key=None: _FakeReport() + durable_mod, + "resume", + lambda run_dir, replayer, key=None, execution_target_kind=None: ( + captured.update(execution_target_kind=execution_target_kind) + or _FakeReport() + ), ) monkeypatch.setattr(report_mod, "render_run_report", lambda run_dir: "REPORT.md") @@ -559,6 +566,7 @@ def __init__(self, backend, **kwargs): backend = captured["backend"] assert type(backend).__name__ == "WindowsBackend" assert backend.server_url == "http://localhost:5001" + assert captured["execution_target_kind"] == "windows" def test_resume_uses_recorded_citrix_target_and_readiness( @@ -597,7 +605,12 @@ def fake_build(cfg, **kwargs): monkeypatch.setattr(factory, "build_backend", fake_build) monkeypatch.setattr(runtime_mod, "Replayer", CapturingReplayer) monkeypatch.setattr( - durable_mod, "resume", lambda run_dir, replayer, key=None: _FakeReport() + durable_mod, + "resume", + lambda run_dir, replayer, key=None, execution_target_kind=None: ( + captured.update(execution_target_kind=execution_target_kind) + or _FakeReport() + ), ) monkeypatch.setattr(report_mod, "render_run_report", lambda run_dir: "REPORT.md") @@ -607,6 +620,7 @@ def fake_build(cfg, **kwargs): assert cfg.rdp_window == "wfica32" assert cfg.rdp_window_title == "Claims - Citrix Workspace" assert cfg.rdp_readiness_text == "Claims queue" + assert captured["execution_target_kind"] == "citrix" def test_resume_refuses_blank_citrix_readiness_before_backend( diff --git a/tests/test_hosted.py b/tests/test_hosted.py index d5d33446..313e65ab 100644 --- a/tests/test_hosted.py +++ b/tests/test_hosted.py @@ -2831,6 +2831,8 @@ def fake_save(attestation, path): "low", "--environment", "clean-room-v1", + "--target-kind", + "web", "--target-url", "https://app.example/login", "--allowed-host", @@ -2860,6 +2862,7 @@ def fake_save(attestation, path): "policy_source": "permissive", "risk_class": "low", "environment": "clean-room-v1", + "target_kind": "web", "target_url": "https://app.example/login", "allowed_hosts": ["cdn.example", "api.example"], "compiler_config": {"mode": "strict"}, diff --git a/tests/test_macos_ax_release_qualification.py b/tests/test_macos_ax_release_qualification.py new file mode 100644 index 00000000..81f9cfe1 --- /dev/null +++ b/tests/test_macos_ax_release_qualification.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +SCRIPT = Path(__file__).parents[1] / "scripts" / "qualify_macos_ax_release.py" +SPEC = importlib.util.spec_from_file_location("macos_ax_release", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +qualification = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(qualification) + + +def _candidate() -> dict: + return { + "git_sha": "a" * 40, + "git_tree": "b" * 40, + "git_dirty": False, + "dirty_paths": [], + "flow_version": "1.20.0", + } + + +def _passing_report() -> dict: + return { + "lane": "release", + "automatic_retry": False, + "environment": {"candidate": _candidate()}, + "oracle": "exact target-file bytes", + "failure_taxonomy": ["identity_false_accept", "cleanup_failure"], + "healthy_trials": [ + { + "trial": trial, + "status": "passed", + "report_success": True, + "all_steps_ok": True, + "oracle": {"status": "confirmed"}, + "resolution_rungs": ["structural"], + "identity": {"status": "verified", "mode": "structured"}, + "model_calls": 0, + } + for trial in range(1, 4) + ], + "wrong_identity_trials": [ + { + "trial": trial, + "status": "passed", + "report_success": False, + "pre_write_halt": True, + "oracle": {"status": "confirmed"}, + "identity": {"status": "mismatch", "mode": "structured"}, + "model_calls": 0, + } + for trial in range(1, 4) + ], + "ambiguity_trials": [ + { + "trial": trial, + "status": "passed", + "report_success": False, + "refused": True, + "oracles": [{"status": "confirmed"}, {"status": "confirmed"}], + "model_calls": 0, + } + for trial in range(1, 4) + ], + "cleanup_errors": [], + } + + +def test_release_matrix_is_fixed_at_three_trials_per_condition() -> None: + assert qualification.HEALTHY_TRIALS == 3 + assert qualification.WRONG_IDENTITY_TRIALS == 3 + assert qualification.AMBIGUITY_TRIALS == 3 + + +def test_evaluator_accepts_exact_clean_zero_error_matrix() -> None: + result = qualification.evaluate_report(_passing_report()) + assert result["accepted"] is True + assert result["decision"] == "accepted" + assert result["matrix"] == { + "healthy_exact_effect_trials": 3, + "one_glyph_wrong_identity_pre_write_halts": 3, + "ambiguous_window_pre_write_halts": 3, + "exact_fixed_matrix": True, + "healthy_semantics": True, + "wrong_identity_semantics": True, + "ambiguity_semantics": True, + } + assert result["metrics"] == { + "silent_incorrect_successes": 0, + "over_halts": 0, + "false_completions": 0, + "writes_after_refusal": 0, + "model_calls": 0, + } + + +def test_evaluator_rejects_dirty_or_unbound_candidate() -> None: + report = _passing_report() + report["environment"]["candidate"]["git_dirty"] = True + assert qualification.evaluate_report(report)["accepted"] is False + + report = _passing_report() + report["environment"]["candidate"]["git_sha"] = "short" + assert qualification.evaluate_report(report)["accepted"] is False + + +def test_evaluator_rejects_fewer_than_three_trials() -> None: + report = _passing_report() + report["wrong_identity_trials"].pop() + result = qualification.evaluate_report(report) + assert result["accepted"] is False + assert result["matrix"]["exact_fixed_matrix"] is False + + +def test_evaluator_counts_silent_incorrect_success_and_over_halt() -> None: + report = _passing_report() + report["healthy_trials"][0]["oracle"]["status"] = "refuted" + report["healthy_trials"][0]["status"] = "failed" + report["healthy_trials"][1]["report_success"] = False + report["healthy_trials"][1]["status"] = "failed" + result = qualification.evaluate_report(report) + assert result["accepted"] is False + assert result["metrics"]["silent_incorrect_successes"] == 1 + assert result["metrics"]["over_halts"] == 1 + + +def test_evaluator_rejects_false_completion_or_write_after_refusal() -> None: + report = _passing_report() + report["wrong_identity_trials"][0]["report_success"] = True + report["wrong_identity_trials"][0]["oracle"]["status"] = "refuted" + result = qualification.evaluate_report(report) + assert result["accepted"] is False + assert result["metrics"]["false_completions"] == 1 + assert result["metrics"]["writes_after_refusal"] == 1 + + +def test_evaluator_rejects_model_calls_cleanup_errors_and_retries() -> None: + report = _passing_report() + report["healthy_trials"][0]["model_calls"] = 1 + report["cleanup_errors"] = ["PID remained"] + report["automatic_retry"] = True + result = qualification.evaluate_report(report) + assert result["accepted"] is False + assert result["metrics"]["model_calls"] == 1 + assert result["cleanup_errors"] == ["PID remained"] diff --git a/tests/test_run_gate.py b/tests/test_run_gate.py index e25ee72c..3b0a6f9f 100644 --- a/tests/test_run_gate.py +++ b/tests/test_run_gate.py @@ -841,9 +841,17 @@ class FakeReport: halt = None calls: list[dict] = [] + replay_kwargs: dict = {} monkeypatch.setattr(factory, "build_backend", lambda cfg: FakeBackend()) + + def fake_replay(backend, **kwargs): + replay_kwargs.update(kwargs) + return FakeReport() + monkeypatch.setattr( - main, "_build_and_run_replayer", lambda backend, **kwargs: FakeReport() + main, + "_build_and_run_replayer", + fake_replay, ) monkeypatch.setattr(report_mod, "render_run_report", lambda run_dir: "REPORT.md") monkeypatch.setattr( @@ -867,6 +875,7 @@ class FakeReport: ] ) assert rc == 0 + assert replay_kwargs["execution_target_kind"] == "citrix" assert calls == [{"workflow_id": "wf_1", "backend": "citrix"}] diff --git a/tests/test_runtime_validation.py b/tests/test_runtime_validation.py index 24656c28..29ae7b25 100644 --- a/tests/test_runtime_validation.py +++ b/tests/test_runtime_validation.py @@ -14,11 +14,13 @@ from openadapt_flow.ir import ParamKind, ParamSpec, RunReport, Workflow from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.runtime_validation import ( + LEGACY_SCHEMA, RuntimeValidationError, _canonical_bytes, _signature, create_runtime_validation_attestation, normalize_execution_scope, + normalize_target_execution, request_validation_challenge, verify_runtime_validation_attestation, ) @@ -43,7 +45,9 @@ def _privacy(): privacy.reset_scrubbers() -def _approved_artifacts(tmp_path: Path) -> tuple[Path, Path, Path]: +def _approved_artifacts( + tmp_path: Path, *, target_kind: str = "web" +) -> tuple[Path, Path, Path]: recording = tmp_path / "recording" recording.mkdir() (recording / "meta.json").write_text('{"patient":"Jane Doe"}') @@ -67,8 +71,9 @@ def _approved_artifacts(tmp_path: Path) -> tuple[Path, Path, Path]: workflow, bundle_dir=bundle, run_dir=run_dir, - execution_origin=_TARGET_ORIGIN, - execution_entry_url=_TARGET_URL, + execution_target_kind=target_kind, + execution_origin=_TARGET_ORIGIN if target_kind == "web" else None, + execution_entry_url=_TARGET_URL if target_kind == "web" else None, ) assert report.success is True return recording_derivative, bundle_derivative, run_dir @@ -109,7 +114,7 @@ def test_attestation_binds_real_evidence_and_matches_schema(tmp_path): schema = json.loads( ( Path(__file__).resolve().parents[1] - / "schemas/runtime-validation-attestation-v1.json" + / "schemas/runtime-validation-attestation-v2.json" ).read_text() ) jsonschema.Draft202012Validator(schema).validate(value) @@ -117,6 +122,8 @@ def test_attestation_binds_real_evidence_and_matches_schema(tmp_path): value, bundle_sha256=value["bundle_sha256"], token=token ) assert value["source_recording_sha256"] != value["bundle_sha256"] + assert value["schema"] == "openadapt.runtime-validation/v2" + assert value["target_kind"] == "web" assert value["lint"] == { "strict": True, "passed": True, @@ -143,6 +150,125 @@ def test_cross_language_canonicalization_golden_vector(): assert _signature(fixture["value"], fixture["token"]) == fixture["signature"] +@pytest.mark.parametrize("target_kind", ["windows", "macos", "linux", "rdp", "citrix"]) +def test_native_and_remote_attestation_is_substrate_bound_without_target_phi( + tmp_path, target_kind +): + token = "oai_ingest_test" + recording, bundle, run_dir = _approved_artifacts(tmp_path, target_kind=target_kind) + value = create_runtime_validation_attestation( + recording_derivative=recording, + bundle_derivative=bundle, + run_dir=run_dir, + policy_source="permissive", + risk_class="low", + environment=f"qualified-{target_kind}-environment", + target_kind=target_kind, + host=hosted.DEFAULT_HOST, + token=token, + challenge=_challenge(), + ) + schema = json.loads( + ( + Path(__file__).resolve().parents[1] + / "schemas/runtime-validation-attestation-v2.json" + ).read_text() + ) + jsonschema.Draft202012Validator(schema).validate(value) + verify_runtime_validation_attestation( + value, bundle_sha256=value["bundle_sha256"], token=token + ) + assert value["target_kind"] == target_kind + assert value["execution"] == {} + encoded = json.dumps(value) + assert f"qualified-{target_kind}-environment" not in encoded + + +def test_target_kind_is_derived_from_report_and_cli_value_is_only_a_cross_check( + tmp_path, +): + recording, bundle, run_dir = _approved_artifacts(tmp_path, target_kind="rdp") + with pytest.raises(RuntimeValidationError, match="does not match"): + create_runtime_validation_attestation( + recording_derivative=recording, + bundle_derivative=bundle, + run_dir=run_dir, + policy_source="permissive", + risk_class="low", + environment="rdp-lab", + target_kind="citrix", + host=hosted.DEFAULT_HOST, + token="oai_ingest_test", + challenge=_challenge(), + ) + + +def test_native_and_remote_targets_refuse_browser_boundary_fields(tmp_path): + recording, bundle, run_dir = _approved_artifacts(tmp_path, target_kind="windows") + common = { + "recording_derivative": recording, + "bundle_derivative": bundle, + "run_dir": run_dir, + "policy_source": "permissive", + "risk_class": "low", + "environment": "windows-lab", + "host": hosted.DEFAULT_HOST, + "token": "oai_ingest_test", + "challenge": _challenge(), + } + with pytest.raises(RuntimeValidationError, match="Target URL is web-only"): + create_runtime_validation_attestation( + target_url=_TARGET_URL, + **common, + ) + with pytest.raises(RuntimeValidationError, match="Allowed hosts are web-only"): + create_runtime_validation_attestation( + allowed_hosts=["cdn.example.com"], + **common, + ) + + +def test_attestation_refuses_report_without_resolved_target_kind(tmp_path): + recording, bundle, run_dir = _approved_artifacts(tmp_path) + report_path = run_dir / "report.json" + report = RunReport.model_validate_json(report_path.read_text()) + report.execution_target_kind = None + report.save(run_dir) + with pytest.raises(RuntimeValidationError, match="no valid resolved"): + create_runtime_validation_attestation( + recording_derivative=recording, + bundle_derivative=bundle, + run_dir=run_dir, + policy_source="permissive", + risk_class="low", + environment="web-lab", + target_url=_TARGET_URL, + host=hosted.DEFAULT_HOST, + token="oai_ingest_test", + challenge=_challenge(), + ) + + +def test_target_execution_contract_is_closed_per_substrate(): + assert normalize_target_execution("citrix", None, None) == {} + with pytest.raises(RuntimeValidationError, match="Target kind"): + normalize_target_execution("unknown", None, None) + + +def test_local_verifier_retains_signed_v1_web_compatibility(tmp_path): + token = "oai_ingest_test" + value, _ = _attestation(tmp_path, token) + legacy = json.loads(json.dumps(value)) + legacy["schema"] = LEGACY_SCHEMA + legacy.pop("target_kind") + legacy["signature"] = _signature(legacy, token) + verify_runtime_validation_attestation( + legacy, + bundle_sha256=legacy["bundle_sha256"], + token=token, + ) + + @pytest.mark.parametrize( "url", [ diff --git a/tests/test_sanitized_artifact.py b/tests/test_sanitized_artifact.py index cfcf7cce..78b506fa 100644 --- a/tests/test_sanitized_artifact.py +++ b/tests/test_sanitized_artifact.py @@ -847,6 +847,7 @@ def test_schema_files_are_valid_json(): "sanitized-artifact-approval-v1.json", "sanitization-ingest-v1.json", "runtime-validation-attestation-v1.json", + "runtime-validation-attestation-v2.json", ): schema = json.loads((root / name).read_text()) assert schema["$schema"].endswith("2020-12/schema") diff --git a/tests/test_validate_claims.py b/tests/test_validate_claims.py index 21ccb84e..0ce73c2e 100644 --- a/tests/test_validate_claims.py +++ b/tests/test_validate_claims.py @@ -32,6 +32,7 @@ # registries. These paths are asserted to exist by the rot guard below, so if # they are ever moved this test fails loudly rather than testing a fiction. OPTIN_TEST = "tests/e2e/test_citrix_pixel_e2e.py" +INLINE_OPTIN_TEST = "tests/e2e/test_citrix_workspace_standin_e2e.py" CI_TEST = "tests/test_replayer.py" DOC_ARTIFACT = "docs/desktop/CITRIX_PIXEL.md" @@ -76,10 +77,12 @@ def test_every_real_evidence_path_exists() -> None: # --------------------------------------------------------------------------- # def test_optin_detector_flags_the_real_optin_tests() -> None: citrix = (REPO_ROOT / OPTIN_TEST).read_text(encoding="utf-8") + citrix_standin = (REPO_ROOT / INLINE_OPTIN_TEST).read_text(encoding="utf-8") parallels = (REPO_ROOT / "tests/e2e/test_parallels_desktop_e2e.py").read_text( encoding="utf-8" ) assert vc.detect_optin_env(citrix) == "OAFLOW_CITRIX_PIXEL_E2E" + assert vc.detect_optin_env(citrix_standin) == "OAFLOW_CITRIX_STANDIN_E2E" assert vc.detect_optin_env(parallels) == "OAFLOW_PARALLELS_E2E" From dc0d78285f4f55dda7b405058c8e123153508105 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 18:31:47 -0400 Subject: [PATCH 03/10] bind BYOC dispatches to execution substrate --- openadapt_flow/connector/executor.py | 12 +++++ openadapt_flow/connector/protocol.py | 19 +++++++ tests/test_connector.py | 78 ++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/openadapt_flow/connector/executor.py b/openadapt_flow/connector/executor.py index 8cfde7b8..350e5458 100644 --- a/openadapt_flow/connector/executor.py +++ b/openadapt_flow/connector/executor.py @@ -111,6 +111,7 @@ def build_run_argv( policy = settings.policy if policy: argv += ["--policy", policy] + argv += ["--backend", job.target_kind] if params_file is not None: argv += ["--params-file", str(params_file)] if job.target_url: @@ -202,6 +203,7 @@ def _write_policy_audit(job: ByocJob, run_dir: Path) -> None: { "run_id": job.run_id, "org_id": job.org_id, + "target_kind": job.target_kind, "safety": job.safety, "grounding_model": job.grounding_model.model_dump(), }, @@ -275,6 +277,15 @@ def execute_job( report = outcome.report or {} status = status_from_report(outcome.returncode, report) + report_target_kind = report.get("execution_target_kind") + if report_target_kind != job.target_kind: + status = "failed" + outcome_error = ( + "governed child report execution_target_kind does not match " + "the signed dispatch target_kind (fail closed)" + ) + else: + outcome_error = None # 4. The PHI-bearing report body goes to the CUSTOMER'S store, never ours. report_ref = job.report_ref() @@ -294,4 +305,5 @@ def execute_job( metrics=metrics_from_report(report), halt=halt_object(report), report_ref=report_ref, + error=outcome_error, ) diff --git a/openadapt_flow/connector/protocol.py b/openadapt_flow/connector/protocol.py index 5d55da1c..a75276cb 100644 --- a/openadapt_flow/connector/protocol.py +++ b/openadapt_flow/connector/protocol.py @@ -24,6 +24,8 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError +from openadapt_flow.ir import ExecutionTargetKind + class ByocJobParseError(ValueError): """A leased job payload could not be parsed as a BYOC dispatch.""" @@ -82,6 +84,10 @@ class ByocJob(BaseModel): org_id: str workflow_id: str + #: Required first-class execution substrate. The control plane derives this + #: from the signed runtime-validation attestation; the connector never + #: defaults a missing value to web. + target_kind: ExecutionTargetKind storage: Optional[ByocStorage] = None report_path: Optional[str] = None target_url: Optional[str] = None @@ -133,6 +139,19 @@ def ensure_governed(self, *, require_run_token: bool = True) -> None: "byoc dispatch is missing the resolved safety policy; refusing to " "run without the org's governed safety posture (fail closed)" ) + compatibility_kind = self.params.get("target_kind") + if compatibility_kind is not None and compatibility_kind != self.target_kind: + raise ByocGovernanceError( + "byoc dispatch target_kind disagrees with the compatibility " + "params.target_kind; refusing ambiguous substrate routing " + "(fail closed)" + ) + if self.target_kind != "web" and (self.target_url or self.allowed_hosts): + raise ByocGovernanceError( + "native/remote byoc dispatch carries a browser target boundary; " + "app, window, host, and readiness hints must come from the " + "customer-owned deployment profile (fail closed)" + ) # grounding_model is always present (default factory); its absence would # mean the whole policy block was dropped, which `safety` already catches. if require_run_token and not self.run_token: diff --git a/tests/test_connector.py b/tests/test_connector.py index bdecb9a6..2407cd92 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -49,6 +49,7 @@ def _payload(**overrides): "run_id": "00000000-0000-4000-8000-000000000005", "org_id": "org_demo", "workflow_id": "wf_demo_1", + "target_kind": "web", "storage": { "backend": "local", "bundle_ref": "bundles/wf_demo_1/bundle.zip", @@ -81,6 +82,7 @@ def runner(argv, run_dir: Path) -> RunOutcome: SUCCESS_REPORT = { "workflow_name": "wf_demo_1", + "execution_target_kind": "web", "results": [ {"step_id": "s1", "ok": True, "intent": "click Login"}, {"step_id": "s2", "ok": True, "intent": "type patient MRN 12345"}, @@ -114,6 +116,21 @@ def test_execute_job_success_writes_report_to_customer_storage_only(): assert storage.written["org_demo/run_5/report.json"]["success"] is True +def test_execute_job_refuses_child_report_for_different_substrate(): + job = parse_job(_payload(), lease_job_id="bjob_1") + settings = ConnectorSettings(profile=None) + storage = InMemoryCustomerStorage(bundle_dir=Path("/tmp")) + mismatched = {**SUCCESS_REPORT, "execution_target_kind": "citrix"} + + result = execute_job( + job, settings, storage, runner=_fake_success_runner(mismatched) + ) + + assert result.status == "failed" + assert result.error is not None + assert "does not match" in result.error + + def test_callback_body_is_phi_free(): job = parse_job(_payload(), lease_job_id="bjob_1") result = ExecutionResult( @@ -145,6 +162,7 @@ def test_callback_body_is_phi_free(): def test_halt_maps_to_halt_status_and_present_flag(): halt_report = { + "execution_target_kind": "web", "results": [{"step_id": "s1", "ok": True}], "success": False, "terminal_outcome": "halt", @@ -175,6 +193,50 @@ def test_dispatch_without_policy_is_refused(): job.ensure_governed() +def test_dispatch_without_first_class_target_kind_is_refused(): + payload = _payload() + payload.pop("target_kind") + with pytest.raises(ValueError, match="connector contract"): + parse_job(payload, lease_job_id="bjob_1") + + +def test_dispatch_with_unknown_target_kind_is_refused(): + with pytest.raises(ValueError, match="connector contract"): + parse_job(_payload(target_kind="mainframe"), lease_job_id="bjob_1") + + +def test_compatibility_target_kind_must_equal_first_class_value(): + job = parse_job( + _payload( + target_kind="citrix", + target_url=None, + allowed_hosts=[], + params={"vendor": "ACME", "target_kind": "rdp"}, + ), + lease_job_id="bjob_1", + ) + with pytest.raises(ByocGovernanceError, match="disagrees"): + job.ensure_governed() + + +def test_compatibility_target_kind_is_optional(): + job = parse_job( + _payload(params={"vendor": "ACME"}), + lease_job_id="bjob_1", + ) + job.ensure_governed() + + +@pytest.mark.parametrize("target_kind", ["windows", "macos", "linux", "rdp", "citrix"]) +def test_native_remote_dispatch_refuses_browser_boundary(target_kind): + job = parse_job( + _payload(target_kind=target_kind, params={"target_kind": target_kind}), + lease_job_id="bjob_1", + ) + with pytest.raises(ByocGovernanceError, match="browser target boundary"): + job.ensure_governed() + + def test_dispatch_without_run_token_is_refused(): job = parse_job(_payload(run_token=None), lease_job_id="bjob_1") with pytest.raises(ByocGovernanceError, match="callback token"): @@ -228,9 +290,25 @@ def test_governed_run_argv_uses_the_fail_closed_run_verb(): assert "--config" in argv and "/opt/openadapt/deployment.yaml" in argv assert "--policy" in argv and "clinical-write" in argv assert "--params-file" in argv + assert argv[argv.index("--backend") + 1] == "web" assert "replay" not in argv +def test_native_dispatch_routes_exact_first_class_target_kind(): + job = parse_job( + _payload( + target_kind="citrix", + target_url=None, + allowed_hosts=[], + params={"vendor": "ACME", "target_kind": "citrix"}, + ), + lease_job_id="bjob_1", + ) + argv = build_run_argv(job, ConnectorSettings(), Path("/b"), Path("/r"), None) + assert argv[argv.index("--backend") + 1] == "citrix" + assert "--url" not in argv + + # -------------------------------------------------------------------------- # Enrollment + the full loop over a stub control plane (zero network). # -------------------------------------------------------------------------- From 23be20f0c7acd4ea6ef4fffaa9d56f228a9d48b0 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 18:33:35 -0400 Subject: [PATCH 04/10] refresh substrate verification inventory --- docs/VERIFICATION.md | 33 ++++++++++--- docs/verification.json | 107 ++++++++++++++++++++++++++++++++++++++--- public-artifacts.json | 10 ++-- 3 files changed, 135 insertions(+), 15 deletions(-) diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index c982d8f4..b5b639c1 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -10,7 +10,7 @@ ## What is CI-proven today vs. being validated -- **CI-proven today (7):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `mockmed-benchmark-ci-reproducible` +- **CI-proven today (8):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported` - **Being validated — opt-in / infra-gated or field (8):** `effect-verifier-kit`, `windows-desktop-validating`, `macos-native-validating`, `linux-native-validating`, `rdp-validating`, `desktop-recording-validating`, `citrix-pixel-validating`, `openemr-field-benchmark` - **Roadmap / research (1):** `win32-window-replay-roadmap` @@ -232,16 +232,33 @@ - Acceptance covers the named 1280x800 Windows Run/file task and exact VM environment, not arbitrary applications, identity policies, or Citrix. +### `substrate-runtime-validation-supported` — supported — CI-proven today + +> Runtime-validation v2 binds the successful run's resolved target kind (`web|windows|macos|linux|rdp|citrix`) into the signed operator attestation while preserving the same exact artifact, compiler, parameter-schema, strict-lint, policy, risk, report, and opaque environment bindings. Web retains its exact HTTPS/host boundary; native and remote execution emits no app, window, host, readiness, or backend-hint values to Cloud. + +- Surfaces: README.md, docs, website +- Strongest evidence strength: **supported** (tier is `supported`) + +| Backing evidence | Kind | Gating / CI stage | Strength | Proves | +|---|---|---|---|---| +| `tests/test_runtime_validation.py` | test | ci (required PR gate (test)) | supported | Required CI exercises all six target kinds, validates v2 against its closed JSON Schema, refuses target-kind mismatch or missing report provenance, preserves web URL/host checks, keeps native/remote execution empty, verifies the canonical HMAC vector, and retains signed v1 web compatibility. | +| `tests/test_cli_new_commands.py` | test | ci (required PR gate (test)) | supported | Required CI carries the resolved web, Windows, and Citrix backend token through durable resume into the new run report rather than trusting an upload-time relabel. | +| `schemas/runtime-validation-attestation-v2.json` | doc | artifact (doc/benchmark) | roadmap | The public closed envelope requires target_kind and conditionally permits the browser execution fields only for web. | + +**Caveats (honest limits):** + +- The HMAC proves operator-token possession and envelope integrity; it is not independent observation of the local replay or certification of an arbitrary target application. + ### `desktop-recording-validating` — validating — opt-in / infra-gated or field test -> The product CLI records desktop workflows through openadapt-capture and `record --backend windows|rdp` captures the operator's real demonstration and emits the compile-ready recording format, so record -> compile -> replay closes on the desktop substrate, not just the browser. +> The product CLI records desktop workflows through openadapt-capture and `record --backend windows|macos|linux|rdp|citrix` captures the operator's real demonstration and emits the compile-ready recording format, so record -> compile -> replay closes on the desktop substrate, not just the browser. - Surfaces: README.md, docs, website - Strongest evidence strength: **supported** (tier is `validating`) | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| -| `tests/test_desktop_record.py` | test | ci (required PR gate (test)) | supported | The live-capture orchestration + CLI wiring (record --backend windows|rdp -> openadapt-capture -> convert_capture), and that a desktop-shaped recording COMPILES into a bundle and REPLAYS to completion through the desktop backend path, resolving each click to its recorded target (runs on default CI). | +| `tests/test_desktop_record.py` | test | ci (required PR gate (test)) | supported | The live-capture orchestration + CLI wiring (record --backend windows|macos|linux|rdp|citrix -> openadapt-capture -> convert_capture), and that a desktop-shaped recording COMPILES into a bundle and REPLAYS to completion through the desktop backend path, resolving each click to its recorded target (runs on default CI). | | `tests/test_capture_adapter.py` | test | ci (required PR gate (test)) | supported | The capture->recording bridge over a REAL openadapt-capture session (its own event-processing pipeline + frame extraction) feeding the UNMODIFIED compiler (runs on default CI: the `test` job installs the `capture` extra; openadapt-capture >=0.5.4 imports clean headless). | | `docs/desktop/RECORDING.md` | doc | artifact (doc/benchmark) | roadmap | The capture-assisted and live-observer paths, fail-closed secret handling, RDP coordinate binding, and reuse of openadapt-capture + the capture adapter. | @@ -252,7 +269,7 @@ ### `citrix-pixel-validating` — validating — opt-in / infra-gated or field test -> Citrix uses a dedicated exact-Workspace-window backend beneath the governed pixel-only remote-display contract. The backend, readiness gate, durable resume, identity/refusal ladder, and independent effects are code-qualified; live ICA/HDX acceptance remains separately bound to the exact deployment. +> Citrix uses a dedicated exact-Workspace-window backend beneath the governed pixel-only remote-display contract. Required CI covers its readiness gate, governed execution, durable resume, report binding, and refusal contracts; the counted no-DOM qualification covers the driver, visual ladder, independent effects, and drift halts. Live ICA/HDX acceptance remains separately bound to the exact deployment. - Surfaces: README.md, docs, website - Strongest evidence strength: **supported** (tier is `validating`) @@ -261,14 +278,18 @@ |---|---|---|---|---| | `tests/e2e/test_citrix_pixel_e2e.py` | test | opt-in (OAFLOW_CITRIX_PIXEL_E2E) | validating | OPT-IN pixel-only proof: structural_armed_coverage == 0, replay resolves on template/ocr/geometry only, on-screen OCR read-back verifies the write, the identity gate HALTs on a look-alike patient, and render drift triggers halt-on-ambiguity. | | `tests/test_pixel_identity_probe.py` | test | ci (required PR gate (test)) | supported | The pixel-substrate identity probe behind the on-pixels identity gate. | -| `tests/e2e/test_citrix_workspace_standin_e2e.py` | test | ci (post-merge/nightly full suite) | supported | The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success. | +| `tests/test_citrix_workspace_backend.py` | test | ci (required PR gate (test)) | supported | Required CI covers the dedicated backend preset, exact owner/title overrides, readiness probe construction, pixel-only capability boundary, backend factory, and product CLI selectors. | +| `tests/test_run_gate.py` | test | ci (required PR gate (test)) | supported | Required CI refuses governed Citrix execution before action when its readiness binding is absent or blank, admits a sealed bundle carrying a recorded readiness binding, and reports only the resolved backend token rather than sensitive target strings. | +| `tests/test_cli_new_commands.py` | test | ci (required PR gate (test)) | supported | Required CI restores the recorded Citrix owner, exact title, and readiness binding through durable approve/resume, and refuses before backend construction when resumed configuration is incomplete. | +| `tests/test_hosted.py` | test | ci (required PR gate (test)) | supported | Required CI binds a completed local Citrix report to the closed `citrix` execution token without copying target owner, title, or readiness values into the hosted summary. | +| `tests/e2e/test_citrix_workspace_standin_e2e.py` | test | opt-in (OAFLOW_CITRIX_STANDIN_E2E) | validating | The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success. | | `benchmark/citrix_workspace/results.json` | artifact | artifact (doc/benchmark) | roadmap | The retained six-trial record reports code_readiness_accepted=true and ica_hdx_accepted=false, keeping driver readiness distinct from a counted live ICA/HDX qualification. | | `docs/desktop/CITRIX_PIXEL.md` | doc | artifact (doc/benchmark) | roadmap | The Citrix driver model, reusable evidence, exact-deployment acceptance contract, independent-effect boundary, and customer-controlled posture. | **Caveats (honest limits):** - PIXEL-ONLY: no UIA/DOM crosses to the driver. On-screen OCR read-back is SAME-SURFACE, not an independent system-of-record check. -- A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); never runs on default CI, and does not replace a counted live ICA/HDX acceptance record. +- A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1` or `OAFLOW_CITRIX_STANDIN_E2E=1`); neither live-window proof runs on default CI, and neither replaces a counted live ICA/HDX acceptance record. ### `win32-window-replay-roadmap` — roadmap — designed, not yet proven diff --git a/docs/verification.json b/docs/verification.json index e748efc3..3b414a0d 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -666,9 +666,60 @@ ], "errors": [] }, + { + "id": "substrate-runtime-validation-supported", + "claim": "Runtime-validation v2 binds the successful run's resolved target kind (`web|windows|macos|linux|rdp|citrix`) into the signed operator attestation while preserving the same exact artifact, compiler, parameter-schema, strict-lint, policy, risk, report, and opaque environment bindings. Web retains its exact HTTPS/host boundary; native and remote execution emits no app, window, host, readiness, or backend-hint values to Cloud.", + "tier": "supported", + "reproducibility": null, + "surfaces": [ + "README.md", + "docs", + "website" + ], + "strongest_evidence": "supported", + "caveats": [ + "The HMAC proves operator-token possession and envelope integrity; it is not independent observation of the local replay or certification of an arbitrary target application." + ], + "evidence": [ + { + "path": "tests/test_runtime_validation.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "Required CI exercises all six target kinds, validates v2 against its closed JSON Schema, refuses target-kind mismatch or missing report provenance, preserves web URL/host checks, keeps native/remote execution empty, verifies the canonical HMAC vector, and retains signed v1 web compatibility." + }, + { + "path": "tests/test_cli_new_commands.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "Required CI carries the resolved web, Windows, and Citrix backend token through durable resume into the new run report rather than trusting an upload-time relabel." + }, + { + "path": "schemas/runtime-validation-attestation-v2.json", + "kind": "doc", + "exists": true, + "strength": "roadmap", + "gating": "artifact (doc/benchmark)", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "The public closed envelope requires target_kind and conditionally permits the browser execution fields only for web." + } + ], + "errors": [] + }, { "id": "desktop-recording-validating", - "claim": "The product CLI records desktop workflows through openadapt-capture and `record --backend windows|rdp` captures the operator's real demonstration and emits the compile-ready recording format, so record -> compile -> replay closes on the desktop substrate, not just the browser.", + "claim": "The product CLI records desktop workflows through openadapt-capture and `record --backend windows|macos|linux|rdp|citrix` captures the operator's real demonstration and emits the compile-ready recording format, so record -> compile -> replay closes on the desktop substrate, not just the browser.", "tier": "validating", "reproducibility": null, "surfaces": [ @@ -691,7 +742,7 @@ "node": null, "node_found": null, "junit_status": null, - "proves": "The live-capture orchestration + CLI wiring (record --backend windows|rdp -> openadapt-capture -> convert_capture), and that a desktop-shaped recording COMPILES into a bundle and REPLAYS to completion through the desktop backend path, resolving each click to its recorded target (runs on default CI)." + "proves": "The live-capture orchestration + CLI wiring (record --backend windows|macos|linux|rdp|citrix -> openadapt-capture -> convert_capture), and that a desktop-shaped recording COMPILES into a bundle and REPLAYS to completion through the desktop backend path, resolving each click to its recorded target (runs on default CI)." }, { "path": "tests/test_capture_adapter.py", @@ -720,7 +771,7 @@ }, { "id": "citrix-pixel-validating", - "claim": "Citrix uses a dedicated exact-Workspace-window backend beneath the governed pixel-only remote-display contract. The backend, readiness gate, durable resume, identity/refusal ladder, and independent effects are code-qualified; live ICA/HDX acceptance remains separately bound to the exact deployment.", + "claim": "Citrix uses a dedicated exact-Workspace-window backend beneath the governed pixel-only remote-display contract. Required CI covers its readiness gate, governed execution, durable resume, report binding, and refusal contracts; the counted no-DOM qualification covers the driver, visual ladder, independent effects, and drift halts. Live ICA/HDX acceptance remains separately bound to the exact deployment.", "tier": "validating", "reproducibility": null, "surfaces": [ @@ -731,7 +782,7 @@ "strongest_evidence": "supported", "caveats": [ "PIXEL-ONLY: no UIA/DOM crosses to the driver. On-screen OCR read-back is SAME-SURFACE, not an independent system-of-record check.", - "A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1`, macOS screen-recording + accessibility); never runs on default CI, and does not replace a counted live ICA/HDX acceptance record." + "A Parallels VM window is a Citrix ANALOG; real HDX/latency/DPI/lock- screen drift is not simulated. OPT-IN + infra-gated (`OAFLOW_CITRIX_PIXEL_E2E=1` or `OAFLOW_CITRIX_STANDIN_E2E=1`); neither live-window proof runs on default CI, and neither replaces a counted live ICA/HDX acceptance record." ], "evidence": [ { @@ -757,11 +808,55 @@ "proves": "The pixel-substrate identity probe behind the on-pixels identity gate." }, { - "path": "tests/e2e/test_citrix_workspace_standin_e2e.py", + "path": "tests/test_citrix_workspace_backend.py", "kind": "test", "exists": true, "strength": "supported", - "gating": "ci (post-merge/nightly full suite)", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "Required CI covers the dedicated backend preset, exact owner/title overrides, readiness probe construction, pixel-only capability boundary, backend factory, and product CLI selectors." + }, + { + "path": "tests/test_run_gate.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "Required CI refuses governed Citrix execution before action when its readiness binding is absent or blank, admits a sealed bundle carrying a recorded readiness binding, and reports only the resolved backend token rather than sensitive target strings." + }, + { + "path": "tests/test_cli_new_commands.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "Required CI restores the recorded Citrix owner, exact title, and readiness binding through durable approve/resume, and refuses before backend construction when resumed configuration is incomplete." + }, + { + "path": "tests/test_hosted.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "Required CI binds a completed local Citrix report to the closed `citrix` execution token without copying target owner, title, or readiness values into the hosted summary." + }, + { + "path": "tests/e2e/test_citrix_workspace_standin_e2e.py", + "kind": "test", + "exists": true, + "strength": "validating", + "gating": "opt-in (OAFLOW_CITRIX_STANDIN_E2E)", "node": null, "node_found": null, "junit_status": null, diff --git a/public-artifacts.json b/public-artifacts.json index 0d05a6db..e606d46e 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -556,7 +556,7 @@ }, { "path": "claims.yaml", - "sha256": "82877909e7f14167b7076f5516e6654b1d8ab7f9218e9e8869a84173c1005a9a" + "sha256": "8f36ab4c984ba6713ab3944652c9724a2b0e2ba3c3d6fd7449c70d76116c06c6" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1764,7 +1764,7 @@ }, { "path": "docs/verification.json", - "sha256": "debbcdd83db5670295d6878b97a52f7c5350ec9be683effd60799fb34588125d" + "sha256": "f5e8123ef3674e5d5e4ca9bc891016f09e463b50d32053090ba8863d4618a3aa" }, { "path": "openadapt_flow/console/static/console.css", @@ -1838,6 +1838,10 @@ "path": "schemas/runtime-validation-attestation-v1.json", "sha256": "5168eb014c8559d5e722fdd3af4fc2cf394d0c1ae065e1176075c029881cce7c" }, + { + "path": "schemas/runtime-validation-attestation-v2.json", + "sha256": "7aaa25f172842d87560db40cf6e233fb2742710cdde4a0b0126ddb734a327d62" + }, { "path": "schemas/sanitization-ingest-v1.json", "sha256": "3c906fe51d4c6836878ded4194693562d2829df29aedafb423744031e51bf029" @@ -1852,7 +1856,7 @@ }, { "path": "tests/fixtures/runtime-validation-canonical-vector.json", - "sha256": "35d902816ebe997c6351d0e3811c397fdbee45b277ef86461c56ce9c714d68d0" + "sha256": "18c43a710752ce5a930f5db30d4f3af4b5224d7025c5f5b07a0b3bc6ba4e15c9" }, { "path": "tests/fixtures/theme-region-stable-overhalt-v1.16.1.json", From 3bc4f4cca941603e6cf2f493b3772eead36971c8 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 18:54:18 -0400 Subject: [PATCH 05/10] verify exact BYOC bundle archives --- docs/BYOC_CONNECTOR.md | 16 +++-- openadapt_flow/connector/daemon.py | 5 +- openadapt_flow/connector/executor.py | 57 +++++++++++---- openadapt_flow/connector/protocol.py | 10 ++- openadapt_flow/connector/storage.py | 48 ++++++++----- tests/test_connector.py | 103 ++++++++++++++++++++++++--- 6 files changed, 196 insertions(+), 43 deletions(-) diff --git a/docs/BYOC_CONNECTOR.md b/docs/BYOC_CONNECTOR.md index 410ff011..ee2b9def 100644 --- a/docs/BYOC_CONNECTOR.md +++ b/docs/BYOC_CONNECTOR.md @@ -69,13 +69,21 @@ env var (`CONTROL_PLANE_URL`, `BYOC_CONNECTOR_TOKEN`, `BYOC_ORG_ID`, ...). * **The bundle and report bytes** are read from / written to the **customer's own storage** (`--storage-root`, a local encrypted volume). Our control plane holds no URL to them and signs no access. +* **The staged bundle is the exact approved ZIP archive.** `bundle_sha256` binds + the dispatch to those archive bytes (separately from the bundle's semantic + content digest). The Connector copies and hashes that file before extraction, + refuses a missing/different digest or an unpacked directory, then safe-extracts + only the verified copy. A pre-verification failure never echoes the unverified + digest claim in its callback. **Fail-closed everywhere.** A dispatch is refused (reported `failed`, no GUI touched) when it is missing the governed safety policy, missing the run-scoped -callback token, carries an our-owned signed URL, or when the org enabled a -grounding rung whose API key env is not set on this machine. The governed `run` -itself refuses any bundle that is not certified, identity-armed, effect-verified, -and encrypted — those engine gates are unchanged, so identity checks, effect +callback token, exact archive digest, or execution substrate; carries an +our-owned signed URL; points at archive bytes that do not match the signed +digest; conflicts with the substrate compatibility hint; or enables a grounding +rung whose API key env is not set on this machine. The governed `run` itself +refuses any bundle that is not certified, identity-armed, effect-verified, and +encrypted — those engine gates are unchanged, so identity checks, effect verification, and halt-don't-guess all remain intact. ## Enabling the lane (control plane) diff --git a/openadapt_flow/connector/daemon.py b/openadapt_flow/connector/daemon.py index 1f9f77c5..b9d5d6eb 100644 --- a/openadapt_flow/connector/daemon.py +++ b/openadapt_flow/connector/daemon.py @@ -40,7 +40,6 @@ def phi_free_callback_body(job: ByocJob, result: ExecutionResult) -> dict[str, A "workflow_id": job.workflow_id, "bundle_version_id": job.bundle_version_id, "runtime_validation_id": job.runtime_validation_id, - "bundle_sha256": job.bundle_sha256, "mode": job.mode, "status": result.status, "report_path": result.report_ref, @@ -49,6 +48,10 @@ def phi_free_callback_body(job: ByocJob, result: ExecutionResult) -> dict[str, A # name internal paths). error_code is the exact enum run-callback accepts. "error_code": "runner_failure" if result.status == "failed" else None, } + if result.verified_bundle_sha256 is not None: + # Never echo the job's untrusted digest claim. This field exists only + # after the connector copied, hashed, and matched the exact archive. + body["bundle_sha256"] = result.verified_bundle_sha256 if result.halt is not None: # run-callback reads only halt.present; the structured block is additive. body["halt"] = {"present": True} diff --git a/openadapt_flow/connector/executor.py b/openadapt_flow/connector/executor.py index 350e5458..5dbf97e9 100644 --- a/openadapt_flow/connector/executor.py +++ b/openadapt_flow/connector/executor.py @@ -9,8 +9,10 @@ Fail-closed application of the control-plane-delivered policy: -* :meth:`ByocJob.ensure_governed` must pass (policy present, callback token - present, bundle resolvable) or the job is refused before any GUI is touched; +* :meth:`ByocJob.ensure_governed` must pass (policy, callback token, closed + substrate, and exact archive SHA present) or the job is refused before any + GUI is touched; customer storage must then return the exact approved ZIP + bytes, whose digest is checked before safe extraction; * if the org enabled a grounding rung (``grounding_model.enabled``) whose ``api_key_env`` is NOT set in the Connector's own environment, the job is refused — the org required a governed control this machine cannot honor, so we @@ -31,6 +33,7 @@ from __future__ import annotations +import hashlib import json import os import subprocess @@ -42,7 +45,7 @@ from openadapt_flow.connector.config import ConnectorSettings from openadapt_flow.connector.protocol import ByocGovernanceError, ByocJob -from openadapt_flow.connector.storage import CustomerStorage +from openadapt_flow.connector.storage import CustomerStorage, extract_bundle_archive #: A run-gate refusal (fail-closed admission denied) exits 2 before the replay #: creates report.json. @@ -69,6 +72,7 @@ class ExecutionResult: halt: Optional[dict[str, Any]] report_ref: Optional[str] error: Optional[str] = None + verified_bundle_sha256: Optional[str] = None def _grounding_env_available(job: ByocJob) -> bool: @@ -82,6 +86,14 @@ def _grounding_env_available(job: ByocJob) -> bool: return bool(os.environ.get(gm.api_key_env)) +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def build_run_argv( job: ByocJob, settings: ConnectorSettings, @@ -248,6 +260,7 @@ def execute_job( with tempfile.TemporaryDirectory(prefix="oa-byoc-") as tmp: tmp_path = Path(tmp) + bundle_archive = tmp_path / "approved-bundle.zip" bundle_scratch = tmp_path / "bundle" run_dir = tmp_path / "run" bundle_scratch.mkdir(parents=True, exist_ok=True) @@ -255,11 +268,19 @@ def execute_job( _write_policy_audit(job, run_dir) - # 2. Resolve the bundle from the CUSTOMER'S OWN storage (never our URL). + # 2. Copy and verify the EXACT approved archive bytes from the customer's + # storage before extraction. The content digest inside workflow.json is + # a separate semantic binding; neither can substitute for this signed + # archive-byte identity. try: - bundle_dir = storage.fetch_bundle( - job.storage.bundle_ref if job.storage else None, bundle_scratch + fetched_archive = storage.fetch_bundle_archive( + job.storage.bundle_ref if job.storage else None, bundle_archive ) + observed_sha256 = _sha256_file(fetched_archive) + if observed_sha256 != job.bundle_sha256: + raise RuntimeError("bundle archive SHA-256 does not match dispatch") + extract_bundle_archive(fetched_archive, bundle_scratch) + bundle_dir = bundle_scratch except Exception as exc: # storage failure — fail closed, PHI-free msg return ExecutionResult( "failed", @@ -269,11 +290,21 @@ def execute_job( f"customer-storage bundle fetch failed: {type(exc).__name__}", ) - params_file = _write_params_file(job.params, run_dir) + try: + params_file = _write_params_file(job.params, run_dir) - # 3. The governed, fail-closed child invocation. - argv = build_run_argv(job, settings, Path(bundle_dir), run_dir, params_file) - outcome = runner(argv, run_dir) + # 3. The governed, fail-closed child invocation. + argv = build_run_argv(job, settings, Path(bundle_dir), run_dir, params_file) + outcome = runner(argv, run_dir) + except Exception as exc: + return ExecutionResult( + "failed", + {}, + None, + job.report_ref(), + f"governed child invocation failed: {type(exc).__name__}", + verified_bundle_sha256=observed_sha256, + ) report = outcome.report or {} status = status_from_report(outcome.returncode, report) @@ -291,13 +322,14 @@ def execute_job( report_ref = job.report_ref() try: storage.write_report(report_ref, report) - except Exception as exc: # pragma: no cover - report persist best-effort + except Exception as exc: # no durable report means the run is not complete return ExecutionResult( - status, + "failed", metrics_from_report(report), halt_object(report), report_ref, f"customer-storage report write failed: {type(exc).__name__}", + verified_bundle_sha256=observed_sha256, ) return ExecutionResult( @@ -306,4 +338,5 @@ def execute_job( halt=halt_object(report), report_ref=report_ref, error=outcome_error, + verified_bundle_sha256=observed_sha256, ) diff --git a/openadapt_flow/connector/protocol.py b/openadapt_flow/connector/protocol.py index a75276cb..be6734fe 100644 --- a/openadapt_flow/connector/protocol.py +++ b/openadapt_flow/connector/protocol.py @@ -105,7 +105,10 @@ class ByocJob(BaseModel): run_token: Optional[str] = None bundle_version_id: Optional[str] = None runtime_validation_id: Optional[str] = None - bundle_sha256: Optional[str] = None + #: SHA-256 of the exact approved sanitized derivative ZIP bytes staged at + #: ``storage.bundle_ref``. Distinct from the bundle's semantic content + #: digest; required so a valid but different archive cannot be executed. + bundle_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") # --- Governed policy delivery (fail-closed) -------------------------------- #: The org's resolved (baseline-filled) Tier-3 safety block. Always fully @@ -159,6 +162,11 @@ def ensure_governed(self, *, require_run_token: bool = True) -> None: "byoc dispatch is missing a run-scoped callback token; refusing " "to run a job whose outcome we could not report (fail closed)" ) + if not self.bundle_version_id or not self.runtime_validation_id: + raise ByocGovernanceError( + "byoc dispatch is missing its immutable bundle-version or " + "runtime-validation binding (fail closed)" + ) storage_ref = self.storage.bundle_ref if self.storage else None if not storage_ref: raise ByocGovernanceError( diff --git a/openadapt_flow/connector/storage.py b/openadapt_flow/connector/storage.py index 1edf5f20..bd12a16e 100644 --- a/openadapt_flow/connector/storage.py +++ b/openadapt_flow/connector/storage.py @@ -11,9 +11,9 @@ * :class:`LocalCustomerStorage` — REAL: refs resolve under a customer directory (ideally a full-disk-encrypted volume). The on-prem clinic posture. -* :class:`InMemoryCustomerStorage` — tests/dry-run: pretends the bundle is - present and CAPTURES the report in memory, proving the report bytes never - leave for the control plane, with zero infra. +* :class:`InMemoryCustomerStorage` — tests/dry-run: stages exact archive bytes + and CAPTURES the report in memory, proving the report bytes never leave for + the control plane, with zero infra. S3 / Azure Blob backends are documented as production follow-ups (they need a customer cloud to exercise); the operator reference agent in openadapt-cloud @@ -24,6 +24,7 @@ import json import os +import shutil import zipfile from pathlib import Path from typing import Any, Optional, Protocol @@ -34,8 +35,8 @@ class CustomerStorage(Protocol): kind: str - def fetch_bundle(self, ref: Optional[str], dest_dir: Path) -> Path: - """Resolve ``ref`` to an unpacked bundle directory under ``dest_dir``.""" + def fetch_bundle_archive(self, ref: Optional[str], dest_file: Path) -> Path: + """Copy the exact referenced ZIP bytes to ``dest_file``.""" ... def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[str]: @@ -43,7 +44,7 @@ def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[s ... -def _safe_extract(zip_path: Path, dest: Path) -> None: +def extract_bundle_archive(zip_path: Path, dest: Path) -> None: """Extract a zip, refusing any entry that escapes ``dest`` (zip-slip).""" dest = dest.resolve() with zipfile.ZipFile(zip_path) as zf: @@ -62,16 +63,25 @@ class LocalCustomerStorage: def __init__(self, root: str) -> None: self.root = Path(root) - def fetch_bundle(self, ref: Optional[str], dest_dir: Path) -> Path: + def fetch_bundle_archive(self, ref: Optional[str], dest_file: Path) -> Path: if not ref: raise RuntimeError("byoc job has no storage.bundle_ref to read") - src = self.root / ref + root = self.root.resolve() + src = (root / ref).resolve() + if src != root and root not in src.parents: + raise RuntimeError( + "bundle ref escapes the configured customer storage root" + ) if not src.exists(): raise RuntimeError(f"bundle not found in customer storage: {src}") - if src.is_dir(): - return src # already an unpacked bundle directory - _safe_extract(src, dest_dir) - return dest_dir + if not src.is_file(): + raise RuntimeError( + "byoc bundle_ref must resolve to the exact approved ZIP archive, " + "not an unpacked directory" + ) + dest_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dest_file) + return dest_file def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[str]: if not ref: @@ -87,7 +97,7 @@ def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[s class InMemoryCustomerStorage: - """Tests/dry-run backend: a pre-staged bundle dir + an in-memory report sink. + """Tests/dry-run backend: exact bundle bytes + an in-memory report sink. Proves the report bytes are written to the CUSTOMER side and never returned to the control plane, with zero infra. @@ -95,12 +105,16 @@ class InMemoryCustomerStorage: kind = "memory" - def __init__(self, bundle_dir: Optional[Path] = None) -> None: - self.bundle_dir = bundle_dir + def __init__(self, bundle_bytes: Optional[bytes] = None) -> None: + self.bundle_bytes = bundle_bytes self.written: dict[str, dict[str, Any]] = {} - def fetch_bundle(self, ref: Optional[str], dest_dir: Path) -> Path: - return self.bundle_dir if self.bundle_dir is not None else dest_dir + def fetch_bundle_archive(self, ref: Optional[str], dest_file: Path) -> Path: + if self.bundle_bytes is None: + raise RuntimeError("in-memory customer storage has no bundle archive") + dest_file.parent.mkdir(parents=True, exist_ok=True) + dest_file.write_bytes(self.bundle_bytes) + return dest_file def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[str]: self.written[ref or "?"] = report diff --git a/tests/test_connector.py b/tests/test_connector.py index 2407cd92..5d48d282 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -17,7 +17,10 @@ from __future__ import annotations +import hashlib +import io import json +import zipfile from pathlib import Path import httpx @@ -38,6 +41,19 @@ ) from openadapt_flow.connector.executor import RunOutcome from openadapt_flow.connector.protocol import ByocGovernanceError +from openadapt_flow.connector.storage import LocalCustomerStorage + + +def _bundle_archive_bytes() -> bytes: + stream = io.BytesIO() + info = zipfile.ZipInfo("workflow.json", date_time=(2026, 1, 1, 0, 0, 0)) + with zipfile.ZipFile(stream, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(info, "{}") + return stream.getvalue() + + +_BUNDLE_BYTES = _bundle_archive_bytes() +_BUNDLE_SHA256 = hashlib.sha256(_BUNDLE_BYTES).hexdigest() # -------------------------------------------------------------------------- @@ -64,7 +80,7 @@ def _payload(**overrides): "run_token": "a" * 64, "bundle_version_id": "bv_1", "runtime_validation_id": "rv_1", - "bundle_sha256": "b" * 64, + "bundle_sha256": _BUNDLE_SHA256, "safety": {"halt.on_ambiguous": True, "effects.verify": True}, "grounding_model": {"enabled": False, "api_key_env": ""}, } @@ -102,7 +118,7 @@ def runner(argv, run_dir: Path) -> RunOutcome: def test_execute_job_success_writes_report_to_customer_storage_only(): job = parse_job(_payload(), lease_job_id="bjob_1") settings = ConnectorSettings(profile=None) - storage = InMemoryCustomerStorage(bundle_dir=Path("/tmp")) + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) result = execute_job( job, settings, storage, runner=_fake_success_runner(SUCCESS_REPORT) @@ -119,7 +135,7 @@ def test_execute_job_success_writes_report_to_customer_storage_only(): def test_execute_job_refuses_child_report_for_different_substrate(): job = parse_job(_payload(), lease_job_id="bjob_1") settings = ConnectorSettings(profile=None) - storage = InMemoryCustomerStorage(bundle_dir=Path("/tmp")) + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) mismatched = {**SUCCESS_REPORT, "execution_target_kind": "citrix"} result = execute_job( @@ -138,13 +154,14 @@ def test_callback_body_is_phi_free(): metrics={"steps": 2, "steps_ok": 2, "halts": 0}, halt=None, report_ref="org_demo/run_5/report.json", + verified_bundle_sha256=_BUNDLE_SHA256, ) body = phi_free_callback_body(job, result) # The immutable bundle binding the control plane verifies is echoed. assert body["bundle_version_id"] == "bv_1" assert body["runtime_validation_id"] == "rv_1" - assert body["bundle_sha256"] == "b" * 64 + assert body["bundle_sha256"] == _BUNDLE_SHA256 assert body["status"] == "success" assert body["report_path"] == "org_demo/run_5/report.json" @@ -170,7 +187,7 @@ def test_halt_maps_to_halt_status_and_present_flag(): } job = parse_job(_payload(), lease_job_id="bjob_1") settings = ConnectorSettings() - storage = InMemoryCustomerStorage(bundle_dir=Path("/tmp")) + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) def runner(argv, run_dir: Path) -> RunOutcome: (run_dir / "report.json").write_text(json.dumps(halt_report)) @@ -200,6 +217,19 @@ def test_dispatch_without_first_class_target_kind_is_refused(): parse_job(payload, lease_job_id="bjob_1") +def test_dispatch_without_exact_bundle_archive_digest_is_refused(): + payload = _payload() + payload.pop("bundle_sha256") + with pytest.raises(ValueError, match="connector contract"): + parse_job(payload, lease_job_id="bjob_1") + + +@pytest.mark.parametrize("digest", ["B" * 64, "b" * 63, "not-a-digest"]) +def test_dispatch_with_noncanonical_bundle_archive_digest_is_refused(digest): + with pytest.raises(ValueError, match="connector contract"): + parse_job(_payload(bundle_sha256=digest), lease_job_id="bjob_1") + + def test_dispatch_with_unknown_target_kind_is_refused(): with pytest.raises(ValueError, match="connector contract"): parse_job(_payload(target_kind="mainframe"), lease_job_id="bjob_1") @@ -243,6 +273,13 @@ def test_dispatch_without_run_token_is_refused(): job.ensure_governed() +@pytest.mark.parametrize("field", ["bundle_version_id", "runtime_validation_id"]) +def test_dispatch_without_immutable_validation_binding_is_refused(field): + job = parse_job(_payload(**{field: None}), lease_job_id="bjob_1") + with pytest.raises(ByocGovernanceError, match="immutable bundle-version"): + job.ensure_governed() + + def test_dispatch_carrying_our_signed_url_is_refused(): job = parse_job( _payload(bundle_download_url="https://our.storage/signed"), lease_job_id="b" @@ -264,7 +301,7 @@ def test_execute_refuses_when_required_grounding_key_is_absent(monkeypatch): lease_job_id="bjob_1", ) settings = ConnectorSettings() - storage = InMemoryCustomerStorage(bundle_dir=Path("/tmp")) + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) # A runner that would "succeed" — the governance gate must refuse BEFORE it. called = {"ran": False} @@ -279,6 +316,56 @@ def runner(argv, run_dir: Path) -> RunOutcome: assert called["ran"] is False # fail closed: never ran without the rung +def test_execute_refuses_archive_digest_mismatch_before_runner(): + job = parse_job(_payload(bundle_sha256="0" * 64), lease_job_id="bjob_1") + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) + called = {"ran": False} + + def runner(argv, run_dir: Path) -> RunOutcome: + called["ran"] = True + return RunOutcome(returncode=0, report=SUCCESS_REPORT) + + result = execute_job(job, ConnectorSettings(), storage, runner=runner) + assert result.status == "failed" + assert "storage bundle fetch failed" in (result.error or "") + assert called["ran"] is False + assert result.verified_bundle_sha256 is None + assert "bundle_sha256" not in phi_free_callback_body(job, result) + + +def test_child_failure_after_archive_verification_carries_verified_digest(): + job = parse_job(_payload(), lease_job_id="bjob_1") + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) + + def runner(argv, run_dir: Path) -> RunOutcome: + raise RuntimeError("child unavailable") + + result = execute_job(job, ConnectorSettings(), storage, runner=runner) + assert result.status == "failed" + assert result.verified_bundle_sha256 == _BUNDLE_SHA256 + callback = phi_free_callback_body(job, result) + assert callback["bundle_sha256"] == _BUNDLE_SHA256 + + +def test_local_storage_refuses_unpacked_directory_bundle(tmp_path): + unpacked = tmp_path / "bundles" / "wf" + unpacked.mkdir(parents=True) + storage = LocalCustomerStorage(str(tmp_path)) + with pytest.raises(RuntimeError, match="exact approved ZIP"): + storage.fetch_bundle_archive("bundles/wf", tmp_path / "scratch.zip") + + +def test_local_storage_copies_exact_archive_bytes(tmp_path): + archive = tmp_path / "bundles" / "wf.zip" + archive.parent.mkdir(parents=True) + archive.write_bytes(_BUNDLE_BYTES) + storage = LocalCustomerStorage(str(tmp_path)) + copied = storage.fetch_bundle_archive( + "bundles/wf.zip", tmp_path / "scratch" / "approved.zip" + ) + assert copied.read_bytes() == _BUNDLE_BYTES + + def test_governed_run_argv_uses_the_fail_closed_run_verb(): job = parse_job(_payload(), lease_job_id="bjob_1") settings = ConnectorSettings( @@ -412,7 +499,7 @@ def test_full_loop_dispatch_execute_callback_ack(): token=client.token, poll_wait_s=0, ) - storage = InMemoryCustomerStorage(bundle_dir=Path("/tmp")) + storage = InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES) result = run_once( client, settings, @@ -452,7 +539,7 @@ def test_cross_org_isolation_connector_never_sees_another_orgs_job(): client, settings, runner=_fake_success_runner(SUCCESS_REPORT), - storage_factory=lambda job: InMemoryCustomerStorage(bundle_dir=Path("/tmp")), + storage_factory=lambda job: InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES), ) assert result is None # org_A connector sees no work; org_B's job stays queued assert cp.jobs[0]["status"] == "queued" From d105f1f160beb7465ff34fa54f4a73d380ebfd22 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 19:00:40 -0400 Subject: [PATCH 06/10] ack BYOC leases only after callback --- docs/BYOC_CONNECTOR.md | 8 +++++- openadapt_flow/connector/client.py | 8 +++--- openadapt_flow/connector/daemon.py | 30 ++++++++++++++-------- tests/test_connector.py | 40 ++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/docs/BYOC_CONNECTOR.md b/docs/BYOC_CONNECTOR.md index ee2b9def..4ed06ea1 100644 --- a/docs/BYOC_CONNECTOR.md +++ b/docs/BYOC_CONNECTOR.md @@ -28,7 +28,7 @@ poll -> POST /api/connector/poll long-poll; lease the next queued job execute -> openadapt-flow run ... the governed admission gate + Replayer, against the CUSTOMER'S own storage callback -> POST /api/internal/run-callback PHI-free status/metrics -ack -> POST /api/connector/ack release the lease (done|failed) +ack -> POST /api/connector/ack release only after callback acceptance ``` ## Install and run @@ -86,6 +86,12 @@ refuses any bundle that is not certified, identity-armed, effect-verified, and encrypted — those engine gates are unchanged, so identity checks, effect verification, and halt-don't-guess all remain intact. +The callback is retried a bounded three times. The Connector ACKs a job only +after the control plane accepts its outcome. If delivery still fails, the lease +is left unacknowledged; Cloud marks expiry as `lease_expired_uncertain` and +never blindly re-offers it, so an operator checks the real effect before +authorizing a fresh run. + ## Enabling the lane (control plane) The lane is off by default. An operator enables it with diff --git a/openadapt_flow/connector/client.py b/openadapt_flow/connector/client.py index 2bafd468..e3bdc014 100644 --- a/openadapt_flow/connector/client.py +++ b/openadapt_flow/connector/client.py @@ -118,9 +118,9 @@ def run_callback(self, body: dict[str, Any], *, run_token: Optional[str]) -> Non """POST PHI-free run status/metrics via the existing callback boundary. Authenticated by the run-scoped ``x-run-token`` delivered in the job - (proves this run; forbids forging another's status). Best-effort: - observability/status must not crash the loop, but a hard transport error - propagates so the caller records it. + (proves this run; forbids forging another's status). Every non-2xx + response propagates: the caller must not ACK a lease whose authoritative + run outcome the control plane rejected or did not receive. """ headers = {"content-type": "application/json"} if run_token: @@ -128,7 +128,7 @@ def run_callback(self, body: dict[str, Any], *, run_token: Optional[str]) -> Non resp = self._client.post( "/api/internal/run-callback", json=body, headers=headers ) - if resp.status_code >= 500: + if not 200 <= resp.status_code < 300: raise ConnectorClientError( f"run-callback failed: {resp.status_code} {resp.text[:300]}" ) diff --git a/openadapt_flow/connector/daemon.py b/openadapt_flow/connector/daemon.py index b9d5d6eb..7de4585a 100644 --- a/openadapt_flow/connector/daemon.py +++ b/openadapt_flow/connector/daemon.py @@ -14,6 +14,7 @@ from __future__ import annotations +import time from typing import Any, Callable, Optional from openadapt_flow.connector.client import ConnectorClient @@ -25,6 +26,7 @@ #: storage_factory(job) -> CustomerStorage. Injected in tests; the default builds #: from settings + the job's backend hint. StorageFactory = Callable[[ByocJob], CustomerStorage] +CALLBACK_ATTEMPTS = 3 def phi_free_callback_body(job: ByocJob, result: ExecutionResult) -> dict[str, Any]: @@ -81,16 +83,23 @@ def handle_job( "failed", {}, None, job.report_ref(), f"{type(exc).__name__}" ) - # PHI-free callback (best effort; a transport error is caught so we still ack - # and release the lease rather than stranding the job). - try: - client.run_callback( - phi_free_callback_body(job, result), run_token=job.run_token - ) - except Exception: # noqa: BLE001 - callback failure must not strand the lease - pass - - if job.lease_job_id: + # The callback is the authoritative run outcome. Retry briefly for a + # transient transport failure, but NEVER ACK the lease unless Cloud accepted + # it. An unacked lease expires to `lease_expired_uncertain` and is not + # automatically re-offered, preventing blind duplicate actuation after a + # crash or partition. + callback_accepted = False + callback_body = phi_free_callback_body(job, result) + for attempt in range(CALLBACK_ATTEMPTS): + try: + client.run_callback(callback_body, run_token=job.run_token) + callback_accepted = True + break + except Exception: # noqa: BLE001 - bounded retry, then leave lease unacked + if attempt + 1 < CALLBACK_ATTEMPTS: + time.sleep(0.25 * (2**attempt)) + + if job.lease_job_id and callback_accepted: ack_status = "failed" if result.status == "failed" else "done" try: client.ack(job.lease_job_id, ack_status, result.error) @@ -101,6 +110,7 @@ def handle_job( "job_id": job.lease_job_id, "run_id": job.run_id, "status": result.status, + "callback_accepted": callback_accepted, } diff --git a/tests/test_connector.py b/tests/test_connector.py index 5d48d282..375e6c46 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -408,6 +408,8 @@ def __init__(self): self.tokens = {} # token -> org_id self.jobs = [] # queued jobs (each {id, org_id, payload, status, leased_by}) self.callbacks = [] + self.callback_attempts = 0 + self.callback_status = 200 self.acks = [] self._n = 0 @@ -458,6 +460,9 @@ def handler(self, request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"ok": True}) if path == "/api/internal/run-callback": + self.callback_attempts += 1 + if self.callback_status != 200: + return httpx.Response(self.callback_status, json={"ok": False}) self.callbacks.append({"headers": dict(request.headers), "body": body}) return httpx.Response(200, json={"ok": True}) @@ -520,6 +525,41 @@ def test_full_loop_dispatch_execute_callback_ack(): assert cp.acks[0]["status"] == "done" +@pytest.mark.parametrize("callback_status", [400, 503]) +def test_callback_rejection_is_retried_but_never_acked(callback_status, monkeypatch): + monkeypatch.setattr("openadapt_flow.connector.daemon.time.sleep", lambda _: None) + cp = StubControlPlane() + cp.callback_status = callback_status + cp.enqueue("org_demo", _payload()) + transport = httpx.MockTransport(cp.handler) + client = ConnectorClient("https://app.test", transport=transport) + client.enroll(enrollment_secret="s", org_id="org_demo", name="n") + settings = ConnectorSettings( + control_plane_url="https://app.test", + org_id="org_demo", + token=client.token, + poll_wait_s=0, + ) + + result = run_once( + client, + settings, + runner=_fake_success_runner(SUCCESS_REPORT), + storage_factory=lambda job: InMemoryCustomerStorage(bundle_bytes=_BUNDLE_BYTES), + ) + + assert result == { + "job_id": "bjob_1", + "run_id": _payload()["run_id"], + "status": "success", + "callback_accepted": False, + } + assert cp.callback_attempts == 3 + assert cp.callbacks == [] + assert cp.acks == [] + assert cp.jobs[0]["status"] == "leased" + + def test_cross_org_isolation_connector_never_sees_another_orgs_job(): cp = StubControlPlane() cp.enqueue("org_B", _payload(org_id="org_B")) # a DIFFERENT org's job From 8cbed6d41e116aeb50dc0eb9719257b71b3ecc81 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 19:01:51 -0400 Subject: [PATCH 07/10] align connector lease documentation --- openadapt_flow/connector/__init__.py | 2 +- openadapt_flow/connector/daemon.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/openadapt_flow/connector/__init__.py b/openadapt_flow/connector/__init__.py index 84f8e479..33347078 100644 --- a/openadapt_flow/connector/__init__.py +++ b/openadapt_flow/connector/__init__.py @@ -13,7 +13,7 @@ (identity gates + effect verification + halt-don't-guess intact), against the CUSTOMER'S own storage callback -> POST /api/internal/run-callback (PHI-free status/metrics) - ack -> POST /api/connector/ack (release the lease done|failed) + ack -> POST /api/connector/ack (only after callback acceptance) CLI: ``openadapt-flow connector enroll`` then ``openadapt-flow connector run``. """ diff --git a/openadapt_flow/connector/daemon.py b/openadapt_flow/connector/daemon.py index 7de4585a..7b80d5a5 100644 --- a/openadapt_flow/connector/daemon.py +++ b/openadapt_flow/connector/daemon.py @@ -71,7 +71,9 @@ def handle_job( ) -> dict[str, Any]: """Run one leased job end to end: execute -> PHI-free callback -> ack. - Never raises: a failure still reports PHI-free + releases the lease. + Never raises: an execution failure still reports PHI-free and is ACKed only + after Cloud accepts that outcome. A callback/ACK delivery failure leaves the + lease to expire as terminal ``lease_expired_uncertain`` without re-offer. """ try: storage = storage_factory(job) @@ -103,7 +105,7 @@ def handle_job( ack_status = "failed" if result.status == "failed" else "done" try: client.ack(job.lease_job_id, ack_status, result.error) - except Exception: # noqa: BLE001 - lease will expire + re-offer if ack fails + except Exception: # noqa: BLE001 - terminal uncertainty, never re-offered pass return { From 73cfad890dd238ae991fc03ca4ef8cef020bd26c Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 19:10:04 -0400 Subject: [PATCH 08/10] fix: contain BYOC report writes --- openadapt_flow/connector/storage.py | 20 +++++++++++++------- tests/test_connector.py | 7 +++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/openadapt_flow/connector/storage.py b/openadapt_flow/connector/storage.py index bd12a16e..67c06633 100644 --- a/openadapt_flow/connector/storage.py +++ b/openadapt_flow/connector/storage.py @@ -44,6 +44,17 @@ def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[s ... +def _customer_path(root: Path, ref: str, *, kind: str) -> Path: + """Resolve one relative customer-storage key without crossing its root.""" + if Path(ref).is_absolute() or "\\" in ref: + raise RuntimeError(f"{kind} ref must be a relative customer-storage key") + resolved_root = root.resolve() + resolved = (resolved_root / ref).resolve() + if resolved == resolved_root or resolved_root not in resolved.parents: + raise RuntimeError(f"{kind} ref escapes the configured customer storage root") + return resolved + + def extract_bundle_archive(zip_path: Path, dest: Path) -> None: """Extract a zip, refusing any entry that escapes ``dest`` (zip-slip).""" dest = dest.resolve() @@ -66,12 +77,7 @@ def __init__(self, root: str) -> None: def fetch_bundle_archive(self, ref: Optional[str], dest_file: Path) -> Path: if not ref: raise RuntimeError("byoc job has no storage.bundle_ref to read") - root = self.root.resolve() - src = (root / ref).resolve() - if src != root and root not in src.parents: - raise RuntimeError( - "bundle ref escapes the configured customer storage root" - ) + src = _customer_path(self.root, ref, kind="bundle") if not src.exists(): raise RuntimeError(f"bundle not found in customer storage: {src}") if not src.is_file(): @@ -86,7 +92,7 @@ def fetch_bundle_archive(self, ref: Optional[str], dest_file: Path) -> Path: def write_report(self, ref: Optional[str], report: dict[str, Any]) -> Optional[str]: if not ref: return None - dest = self.root / ref + dest = _customer_path(self.root, ref, kind="report") dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(json.dumps(report, indent=2), encoding="utf-8") try: diff --git a/tests/test_connector.py b/tests/test_connector.py index 375e6c46..426e3fa7 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -366,6 +366,13 @@ def test_local_storage_copies_exact_archive_bytes(tmp_path): assert copied.read_bytes() == _BUNDLE_BYTES +def test_local_storage_refuses_report_ref_outside_root(tmp_path): + storage = LocalCustomerStorage(str(tmp_path / "customer")) + with pytest.raises(RuntimeError, match="report ref escapes"): + storage.write_report("../escaped.json", {"success": True}) + assert not (tmp_path / "escaped.json").exists() + + def test_governed_run_argv_uses_the_fail_closed_run_verb(): job = parse_job(_payload(), lease_job_id="bjob_1") settings = ConnectorSettings( From ba9fea49dd3ba327daf0a23dce3b9cd6db5096cb Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 19:12:36 -0400 Subject: [PATCH 09/10] docs: align connector enrollment with Cloud --- docs/BYOC_CONNECTOR.md | 33 +++++++++++++++------------- openadapt_flow/__main__.py | 7 +++--- openadapt_flow/connector/__init__.py | 5 +++-- openadapt_flow/connector/client.py | 9 ++++---- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/docs/BYOC_CONNECTOR.md b/docs/BYOC_CONNECTOR.md index 4ed06ea1..c4af427f 100644 --- a/docs/BYOC_CONNECTOR.md +++ b/docs/BYOC_CONNECTOR.md @@ -23,7 +23,7 @@ inbound ports.** ## The loop (all outbound) ``` -register -> POST /api/connector/register enroll once, get a per-connector token +create -> authenticated Cloud settings mint a revocable org-scoped token once poll -> POST /api/connector/poll long-poll; lease the next queued job execute -> openadapt-flow run ... the governed admission gate + Replayer, against the CUSTOMER'S own storage @@ -39,22 +39,24 @@ the *same* fail-closed `openadapt-flow run` you would run locally. ```bash pip install openadapt-flow -# 1. Enroll this machine once (mints + persists a per-connector token, 0600). -openadapt-flow connector enroll \ +# 1. In app.openadapt.ai, open Dashboard → Settings → +# Customer-controlled connector, create a connector, and copy its token once +# into this machine's service manager or secret store. +export BYOC_CONNECTOR_TOKEN='' + +# 2. Run the daemon (poll -> execute -> report -> ack, until interrupted). +openadapt-flow connector run \ --control-plane-url https://app.openadapt.ai \ - --enrollment-secret "$BYOC_ENROLLMENT_SECRET" \ --org-id org_your_clinic \ --profile /opt/openadapt/deployment.yaml \ --storage-backend local \ --storage-root /srv/openadapt # a full-disk-encrypted customer volume - -# 2. Run the daemon (poll -> execute -> report -> ack, until interrupted). -openadapt-flow connector run ``` -Enrollment persists to `~/.openadapt/connector.toml` (mode 0600 — it holds the -token). A bare `connector run` reads it back. Every flag also resolves from an -env var (`CONTROL_PLANE_URL`, `BYOC_CONNECTOR_TOKEN`, `BYOC_ORG_ID`, ...). +Every flag also resolves from an environment variable +(`CONTROL_PLANE_URL`, `BYOC_CONNECTOR_TOKEN`, `BYOC_ORG_ID`, ...). +`connector enroll` remains available for mock/development control planes; live +Cloud token creation is authenticated and organization-scoped in the dashboard. ## Data boundary and safety @@ -95,8 +97,9 @@ authorizing a fresh run. ## Enabling the lane (control plane) The lane is off by default. An operator enables it with -`BYOC_ENABLED=true` **and** a configured `BYOC_ENROLLMENT_SECRET`, and flips the -org's `deployment_kind` to `byoc`. See openadapt-cloud `src/lib/byocLane.ts`. +`BYOC_ENABLED=true` and authenticated run callbacks +(`RUNNER_SHARED_SECRET`), then sets the organization's `deployment_kind` to +`byoc`. See openadapt-cloud `src/lib/byocLane.ts`. ## What works today vs. what remains for production @@ -112,9 +115,9 @@ org's `deployment_kind` to `byoc`. See openadapt-cloud `src/lib/byocLane.ts`. **Remaining for production:** -* **Hardened auth.** Enrollment is a single org-shared secret today; production - wants per-org (ideally short-lived, rotating) enrollment tokens, plus - mTLS/private-link between the Connector and the control plane. +* **Private transport options.** Connector tokens are organization-scoped, + revocable, and stored only as hashes in Cloud. Deployments that require + mTLS/private-link add that transport at the customer network boundary. * **Full policy materialization.** The delivered safety block governs *dispatch* and the two fail-closed gates above; materializing every safety key into the engine's runtime config (so e.g. the delivered grounding endpoint is the one diff --git a/openadapt_flow/__main__.py b/openadapt_flow/__main__.py index db2dc702..9fea1cf0 100644 --- a/openadapt_flow/__main__.py +++ b/openadapt_flow/__main__.py @@ -3411,7 +3411,7 @@ def build_parser() -> argparse.ArgumentParser: pe = csub.add_parser( "enroll", - help="Enroll this machine as a Connector and persist the token (0600)", + help="Enroll against a mock/development control plane and persist its token", ) pe.add_argument( "--control-plane-url", help="Control plane base URL (or CONTROL_PLANE_URL)" @@ -3632,8 +3632,9 @@ def _cmd_connector(args: argparse.Namespace) -> int: # verb == "run" if not settings.token: print( - "connector run: not enrolled. Run `openadapt-flow connector enroll` " - "first, or pass --token / BYOC_CONNECTOR_TOKEN." + "connector run: no token. Create an organization connector in " + "OpenAdapt Cloud → Settings, then pass --token or set " + "BYOC_CONNECTOR_TOKEN." ) return 2 client = ConnectorClient(settings.control_plane_url, token=settings.token) diff --git a/openadapt_flow/connector/__init__.py b/openadapt_flow/connector/__init__.py index 33347078..9f86e246 100644 --- a/openadapt_flow/connector/__init__.py +++ b/openadapt_flow/connector/__init__.py @@ -7,7 +7,7 @@ PHI-free status/halt metadata ever flows back to the control plane. Loop (all outbound HTTPS; zero inbound ports): - register -> POST /api/connector/register (enroll once, get a token) + create -> authenticated Cloud settings (mint an org-scoped token once) poll -> POST /api/connector/poll (long-poll; lease the next job) execute -> the governed ``openadapt-flow run`` admission gate + Replayer (identity gates + effect verification + halt-don't-guess intact), @@ -15,7 +15,8 @@ callback -> POST /api/internal/run-callback (PHI-free status/metrics) ack -> POST /api/connector/ack (only after callback acceptance) -CLI: ``openadapt-flow connector enroll`` then ``openadapt-flow connector run``. +CLI: create the token in Cloud settings, then ``openadapt-flow connector run``. +``connector enroll`` is retained for mock/development control planes. """ from openadapt_flow.connector.client import ConnectorClient, ConnectorClientError diff --git a/openadapt_flow/connector/client.py b/openadapt_flow/connector/client.py index e3bdc014..0b2a0d4f 100644 --- a/openadapt_flow/connector/client.py +++ b/openadapt_flow/connector/client.py @@ -6,14 +6,15 @@ runner or a Citrix Cloud Connector). The customer opens ZERO inbound ports. Endpoints (all outbound): - * ``POST /api/connector/register`` enroll once -> per-connector token + * ``POST /api/connector/register`` mock/dev enrollment compatibility * ``POST /api/connector/poll`` long-poll -> lease the next job * ``POST /api/connector/ack`` release the lease (done|failed) * ``POST /api/internal/run-callback`` PHI-free status/metrics (existing boundary) -Auth: the per-connector token as ``Authorization: Bearer`` on poll/ack; the org -enrollment secret (``x-byoc-enrollment-secret``) once on enroll; the run-scoped -``x-run-token`` on the callback. Only ``httpx`` (already core) + stdlib is used. +Auth: the organization-scoped per-connector token as ``Authorization: Bearer`` +on poll/ack and the run-scoped ``x-run-token`` on the callback. Live Cloud +mints the connector token through its authenticated settings API; the legacy +enrollment secret is used only by mock/development control planes. A custom ``transport`` (an :class:`httpx.BaseTransport`) may be injected so tests drive the whole enroll -> poll -> callback -> ack loop with ZERO network. From 5c8a03d7af57847560398ec4c26860d24de5862b Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 24 Jul 2026 19:13:39 -0400 Subject: [PATCH 10/10] fix: fail closed on malformed connector reports --- openadapt_flow/connector/executor.py | 20 +++++++++++++++----- tests/test_connector.py | 6 ++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/openadapt_flow/connector/executor.py b/openadapt_flow/connector/executor.py index 5dbf97e9..24f999e1 100644 --- a/openadapt_flow/connector/executor.py +++ b/openadapt_flow/connector/executor.py @@ -154,10 +154,17 @@ def status_from_report(returncode: int, report: dict[str, Any]) -> str: failed : anything else (a hard failure, a crash, a gate refusal, or a missing report). """ - if returncode == 0 and report.get("success"): - return "success" - if report and (report.get("terminal_outcome") == "halt" or report.get("halt")): + if report and ( + report.get("terminal_outcome") == "halt" + or ("halt" in report and report["halt"] is not None) + ): return "halt" + if ( + returncode == 0 + and report.get("success") is True + and report.get("terminal_outcome") in (None, "success") + ): + return "success" return "failed" @@ -174,7 +181,10 @@ def numeric(value: Any, default: float) -> float: "steps": len(results), "steps_ok": sum(1 for r in results if r.get("ok")), "halts": 1 - if (report.get("terminal_outcome") == "halt" or report.get("halt")) + if ( + report.get("terminal_outcome") == "halt" + or ("halt" in report and report["halt"] is not None) + ) else 0, "heals": numeric(report.get("heal_count"), 0), "model_calls": numeric(report.get("model_calls"), 0), @@ -183,7 +193,7 @@ def numeric(value: Any, default: float) -> float: def halt_object(report: dict[str, Any]) -> Optional[dict[str, Any]]: - if report.get("halt"): + if "halt" in report and report["halt"] is not None: return report["halt"] if report.get("terminal_outcome") == "halt": return {"outcome": "halt"} diff --git a/tests/test_connector.py b/tests/test_connector.py index 426e3fa7..e0b4c635 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -38,6 +38,7 @@ phi_free_callback_body, run_once, save_enrollment, + status_from_report, ) from openadapt_flow.connector.executor import RunOutcome from openadapt_flow.connector.protocol import ByocGovernanceError @@ -147,6 +148,11 @@ def test_execute_job_refuses_child_report_for_different_substrate(): assert "does not match" in result.error +def test_malformed_child_report_never_becomes_success(): + assert status_from_report(0, {"success": "true"}) == "failed" + assert status_from_report(0, {"success": True, "halt": {}}) == "halt" + + def test_callback_body_is_phi_free(): job = parse_job(_payload(), lease_job_id="bjob_1") result = ExecutionResult(