diff --git a/docs/eval_results/current_flow_v1_16_1_local_20260718/REPORT.md b/docs/eval_results/current_flow_v1_16_1_local_20260718/REPORT.md new file mode 100644 index 0000000..ee8d85c --- /dev/null +++ b/docs/eval_results/current_flow_v1_16_1_local_20260718/REPORT.md @@ -0,0 +1,70 @@ +# Exact-current local Flow performance + +This is a deterministic-runtime overhead and bounded robustness comparison. It is **not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `113ce992b491576d77236f495b983165ce7a63bd` (version `1.16.1`; tracked-clean source) +- Release tag: `v1.16.1` +- Wheel SHA-256: `c7073283475e7ae722db2478b499d962364851104e09edb71f254ba21c1310cd` +- Evals base commit: `7629ba5a2447c919e499e88e9b4eedbc80c8f3ab` +- Runner SHA-256: `ac58c0b9a02cfc991144a1f5c7a2814c6b6b748bcba27b87f8e1027ee575970f` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.7` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed only; no cloud VM, hosted runner, or model API + +## Counted result + +One synthetic MockMed workflow: sign in, open the intended referral, create a Triage encounter, enter a trial-unique note, and save. Each arm used a fresh browser. The arm-independent screenshot/OCR oracle required the exact saved note, Triage row, and intended patient, and separately flagged wrong-target writes. + +| Condition | Arm | Runs | Task success | Silent incorrect | Wrong action | Over-halt | Halt/error | Steady median | Steady p95 | End-to-end median | End-to-end p95 | Model calls | Cost | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `clean` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.189s | 7.258s | 7.774s | 7.842s | 0 | $0.00 | +| `clean` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.202s | 0.216s | 0.852s | 0.860s | 0 | $0.00 | +| `clean` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.201s | 0.205s | 0.837s | 0.844s | 0 | $0.00 | +| `theme` | compiled replay | 3 | 3/3 | 0 | 0 | 3 | 0 | 12.661s | 13.142s | 13.245s | 13.723s | 0 | $0.00 | +| `theme` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.200s | 0.203s | 0.839s | 0.861s | 0 | $0.00 | +| `theme` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.203s | 0.205s | 0.845s | 0.878s | 0 | $0.00 | +| `rename` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.114s | 7.152s | 7.757s | 7.917s | 0 | $0.00 | +| `rename` | DOM positional | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.072s | 30.074s | 30.715s | 30.766s | 0 | $0.00 | +| `rename` | DOM name-scoped | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.063s | 30.064s | 30.686s | 30.693s | 0 | $0.00 | + +Steady time wraps only the replay/script action loop. End-to-end time additionally includes fresh browser launch, the independent final screenshot/OCR oracle, and browser teardown. Local server startup and one-time record/compile are excluded from both and reported separately below. p95 is nearest-rank; with n=3 it is the slowest counted trial. + +Outcome definitions: **silent incorrect success** means the arm reported completion but the independent oracle did not confirm the intended effect; **wrong action** means the oracle observed a write to the wrong patient or encounter type; **over-halt** means the arm reported halt/incomplete while the independent oracle confirmed the intended effect; **halt/error** means the arm stopped and the intended effect was absent. The same definitions apply to every arm. The selector controls therefore can count as over-halts when alternate final-state evidence contradicts them; their rename failures did not, because the oracle confirmed no write. + +## One-time setup + +- Bundled app server startup: 0.004s (n=1 diagnostic) +- Setup trial 1: record 2.927s; compile 6.786s; bundle `32c086c327696a56a584c3946521d922f0111166635b2a13453c96032207c65a` +- Setup trial 2: record 2.410s; compile 6.273s; bundle `c050ebe60f547637cb0944f44680716c7f4c15e5e3773c191a839ce01feaf518` +- Setup trial 3: record 2.478s; compile 6.362s; bundle `c8efd66124ff47339a533aa53a037c11d3f7c942e615e6ca3ec9bb01881c01a0` + +## What this establishes + +- The independent final-state oracle confirmed the intended effect for every compiled clean, theme-drift, and label-drift trial. Compiled theme runs reported 3/3 over-halts; these are counted rather than relabelled as clean completions. +- The selector controls are steelmanned Playwright scripts. Their clean/theme speed is the correct reminder that API/structural actuation should remain the preferred tier where available. +- The `rename` surface changes `Open` to `View` and `Save Encounter` to `Submit Encounter`. Both selector controls failed loudly at the first renamed locator before mutation. These are unsupported-drift halts, not silent wrong actions. +- Label drift is an intentionally bounded robustness probe, not evidence for arbitrary drift or arbitrary applications. +- The compiled bundle had 1/8 identity-applicable clicks armed. Task completion is not a universal wrong-target-immunity claim for unarmed steps. +- The exact theme postcondition failures are retained in `theme_postcondition_over_halt.json` as a compact regression artifact. + +## Exact-current Flow versus zero-shot: not run + +No current Flow-versus-zero-shot result is claimed. The Azure WAA VM was not started and no model was called. The existing `scripts/eval_flow_on_waa.py` live replay path currently leaves the WAA evaluator unwired (so it cannot independently score success), while its hybrid live path explicitly returns before execution because the adapter is not wired. A valid future run therefore requires, in order: + +1. Wire `WAALiveAdapter.evaluate` into the Flow replay path and wire the same model/adapter into the zero-shot arm. +2. Prepare one exact compiled bundle per retained task and bind Flow/evals/model/environment revisions in the run manifest. +3. Obtain explicit approval for Azure VM start and a hard model spend cap; then confirm VM snapshot/readiness without changing the task set. +4. Run at least three trials per task per condition for both arms, using WAA's evaluator as the oracle and recording correct, silent incorrect, over-halt, timeout/error, latency, tokens, and cost. +5. Deallocate the VM, verify no orphaned resources, and publish the immutable raw summaries plus a normalized report. + +## Caveats + +- Synthetic MockMed, one workflow, one macOS host, headless Chromium. +- This report runs the exact published wheel named above, extracted locally, and binds it to the release-tagged tracked-clean source. +- Browser startup is fresh per run but OS/browser caches are warm after the first launch; the Latin-rotated arm order reduces but does not eliminate host-order effects. +- Final screenshots were inspected and hashed during the run; only their SHA-256 hashes are retained in this committed report. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_16_1_local_20260718/results.json b/docs/eval_results/current_flow_v1_16_1_local_20260718/results.json new file mode 100644 index 0000000..7285394 --- /dev/null +++ b/docs/eval_results/current_flow_v1_16_1_local_20260718/results.json @@ -0,0 +1,1135 @@ +{ + "aggregate": { + "compiled": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.584117708960548, + "end_to_end_wall_s_median": 7.773508124984801, + "end_to_end_wall_s_p95_nearest_rank": 7.842011750093661, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.189390416024253, + "steady_wall_s_p95_nearest_rank": 7.257713833008893, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6058565421262756, + "end_to_end_wall_s_median": 7.757476333063096, + "end_to_end_wall_s_p95_nearest_rank": 7.916532875038683, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.113519667065702, + "steady_wall_s_p95_nearest_rank": 7.15161979093682, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.5842226670356467, + "end_to_end_wall_s_median": 13.24499462498352, + "end_to_end_wall_s_p95_nearest_rank": 13.722865749965422, + "failure_taxonomy": { + "over_halt": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 3, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 12.660771957947873, + "steady_wall_s_p95_nearest_rank": 13.141616875072941, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6433688749093562, + "end_to_end_wall_s_median": 0.8520100830355659, + "end_to_end_wall_s_p95_nearest_rank": 0.8595500829396769, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20174725004471838, + "steady_wall_s_p95_nearest_rank": 0.2161812080303207, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6429007089463994, + "end_to_end_wall_s_median": 30.715099583962, + "end_to_end_wall_s_p95_nearest_rank": 30.766386165982112, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.0721988750156, + "steady_wall_s_p95_nearest_rank": 30.073791957926005, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.6365984989097342, + "end_to_end_wall_s_median": 0.8393889159196988, + "end_to_end_wall_s_p95_nearest_rank": 0.8611749580595642, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.19998587504960597, + "steady_wall_s_p95_nearest_rank": 0.20279041700996459, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom_named": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6392502910457551, + "end_to_end_wall_s_median": 0.8365530000301078, + "end_to_end_wall_s_p95_nearest_rank": 0.8438201659591869, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.20078483293764293, + "steady_wall_s_p95_nearest_rank": 0.20462062500882894, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6230927919968963, + "end_to_end_wall_s_median": 30.685788417002186, + "end_to_end_wall_s_p95_nearest_rank": 30.692634499981068, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.06269562500529, + "steady_wall_s_p95_nearest_rank": 30.064017542055808, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.6433472079224885, + "end_to_end_wall_s_median": 0.8453765829326585, + "end_to_end_wall_s_p95_nearest_rank": 0.8779241249430925, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.203157041920349, + "steady_wall_s_p95_nearest_rank": 0.20545862498693168, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + } + }, + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.7" + }, + "generated_at": "2026-07-19T02:04:44.059239+00:00", + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 1, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_006" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_007" + }, + { + "reason": "no readable text in the target's row band at compile time (icon-only or unlabeled row)", + "step_id": "step_008" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_010" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "runs": [ + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.842011750093661, + "error": null, + "final_screenshot_sha256": "d2ca9edc1e9b3717013879c6f01b5e70dbf6ea80d7290d3c2dc7c06b477b5b61", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.257713833008893, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8595500829396769, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "0c5858b1a9a87eb8ebfbc40843b5192cdcfcb1b8733d992b899cd8a55a8eba7d", + "input_tokens": 0, + "note_found": true, + "note_sha256": "487a06d2518e7cbd44c2bf909f5e1c75efb4a74677014fa6a397eb3199abdba0", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2161812080303207, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8342442080611363, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "bc5e0fc753d6f16af1fb802e16dcb2148c793b1128b363324dbece2d7899d24d", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0bab187ec8f858a6b7c76c7488cb233a0e63bd4b622c9ab5f0e13cd5e9545164", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20462062500882894, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8520100830355659, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "2802b2aedf82fdc7caafdd5e6af94cc1baee04ca082a11a88d073031fac7bdf6", + "input_tokens": 0, + "note_found": true, + "note_sha256": "fd9a7feca40d5477630dc008704a4cc90c73c8712333125a3fa0c75107e1def9", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19955804198980331, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8438201659591869, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "3419e0b0c6ba1890f91d168da6cb70adfa0d85d69a27fe58cbec038e6ef3914f", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0a79359c51a1562520f7cd213997e0e339a2cd46bba70365624dd0eda5545753", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20078483293764293, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.773508124984801, + "error": null, + "final_screenshot_sha256": "617465404eb865f4f0c3152f836cb90cd9f1cf7056ddaf261fe5214dd0a5a3e6", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.189390416024253, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8365530000301078, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "d1c051e1b448c5df5e9d86b4c82aa94642568adc150ac74f08e24c45cf90e92f", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0aaceaff9dd0e0f73480518bf40ef1833fff34cc82644c7aca4c7598430b5b0e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19730270898435265, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.57424120802898, + "error": null, + "final_screenshot_sha256": "ac5e195ab60a79a9a2d47a196dcc25a09a872d7f70c2ec3a98dc63ecf320bb43", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.99311325000599, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8095974999014288, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "2838fc4990800e090236881633e186059043b728c4c3e52703f5a9d11d9a4dcd", + "input_tokens": 0, + "note_found": true, + "note_sha256": "16bd942cfa1315c6494e526f3c06f4e240f6a41aa434725d3558e69c3c639152", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20174725004471838, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 13.24499462498352, + "error": null, + "final_screenshot_sha256": "0a5030645fdd40ab9b765f9dd02f3dc1d28a5e9dc35393f729a20c444a89a48d", + "first_failure": { + "error": "Postconditions failed for step 'step_010' (click 'Save Encounter'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 43, 968, 169) \u2014 run aborted", + "step": "step_010" + }, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "output_tokens": 0, + "over_halt": true, + "primary_outcome": "over_halt", + "replayer_success": false, + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 12.660771957947873, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.796460208017379, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "67314f98c40884738a5c7c07f640662b78d0c689d5141dd6b081dccdd0e2a57a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "c1612f0bac6a65e866e923f68a636bce28368cb3bcb77d9d6fdb690364511f66", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19994687498547137, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8239759999560192, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "3945d607e5f148461301fc9d8a485ed2ffd9cec33ea0e8f19aa344e0ea15ee2f", + "input_tokens": 0, + "note_found": true, + "note_sha256": "96c9f54ba668b83d716997b38595f7eb3ba6003edba85ae47efeb0fe36ba2ec2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.203157041920349, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8611749580595642, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "3aed780c017eb2e07ae806a3beb8df2ea9a5c6738803b6ce84a0fa45520a61df", + "input_tokens": 0, + "note_found": true, + "note_sha256": "3ac4161fc038822d09fc851a31c46ce6933b378df41173685e962b66083afd63", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.19998587504960597, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8453765829326585, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "5ac7b7f080bb195038e798ca65039da0f73fb413f3377b0845388287990d46fa", + "input_tokens": 0, + "note_found": true, + "note_sha256": "ba7e43d4c36aece2b8383ce9206a4cb77e63f9803ad80eceacf42d44161fc102", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20202937501017004, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 13.24434537498746, + "error": null, + "final_screenshot_sha256": "1c36ee628955e5a7ec06d847bef8ce3fce8c59a0fd2296c20a77bd932af9ce0b", + "first_failure": { + "error": "Postconditions failed for step 'step_010' (click 'Save Encounter'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 43, 968, 169) \u2014 run aborted", + "step": "step_010" + }, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "output_tokens": 0, + "over_halt": true, + "primary_outcome": "over_halt", + "replayer_success": false, + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 12.649677417008206, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8779241249430925, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e190449322e9bf8ed7b240a7145fe4ad94617f1f6484a38d836c64685a1e6085", + "input_tokens": 0, + "note_found": true, + "note_sha256": "79a353ca71e790724eb96a69724455151e8469bf24405e90565090dd80a3c465", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20545862498693168, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 13.722865749965422, + "error": null, + "final_screenshot_sha256": "5daea16d284fa53e764ad01caa187fd589711c2f35a4dc3cebb536a5b13c2f94", + "first_failure": { + "error": "Postconditions failed for step 'step_010' (click 'Save Encounter'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 43, 968, 169) \u2014 run aborted", + "step": "step_010" + }, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "output_tokens": 0, + "over_halt": true, + "primary_outcome": "over_halt", + "replayer_success": false, + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 13.141616875072941, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.8393889159196988, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "ba2f9f64a950c59dcfce1633eff30cbbcaf489822a9522b410355e5a3f4dee67", + "input_tokens": 0, + "note_found": true, + "note_sha256": "f3ca35760733ac421075853af9cc0d0e6c44fcaf6bbfbc37f10c63d2e56def1a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20279041700996459, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.916532875038683, + "error": null, + "final_screenshot_sha256": "11f980da0c184e3e70ae76630e17262d74d285ad209777f72fee8106ea26c7ca", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.113519667065702, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.766386165982112, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "29c4fef4f1bd398474762a8fe81c754db30e30fe883e16222de95d5c42004aee", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.073791957926005, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.668756291968748, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "1ab448b77f9ba6bcc9353ab49979cbbf59f3c143ca89b23f0c3acf446089d191", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.064017542055808, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.66424570896197, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "ac4a87fd1f10c165f46f0bd11ad9ac0566c8c3c5bbf143ee4a38e68fc8581f91", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.064888458000496, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.685788417002186, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "6b40530e788765ea3d93cbe9bcc5c6b4329da187a82172b6929a0c310206aa40", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.06269562500529, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.565182958031073, + "error": null, + "final_screenshot_sha256": "58ffe2f02e10f5bdf2894d4426461f29421e5b9e058abcbe2ea5a4170e1d63a7", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.988811000017449, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.692634499981068, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "db3899469ecf79973b1072c9b3c0acab3bd2c3be46995d2c6f509456b449ea8d", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.062342874938622, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.757476333063096, + "error": null, + "final_screenshot_sha256": "3dc19301940d2a771e9267bf2aede5193238b05a4e3531414720e59937fff63e", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.15161979093682, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.715099583962, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "f68010e8551fe0b53d63da9c2cfa5ec7412e85c700264aedcd369c18b4211856", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.0721988750156, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + } + ], + "schema_version": 1, + "scope": "exact-current local deterministic runtime overhead and robustness", + "setup": { + "record_compile_trials": [ + { + "bundle_tree_sha256": "32c086c327696a56a584c3946521d922f0111166635b2a13453c96032207c65a", + "compile_wall_s": 6.786271082935855, + "manifest_sha256": "7c485fb5abbc76f8b3cced90e4fa014b66e3380cd91ecbf585c5038a7f6b9a80", + "record_wall_s": 2.9271734170615673, + "trial": 1, + "workflow_sha256": "f8e9a52cf580e53cdf87d47c22c0992b92d493068d8e01c40ff56e6808923de0" + }, + { + "bundle_tree_sha256": "c050ebe60f547637cb0944f44680716c7f4c15e5e3773c191a839ce01feaf518", + "compile_wall_s": 6.272781000006944, + "manifest_sha256": "2ac797d841a69222bd13152e4e32feaea30fdc5d9f396420a70ac31cc8e04baf", + "record_wall_s": 2.4100100410869345, + "trial": 2, + "workflow_sha256": "70681377dee7f131445999196b86e9254ed1c577bcbe93c138f52b691fc0b364" + }, + { + "bundle_tree_sha256": "c8efd66124ff47339a533aa53a037c11d3f7c942e615e6ca3ec9bb01881c01a0", + "compile_wall_s": 6.361686500022188, + "manifest_sha256": "b8581add97a1e269a3afc03990d50b347f8ac02633b7dc754d2f9b5e53f9401d", + "record_wall_s": 2.4781212500529364, + "trial": 3, + "workflow_sha256": "a469d2ca98f85f04f487fa8cb17007179b370b59cc4857ce6db9176cff2249e9" + } + ], + "server_start_wall_s": 0.004492916050367057 + }, + "source": { + "evals": { + "commit": "7629ba5a2447c919e499e88e9b4eedbc80c8f3ab" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.16.1-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "c7073283475e7ae722db2478b499d962364851104e09edb71f254ba21c1310cd" + }, + "commit": "113ce992b491576d77236f495b983165ce7a63bd", + "release_tag": "v1.16.1", + "tags": [ + "v1.16.1" + ], + "tracked_clean": true, + "version": "1.16.1" + }, + "runner_sha256": "ac58c0b9a02cfc991144a1f5c7a2814c6b6b748bcba27b87f8e1027ee575970f" + }, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3, + "zero_shot_comparison": { + "reason": "requires paid model plus an available paid WAA environment; current live Flow replay also lacks a wired WAA evaluator", + "status": "not_run" + } +} diff --git a/docs/eval_results/current_flow_v1_16_1_local_20260718/theme_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_16_1_local_20260718/theme_postcondition_over_halt.json new file mode 100644 index 0000000..868534b --- /dev/null +++ b/docs/eval_results/current_flow_v1_16_1_local_20260718/theme_postcondition_over_halt.json @@ -0,0 +1,73 @@ +{ + "condition": { + "kind": "cosmetic theme drift", + "query": "?drift=theme" + }, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.16.1-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "c7073283475e7ae722db2478b499d962364851104e09edb71f254ba21c1310cd" + }, + "commit": "113ce992b491576d77236f495b983165ce7a63bd", + "release_tag": "v1.16.1", + "tags": [ + "v1.16.1" + ], + "tracked_clean": true, + "version": "1.16.1" + }, + "observations": [ + { + "end_to_end_wall_s": 13.24499462498352, + "final_screenshot_sha256": "0a5030645fdd40ab9b765f9dd02f3dc1d28a5e9dc35393f729a20c444a89a48d", + "first_failure": { + "error": "Postconditions failed for step 'step_010' (click 'Save Encounter'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 43, 968, 169) \u2014 run aborted", + "step": "step_010" + }, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "oracle_success": true, + "primary_outcome": "over_halt", + "replayer_success": false, + "steady_wall_s": 12.660771957947873, + "trial": 1 + }, + { + "end_to_end_wall_s": 13.24434537498746, + "final_screenshot_sha256": "1c36ee628955e5a7ec06d847bef8ce3fce8c59a0fd2296c20a77bd932af9ce0b", + "first_failure": { + "error": "Postconditions failed for step 'step_010' (click 'Save Encounter'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 43, 968, 169) \u2014 run aborted", + "step": "step_010" + }, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "oracle_success": true, + "primary_outcome": "over_halt", + "replayer_success": false, + "steady_wall_s": 12.649677417008206, + "trial": 2 + }, + { + "end_to_end_wall_s": 13.722865749965422, + "final_screenshot_sha256": "5daea16d284fa53e764ad01caa187fd589711c2f35a4dc3cebb536a5b13c2f94", + "first_failure": { + "error": "Postconditions failed for step 'step_010' (click 'Save Encounter'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(0, 43, 968, 169) \u2014 run aborted", + "step": "step_010" + }, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "oracle_success": true, + "primary_outcome": "over_halt", + "replayer_success": false, + "steady_wall_s": 13.141616875072941, + "trial": 3 + } + ], + "observed_count": 3, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ", + "required_trials": 3, + "runner_sha256": "ac58c0b9a02cfc991144a1f5c7a2814c6b6b748bcba27b87f8e1027ee575970f", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "theme effect succeeds but region_stable reports halt" +} diff --git a/scripts/run_current_flow_local_benchmark.py b/scripts/run_current_flow_local_benchmark.py new file mode 100644 index 0000000..f00d466 --- /dev/null +++ b/scripts/run_current_flow_local_benchmark.py @@ -0,0 +1,655 @@ +#!/usr/bin/env python3 +"""Run a source-bound, local-only benchmark of current compiled Flow. + +The benchmark intentionally avoids paid models and provider infrastructure. It +records and compiles the bundled synthetic MockMed workflow, then compares: + +* OpenAdapt compiled replay; +* a positional Playwright selector script; and +* a name-scoped Playwright selector script. + +Every arm gets a fresh browser and is judged by the same screenshot/OCR oracle: +the expected saved encounter, exact note, and intended patient must be visible, +and a wrong-type or wrong-patient write is a wrong action. Neither the +replayer nor either script can self-score success. + +The default matrix has three trials per arm in each of three conditions: +clean, cosmetic theme drift, and user-facing label drift. Healthy runs make no +model calls and no network/provider calls beyond loopback to the bundled app. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import math +import platform +import statistics +import subprocess +import sys +import tempfile +import time +import zipfile +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ARMS = ("compiled", "dom", "dom_named") +CONDITIONS = ("clean", "theme", "rename") +ARM_LABELS = { + "compiled": "compiled replay", + "dom": "DOM positional", + "dom_named": "DOM name-scoped", +} + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _tree_sha256(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + payload = path.read_bytes() + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + return digest.hexdigest() + + +def _git(root: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(root), *args], + check=True, + text=True, + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return result.stdout.strip() + + +def _source_binding(root: Path) -> dict[str, Any]: + commit = _git(root, "rev-parse", "HEAD") + status = _git(root, "status", "--porcelain", "--untracked-files=no") + if status: + raise RuntimeError(f"Flow source must be tracked-clean: {root}") + tags = sorted(tag for tag in _git(root, "tag", "--points-at", "HEAD").splitlines() if tag) + return { + "commit": commit, + "tags": tags, + "tracked_clean": True, + } + + +def _extract_wheel(wheel: Path, destination: Path) -> None: + with zipfile.ZipFile(wheel) as archive: + for member in archive.infolist(): + path = Path(member.filename) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"unsafe wheel member: {member.filename}") + archive.extractall(destination) + + +def _nearest_rank(values: list[float], quantile: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + return ordered[max(0, math.ceil(quantile * len(ordered)) - 1)] + + +def _reported_complete(row: dict[str, Any]) -> bool: + if row["arm"] == "compiled": + return row.get("replayer_success") is True + return ( + row.get("error") is None + and row.get("failed_step") is None + and row.get("actions") == row.get("steps_total") + ) + + +def _classify(row: dict[str, Any]) -> str: + oracle_success = row.get("success") is True + reported_complete = _reported_complete(row) + if oracle_success and reported_complete: + return "correct" + if oracle_success: + return "over_halt" + if reported_complete: + return "silent_incorrect_success" + if row.get("wrong_action"): + return "wrong_action_after_halt_or_error" + return "halt_or_error" + + +def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: + steady = [float(row["steady_wall_s"]) for row in rows] + end_to_end = [float(row["end_to_end_wall_s"]) for row in rows] + taxonomy = Counter(row["primary_outcome"] for row in rows) + return { + "n": len(rows), + "task_success_count": sum(row.get("success") is True for row in rows), + "task_success_rate": ( + sum(row.get("success") is True for row in rows) / len(rows) if rows else 0.0 + ), + "reported_complete_count": sum(_reported_complete(row) for row in rows), + "silent_incorrect_success_count": taxonomy["silent_incorrect_success"], + "wrong_action_count": sum(bool(row.get("wrong_action")) for row in rows), + "over_halt_count": taxonomy["over_halt"], + "halt_or_error_count": taxonomy["halt_or_error"], + "failure_taxonomy": dict(sorted(taxonomy.items())), + "steady_wall_s_median": statistics.median(steady) if steady else 0.0, + "steady_wall_s_p95_nearest_rank": _nearest_rank(steady, 0.95), + "end_to_end_wall_s_median": (statistics.median(end_to_end) if end_to_end else 0.0), + "end_to_end_wall_s_p95_nearest_rank": _nearest_rank(end_to_end, 0.95), + "browser_oracle_teardown_overhead_s_median": ( + statistics.median(row["end_to_end_wall_s"] - row["steady_wall_s"] for row in rows) + if rows + else 0.0 + ), + "model_calls_total": sum(int(row.get("api_calls") or 0) for row in rows), + "model_cost_usd_total": round(sum(float(row.get("cost_usd") or 0.0) for row in rows), 8), + } + + +def _render_markdown(report: dict[str, Any]) -> str: + lines = [ + "# Exact-current local Flow performance", + "", + ( + "This is a deterministic-runtime overhead and bounded robustness " + "comparison. It is **not** a zero-shot computer-use comparison." + ), + "", + "## Source and environment", + "", + f"- Flow commit: `{report['source']['flow']['commit']}` " + f"(version `{report['source']['flow']['version']}`; tracked-clean source)", + f"- Release tag: `{report['source']['flow']['release_tag']}`", + f"- Wheel SHA-256: `{report['source']['flow']['artifact']['sha256']}`", + f"- Evals base commit: `{report['source']['evals']['commit']}`", + f"- Runner SHA-256: `{report['source']['runner_sha256']}`", + f"- Platform: `{report['environment']['platform']}`", + f"- Python: `{report['environment']['python']}`", + f"- Playwright: `{report['environment']['playwright']}`", + f"- Chromium: `{report['environment']['chromium']}`", + "- Network/provider use: loopback bundled MockMed only; no cloud VM, " + "hosted runner, or model API", + "", + "## Counted result", + "", + ( + "One synthetic MockMed workflow: sign in, open the intended " + "referral, create a Triage encounter, enter a trial-unique note, " + "and save. Each arm used a fresh browser. The arm-independent " + "screenshot/OCR oracle required the exact saved note, Triage row, " + "and intended patient, and separately flagged wrong-target writes." + ), + "", + ( + "| Condition | Arm | Runs | Task success | Silent incorrect | " + "Wrong action | Over-halt | Halt/error | Steady median | " + "Steady p95 | End-to-end median | End-to-end p95 | Model calls | Cost |" + ), + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for condition in report["conditions"]: + for arm in report["arms"]: + cell = report["aggregate"][arm][condition] + lines.append( + f"| `{condition}` | {ARM_LABELS[arm]} | {cell['n']} | " + f"{cell['task_success_count']}/{cell['n']} | " + f"{cell['silent_incorrect_success_count']} | " + f"{cell['wrong_action_count']} | {cell['over_halt_count']} | " + f"{cell['halt_or_error_count']} | " + f"{cell['steady_wall_s_median']:.3f}s | " + f"{cell['steady_wall_s_p95_nearest_rank']:.3f}s | " + f"{cell['end_to_end_wall_s_median']:.3f}s | " + f"{cell['end_to_end_wall_s_p95_nearest_rank']:.3f}s | " + f"{cell['model_calls_total']} | " + f"${cell['model_cost_usd_total']:.2f} |" + ) + lines.extend( + [ + "", + "Steady time wraps only the replay/script action loop. End-to-end " + "time additionally includes fresh browser launch, the independent " + "final screenshot/OCR oracle, and browser teardown. Local server " + "startup and one-time record/compile are excluded from both and " + "reported separately below. p95 is nearest-rank; with n=3 it is " + "the slowest counted trial.", + "", + "Outcome definitions: **silent incorrect success** means the arm " + "reported completion but the independent oracle did not confirm " + "the intended effect; **wrong action** means the oracle observed a " + "write to the wrong patient or encounter type; **over-halt** means " + "the arm reported halt/incomplete while the independent oracle " + "confirmed the intended effect; **halt/error** means the arm " + "stopped and the intended effect was absent. The same definitions " + "apply to every arm. The selector controls therefore can count as " + "over-halts when alternate final-state evidence contradicts them; " + "their rename failures did not, because the oracle confirmed no " + "write.", + "", + "## One-time setup", + "", + ( + f"- Bundled app server startup: " + f"{report['setup']['server_start_wall_s']:.3f}s (n=1 diagnostic)" + ), + ] + ) + for setup in report["setup"]["record_compile_trials"]: + lines.append( + f"- Setup trial {setup['trial']}: record " + f"{setup['record_wall_s']:.3f}s; compile " + f"{setup['compile_wall_s']:.3f}s; bundle " + f"`{setup['bundle_tree_sha256']}`" + ) + identity = report["identity_coverage"] + lines.extend( + [ + "", + "## What this establishes", + "", + "- The independent final-state oracle confirmed the intended effect " + "for every compiled clean, theme-drift, and label-drift trial. " + f"Compiled theme runs reported " + f"{report['aggregate']['compiled']['theme']['over_halt_count']}/" + f"{report['aggregate']['compiled']['theme']['n']} over-halts; these " + "are counted rather than relabelled as clean completions.", + "- The selector controls are steelmanned Playwright scripts. Their " + "clean/theme speed is the correct reminder that API/structural " + "actuation should remain the preferred tier where available.", + "- The `rename` surface changes `Open` to `View` and `Save " + "Encounter` to `Submit Encounter`. Both selector controls failed " + "loudly at the first renamed locator before mutation. These are " + "unsupported-drift halts, not silent wrong actions.", + "- Label drift is an intentionally bounded robustness probe, not " + "evidence for arbitrary drift or arbitrary applications.", + f"- The compiled bundle had " + f"{identity['armed_steps']}/" + f"{identity['applicable_steps']} " + "identity-applicable clicks armed. Task completion is not a " + "universal wrong-target-immunity claim for unarmed steps.", + "- The exact theme postcondition failures are retained in " + "`theme_postcondition_over_halt.json` as a compact regression artifact.", + "", + "## Exact-current Flow versus zero-shot: not run", + "", + "No current Flow-versus-zero-shot result is claimed. The Azure WAA " + "VM was not started and no model was called. The existing " + "`scripts/eval_flow_on_waa.py` live replay path currently leaves " + "the WAA evaluator unwired (so it cannot independently score " + "success), while its hybrid live path explicitly returns before " + "execution because the adapter is not wired. A valid future run " + "therefore requires, in order:", + "", + "1. Wire `WAALiveAdapter.evaluate` into the Flow replay path and " + "wire the same model/adapter into the zero-shot arm.", + "2. Prepare one exact compiled bundle per retained task and bind " + "Flow/evals/model/environment revisions in the run manifest.", + "3. Obtain explicit approval for Azure VM start and a hard model " + "spend cap; then confirm VM snapshot/readiness without changing the " + "task set.", + "4. Run at least three trials per task per condition for both arms, " + "using WAA's evaluator as the oracle and recording correct, silent " + "incorrect, over-halt, timeout/error, latency, tokens, and cost.", + "5. Deallocate the VM, verify no orphaned resources, and publish " + "the immutable raw summaries plus a normalized report.", + "", + "## Caveats", + "", + "- Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "- This report runs the exact published wheel named above, extracted " + "locally, and binds it to the release-tagged tracked-clean source.", + "- Browser startup is fresh per run but OS/browser caches are warm " + "after the first launch; the Latin-rotated arm order reduces but " + "does not eliminate host-order effects.", + "- Final screenshots were inspected and hashed during the run; " + "only their SHA-256 hashes are retained in this committed report.", + "- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer " + "application is represented.", + "", + ] + ) + return "\n".join(lines) + + +def run_benchmark( + flow_source: Path, + flow_wheel: Path, + out_dir: Path, + *, + trials: int = 3, + setup_trials: int = 3, + headed: bool = False, +) -> dict[str, Any]: + if trials < 3: + raise ValueError("comparative evaluation requires at least three trials") + if setup_trials < 1: + raise ValueError("setup_trials must be positive") + flow_source = flow_source.resolve() + flow_wheel = flow_wheel.resolve() + out_dir = out_dir.resolve() + if not (flow_source / "openadapt_flow").is_dir(): + raise ValueError(f"not an openadapt-flow source checkout: {flow_source}") + if not flow_wheel.is_file() or flow_wheel.suffix != ".whl": + raise ValueError(f"not a Flow wheel: {flow_wheel}") + + source = _source_binding(flow_source) + artifact_temp = tempfile.TemporaryDirectory(prefix="oa-flow-wheel-") + artifact_root = Path(artifact_temp.name).resolve() + _extract_wheel(flow_wheel, artifact_root) + sys.path.insert(0, str(artifact_root)) + + from openadapt_flow.benchmark.dom_arm import ( # noqa: PLC0415 + TARGET_PATIENT, + condition_url, + dom_run, + note_for_slot, + verify_final_state, + ) + from openadapt_flow.benchmark.run_benchmark import ( # noqa: PLC0415 + _compiled_run, + ) + from openadapt_flow.compiler import compile_recording # noqa: PLC0415 + from openadapt_flow.demo_driver import record_triage_demo # noqa: PLC0415 + from openadapt_flow.mockmed.server import serve # noqa: PLC0415 + + imported = Path(sys.modules["openadapt_flow"].__file__).resolve() + if artifact_root not in imported.parents: + raise RuntimeError(f"Flow import escaped requested wheel: {imported}") + flow_version = str(sys.modules["openadapt_flow"].__version__) + release_tag = f"v{flow_version}" + if release_tag not in source["tags"]: + raise RuntimeError(f"source commit is not tagged {release_tag}: {source['tags']}") + + evals_root = Path(__file__).resolve().parents[1] + evals_commit = _git(evals_root, "rev-parse", "HEAD") + out_dir.mkdir(parents=True, exist_ok=False) + finals_temp = tempfile.TemporaryDirectory(prefix="oa-current-flow-finals-") + finals = Path(finals_temp.name) + + server_started = time.monotonic() + url, stop = serve(port=0) + server_start_wall_s = time.monotonic() - server_started + rows: list[dict[str, Any]] = [] + setups: list[dict[str, Any]] = [] + try: + with tempfile.TemporaryDirectory(prefix="oa-current-flow-local-") as tmp: + root = Path(tmp) + bundles: list[Path] = [] + for setup_trial in range(1, setup_trials + 1): + setup_root = root / f"setup-{setup_trial}" + recording_started = time.monotonic() + recording = record_triage_demo( + url, + setup_root / "recording", + note_text=f"Exact-current setup {setup_trial}", + headed=headed, + ) + record_wall_s = time.monotonic() - recording_started + bundle = setup_root / "bundle" + compile_started = time.monotonic() + compile_recording( + recording, + bundle, + name=f"current-flow-local-setup-{setup_trial}", + ) + compile_wall_s = time.monotonic() - compile_started + bundles.append(bundle) + setups.append( + { + "trial": setup_trial, + "record_wall_s": record_wall_s, + "compile_wall_s": compile_wall_s, + "bundle_tree_sha256": _tree_sha256(bundle), + "workflow_sha256": _sha256(bundle / "workflow.json"), + "manifest_sha256": _sha256(bundle / "manifest.json"), + } + ) + bundle = bundles[-1] + + # Latin-rotate arm order across trials to limit warm-cache/order bias. + for condition_index, condition in enumerate(CONDITIONS): + for trial in range(1, trials + 1): + slot = condition_index * trials + (trial - 1) + rotation = (trial - 1) % len(ARMS) + arm_order = ARMS[rotation:] + ARMS[:rotation] + for arm in arm_order: + note = note_for_slot(arm, slot) + target = condition_url(url, condition) + final_path = finals / f"{condition}-{trial}-{arm}.png" + run_dir = root / "runs" / f"{condition}-{trial}-{arm}" + started = time.monotonic() + try: + if arm == "compiled": + row = _compiled_run( + bundle, + target, + run_dir, + note, + verify_fn=verify_final_state, + save_final_to=final_path, + headed=headed, + ) + else: + row = dom_run( + target, + note, + target_patient=(TARGET_PATIENT if arm == "dom_named" else None), + verify_fn=verify_final_state, + save_final_to=final_path, + headed=headed, + ) + except Exception as exc: # a failed run is evidence + row = { + "arm": arm, + "wall_s": 0.0, + "success": False, + "actions": 0, + "api_calls": 0, + "cost_usd": 0.0, + "error": f"{type(exc).__name__}: {exc}", + "wrong_action": False, + } + end_to_end = time.monotonic() - started + row["condition"] = condition + row["trial"] = trial + row["note_sha256"] = hashlib.sha256(note.encode("utf-8")).hexdigest() + row["steady_wall_s"] = float(row.pop("wall_s", 0.0)) + row["end_to_end_wall_s"] = end_to_end + row["reported_complete"] = _reported_complete(row) + row["primary_outcome"] = _classify(row) + row["silent_incorrect_success"] = ( + row["primary_outcome"] == "silent_incorrect_success" + ) + row["over_halt"] = row["primary_outcome"] == "over_halt" + row["final_screenshot_sha256"] = ( + _sha256(final_path) if final_path.is_file() else None + ) + rows.append(row) + print( + f"{condition} trial={trial} arm={arm}: " + f"{row['primary_outcome']} steady=" + f"{row['steady_wall_s']:.3f}s end_to_end=" + f"{end_to_end:.3f}s", + flush=True, + ) + finally: + stop() + + aggregate = { + arm: { + condition: _aggregate( + [row for row in rows if row["arm"] == arm and row["condition"] == condition] + ) + for condition in CONDITIONS + } + for arm in ARMS + } + compiled_example = next(row for row in rows if row["arm"] == "compiled") + identity_coverage = { + "applicable_steps": int(compiled_example.get("identity_applicable_steps") or 0), + "armed_steps": int(compiled_example.get("identity_armed_steps") or 0), + "unarmed": compiled_example.get("identity_unarmed") or [], + } + for row in rows: + row.pop("identity_applicable_steps", None) + row.pop("identity_armed_steps", None) + row.pop("identity_unarmed", None) + report = { + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "scope": "exact-current local deterministic runtime overhead and robustness", + "task": ( + "MockMed triage: sign in, open intended referral, create Triage " + "encounter, enter a trial-unique note, save" + ), + "oracle": ( + "arm-independent screenshot/OCR final-state check: exact note and " + "saved banner, Triage row, intended patient, no wrong-type write" + ), + "outcome_definitions": { + "correct": ("arm reported completion and independent oracle confirmed effect"), + "silent_incorrect_success": ( + "arm reported completion but independent oracle did not confirm effect" + ), + "wrong_action": ("independent oracle observed a wrong-patient or wrong-type write"), + "over_halt": ("arm reported halt/incomplete but independent oracle confirmed effect"), + "halt_or_error": ( + "arm reported halt/incomplete and independent oracle found effect absent" + ), + }, + "trials_per_arm_condition": trials, + "arms": list(ARMS), + "conditions": list(CONDITIONS), + "source": { + "flow": { + **source, + "version": flow_version, + "release_tag": release_tag, + "artifact": { + "filename": flow_wheel.name, + "sha256": _sha256(flow_wheel), + "import_mode": "locally extracted published wheel", + }, + }, + "evals": {"commit": evals_commit}, + "runner_sha256": _sha256(Path(__file__).resolve()), + }, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "playwright": importlib.metadata.version("playwright"), + "chromium": "Playwright-managed headless Chromium", + }, + "setup": { + "server_start_wall_s": server_start_wall_s, + "record_compile_trials": setups, + }, + "identity_coverage": identity_coverage, + "aggregate": aggregate, + "runs": rows, + "paid_or_remote_mutations": [], + "zero_shot_comparison": { + "status": "not_run", + "reason": ( + "requires paid model plus an available paid WAA environment; " + "current live Flow replay also lacks a wired WAA evaluator" + ), + }, + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented.", + ], + } + json_path = out_dir / "results.json" + json_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + (out_dir / "REPORT.md").write_text(_render_markdown(report), encoding="utf-8") + theme_over_halts = [ + { + "trial": row["trial"], + "replayer_success": row.get("replayer_success"), + "oracle_success": row.get("success"), + "primary_outcome": row["primary_outcome"], + "first_failure": row.get("first_failure"), + "steady_wall_s": row["steady_wall_s"], + "end_to_end_wall_s": row["end_to_end_wall_s"], + "note_sha256": row["note_sha256"], + "final_screenshot_sha256": row["final_screenshot_sha256"], + } + for row in rows + if row["arm"] == "compiled" + and row["condition"] == "theme" + and row["primary_outcome"] == "over_halt" + ] + regression = { + "schema_version": 1, + "title": "theme effect succeeds but region_stable reports halt", + "flow": report["source"]["flow"], + "runner_sha256": report["source"]["runner_sha256"], + "task": report["task"], + "condition": { + "query": "?drift=theme", + "kind": "cosmetic theme drift", + }, + "oracle": report["oracle"], + "expected": ( + "if the independently verified effect succeeded, the runtime must " + "not leave the run in an unresumable false-incomplete state" + ), + "observed_count": len(theme_over_halts), + "required_trials": trials, + "observations": theme_over_halts, + "reproduce": ( + "python scripts/run_current_flow_local_benchmark.py " + "--flow-source " + "--flow-wheel " + "--out " + ), + } + (out_dir / "theme_postcondition_over_halt.json").write_text( + json.dumps(regression, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + finals_temp.cleanup() + artifact_temp.cleanup() + print(f"Wrote {json_path} and {out_dir / 'REPORT.md'}", flush=True) + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Run exact-current local Flow versus selector controls." + ) + parser.add_argument("--flow-source", type=Path, required=True) + parser.add_argument("--flow-wheel", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--setup-trials", type=int, default=3) + parser.add_argument("--headed", action="store_true") + args = parser.parse_args(argv) + run_benchmark( + args.flow_source, + args.flow_wheel, + args.out, + trials=args.trials, + setup_trials=args.setup_trials, + headed=args.headed, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_current_flow_local_benchmark.py b/tests/test_current_flow_local_benchmark.py new file mode 100644 index 0000000..c23049c --- /dev/null +++ b/tests/test_current_flow_local_benchmark.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import importlib.util +import zipfile +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "run_current_flow_local_benchmark.py" +SPEC = importlib.util.spec_from_file_location("current_flow_local_benchmark", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def _row( + *, + outcome: str, + steady: float, + end_to_end: float, + success: bool, + reported: bool, +) -> dict: + return { + "arm": "compiled", + "primary_outcome": outcome, + "steady_wall_s": steady, + "end_to_end_wall_s": end_to_end, + "success": success, + "replayer_success": reported, + "wrong_action": False, + "api_calls": 0, + "cost_usd": 0.0, + } + + +def test_aggregate_reports_nearest_rank_and_failure_taxonomy() -> None: + rows = [ + _row( + outcome="correct", + steady=1.0, + end_to_end=2.0, + success=True, + reported=True, + ), + _row( + outcome="silent_incorrect_success", + steady=2.0, + end_to_end=3.5, + success=False, + reported=True, + ), + _row( + outcome="halt_or_error", + steady=3.0, + end_to_end=5.0, + success=False, + reported=False, + ), + ] + + result = MODULE._aggregate(rows) + + assert result["n"] == 3 + assert result["task_success_count"] == 1 + assert result["silent_incorrect_success_count"] == 1 + assert result["halt_or_error_count"] == 1 + assert result["steady_wall_s_median"] == 2.0 + assert result["steady_wall_s_p95_nearest_rank"] == 3.0 + assert result["end_to_end_wall_s_p95_nearest_rank"] == 5.0 + assert result["model_calls_total"] == 0 + assert result["model_cost_usd_total"] == 0.0 + + +def test_classification_never_trusts_actor_completion_as_oracle() -> None: + assert ( + MODULE._classify({"arm": "compiled", "success": False, "replayer_success": True}) + == "silent_incorrect_success" + ) + assert ( + MODULE._classify({"arm": "compiled", "success": True, "replayer_success": False}) + == "over_halt" + ) + + +def test_rejects_less_than_three_counted_trials(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="at least three"): + MODULE.run_benchmark( + tmp_path, + tmp_path / "flow.whl", + tmp_path / "out", + trials=2, + ) + + +def test_wheel_extraction_rejects_path_traversal(tmp_path: Path) -> None: + wheel = tmp_path / "unsafe.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("../escaped.py", "bad") + + with pytest.raises(ValueError, match="unsafe wheel member"): + MODULE._extract_wheel(wheel, tmp_path / "extract")