diff --git a/Package.swift b/Package.swift index 95cc0807..503e7621 100644 --- a/Package.swift +++ b/Package.swift @@ -8,10 +8,14 @@ let package = Package( name: "Oliphaunt", platforms: [ .iOS(.v17), - .macOS(.v14) + .macOS(.v14), ], products: [ - .library(name: "Oliphaunt", targets: ["Oliphaunt"]) + .library(name: "OliphauntBrokerProtocol", targets: ["OliphauntBrokerProtocol"]), + .library(name: "OliphauntBrokerXPC", targets: ["OliphauntBrokerXPC"]), + .library(name: "OliphauntIOSBroker", targets: ["OliphauntIOSBroker"]), + .library(name: "OliphauntBrokerExtension", targets: ["OliphauntBrokerExtension"]), + .library(name: "Oliphaunt", targets: ["Oliphaunt"]), ], targets: [ .target( @@ -19,15 +23,63 @@ let package = Package( path: "src/sdks/swift/Sources/COliphaunt", publicHeadersPath: "include" ), + .target( + name: "OliphauntBrokerProtocol", + path: "src/sdks/swift/Sources/OliphauntBrokerProtocol" + ), + .target( + name: "OliphauntBrokerXPC", + dependencies: ["OliphauntBrokerProtocol"], + path: "src/sdks/swift/Sources/OliphauntBrokerXPC" + ), .target( name: "Oliphaunt", dependencies: ["COliphaunt"], path: "src/sdks/swift/Sources/Oliphaunt" ), + .target( + name: "OliphauntIOSBroker", + dependencies: ["Oliphaunt", "OliphauntBrokerProtocol", "OliphauntBrokerXPC"], + path: "src/sdks/swift/Sources/OliphauntIOSBroker" + ), + .target( + name: "OliphauntBrokerExtension", + dependencies: ["COliphaunt", "Oliphaunt", "OliphauntBrokerProtocol"], + path: "src/sdks/swift/Sources/OliphauntBrokerExtension" + ), .testTarget( name: "OliphauntTests", dependencies: ["Oliphaunt"], path: "src/sdks/swift/Tests/OliphauntTests" - ) + ), + .testTarget( + name: "OliphauntBrokerProtocolTests", + dependencies: ["OliphauntBrokerProtocol"], + path: "src/sdks/swift/Tests/OliphauntBrokerProtocolTests" + ), + .testTarget( + name: "OliphauntBrokerXPCTests", + dependencies: ["OliphauntBrokerProtocol", "OliphauntBrokerXPC"], + path: "src/sdks/swift/Tests/OliphauntBrokerXPCTests" + ), + .testTarget( + name: "OliphauntBrokerExtensionTests", + dependencies: [ + "Oliphaunt", + "OliphauntBrokerExtension", + "OliphauntBrokerProtocol", + ], + path: "src/sdks/swift/Tests/OliphauntBrokerExtensionTests" + ), + .testTarget( + name: "OliphauntIOSBrokerTests", + dependencies: [ + "Oliphaunt", + "OliphauntBrokerProtocol", + "OliphauntBrokerXPC", + "OliphauntIOSBroker", + ], + path: "src/sdks/swift/Tests/OliphauntIOSBrokerTests" + ), ] ) diff --git a/docs/architecture/android-native-broker-spike.md b/docs/architecture/android-native-broker-spike.md new file mode 100644 index 00000000..352fc9d7 --- /dev/null +++ b/docs/architecture/android-native-broker-spike.md @@ -0,0 +1,359 @@ +# Android native broker feasibility spike + +This document records the contract and retained evidence for the DEBUG-only +Android native broker spike. It mirrors the iOS broker's essential failure +semantics using public Android APIs, but it is not an SDK design or a shipping +qualification. + +The authoritative repeated-behavior evidence is the ten-run series: + +```text +target/android-native-broker-spike/runs/pr-final-01-20260811T121603Z/ +through +target/android-native-broker-spike/runs/pr-final-10-20260811T121854Z/ +``` + +The post-document confirmation at +`target/android-native-broker-spike/runs/pr-doc-sync-v1/` also passed and its +source manifest matches the final spike files, including the README. The ten +repeated runs used the same APK and executable inputs; their only source-file +difference is the README's evidence-path text. + +Every `android-broker-report.json` has schema +`oliphaunt-android-native-broker-spike-v1`, strategy `full`, and `status=PASS`. +The series ran on one Android 14 / API 34 arm64 emulator, +`Pixel_9_API_34_Google_API` (`emulator-5554`, `arm64-v8a`). Every report sets +`environment=android-emulator` and `physicalDeviceEvidence=false`. Nothing in +this document should be read as physical-device evidence. + +The ten cold-host runs exercised 30 injected worker deaths and 40 worker +generations. All 10 host PIDs, all 40 worker PIDs, and all 40 worker UUID epochs +were unique in the retained series. Each run also repeated cancellation, +ambiguous-commit/no-replay, persistence, and controlled 8 MiB and 32 MiB +slow-reader probes. The runner independently checked source and native-library +currentness, APK and native hashes, process identity, generation-scoped Binder +death, retained SIGABRT log records for each faulted worker, and the exact +machine-readable report contract. + +Evidence terms used below: + +- **Emulator-proven in this series** means all ten exact-current runs and the + independent runner assertions passed. +- **Implemented** means the behavior exists in source but the series did not + independently qualify every broader condition it might imply. +- **Unproven** means the series does not establish the behavior, even if the + architecture intends to support it. + +## 1. Process and transport architecture + +```mermaid +flowchart LR + subgraph Host["Host application process"] + Experiment["BrokerExperiment"] + Client["BrokerClient"] + HostFD["Reliable socket endpoint"] + Experiment --> Client --> HostFD + end + + subgraph Broker["Private :broker service process"] + Binder["BrokerService AIDL stub"] + Socket["BrokerSocketEndpoint"] + Executor["Single database executor"] + Watchdog["Independent scheduled watchdog"] + Native["AndroidNativeDirectEngine / liboliphaunt"] + PGDATA["Persistent default PGDATA"] + Binder --> Socket --> Executor --> Native --> PGDATA + Binder -->|"cancel outside executor"| Native + Watchdog -->|"SIGABRT fail-stop"| Broker + end + + Client <-->|"AIDL: hello and control"| Binder + HostFD <-->|"40-byte OLPB frames carrying PostgreSQL bytes"| Socket +``` + +The manifest declares an unexported service in `android:process=":broker"`. +This gives the database a different Linux process and failure domain from the +host. It does **not** use Android's `isolatedProcess=true`: host and broker keep +the same app UID and private-storage access. The boundary is crash isolation, +not privilege or adversarial-code isolation. + +The host creates `ParcelFileDescriptor.createReliableSocketPair()`, retains one +endpoint, and transfers the other in the `hello` Binder call. Binder carries +handshake, cancellation, diagnostics, and DEBUG fault controls. PostgreSQL +frontend and backend bytes never travel in Binder transactions. + +## 2. Minimal Binder control plane + +The AIDL surface has only two methods: + +```aidl +Bundle hello(in Bundle request, in ParcelFileDescriptor dataChannel); +Bundle control(in Bundle request); +``` + +`hello` negotiates OLPB protocol v1, native ABI 6, logical root `default`, the +startup-configuration digest, and capabilities. A successful `ready` response +includes the worker PID and a UUID process epoch generated by that service +instance. The client links a `DeathRecipient` to the Binder before completing +the handshake and admits the generation only after socket ping/pong succeeds. + +`control` supports: + +| Message | Purpose | +| --- | --- | +| `cancel` | Cancel a request by ID directly from a Binder thread, outside the database executor. | +| `diagnostics` | Return epoch, PID, state, active request, native-dispatch flag, native PostgreSQL-output witness marker, PSS/RSS, requested socket send-buffer size, public descriptor blocking/readiness probes, and lock-free active/completed socket-write counters. | +| `injectFault` | Arm one DEBUG-only deliberate fault. | +| `detach` | Close the current experimental session. | + +The fault values are `executorDeadlockWithFailStop`, +`nativeFailStopWatchdog`, and `afterNativeSuccessBeforeCompleted`. These are +test mechanisms, not proposed production API. + +## 3. Data plane and request outcome + +The socket protocol uses a fixed 40-byte OLPB header: `OLPB` magic, protocol +version, header length, frame type, zero flags/reserved fields, 16-byte epoch, +64-bit request ID, and 32-bit payload length. A frame payload is at most 256 KiB +and aggregate queued/request data is capped at 8 MiB. + +Request frames are `requestBegin`, `requestBytes`, and `requestEnd`. Response +frames include `responseBytes`, `completed`, `rejected`, and `outcomeUnknown`. +Generation-level frames include ping/pong, protocol error, and channel close. +The epoch in every frame prevents bytes from a stale generation being accepted +after reconnection. + +The critical host rule is: + +> Once any request bytes may have reached the broker, transport or process loss +> terminates that request as `outcomeUnknown`. The host never replays the SQL. + +Connection establishment may retry. A dispatched SQL request may not. Binder +death, `ServiceConnection` loss, and reliable-socket EOF converge on one client +interruption path. The next operation lazily binds again and must see a fresh +epoch and PID plus a successful ping/pong before accepting the replacement. +The epoch is the durable generation identity; a PID alone is insufficient +because operating systems can reuse PIDs. + +## 4. Cancellation and fail-stop mechanism + +Cancellation intentionally bypasses the single database executor. In the +retained fixture, `SELECT pg_sleep(60)` reached native dispatch, Binder cancel +was acknowledged, PostgreSQL returned SQLSTATE `57014`, and the same worker PID +and epoch then served healthy SQL. The retained logcat recorded a 5 ms cancel +control acknowledgement. That number measures control acknowledgement in this +one run, not end-to-end cancellation latency or a service-level objective. + +For an executor deadlock, the fixture blocks the database executor on a latch +and arms a scheduled watchdog outside it. The native-hang request is one +ordered query: + +```sql +SELECT repeat('w', 8192) AS witness FROM generate_series(1, 513) +UNION ALL +SELECT ''::text FROM pg_sleep(60) AS blocker(ignored) +``` + +Its PostgreSQL plan is an `Append` whose first child is the `generate_series` +function scan and whose second child is the `pg_sleep` function scan. The +worker does not arm its watchdog before native dispatch. It counts bytes from +the native stream callback without buffering them. Once the callback has +emitted strictly more than 4 MiB, the worker first finishes the +threshold-crossing socket write. It then atomically publishes its request ID, +byte count, and monotonic timestamp in diagnostics and arms a two-second +watchdog. The callback then returns so execution can advance to the second +Append child. A transport stall therefore cannot arm this watchdog before that +threshold-crossing write completes. + +The host accepts that witness only when its PID and epoch match the faulted +generation, its request ID matches the active request, native dispatch is true, +and its byte count exceeds 4 MiB. Fault evidence persists the same marker. The +runner requires `nativePostgresOutputWitnessObserved=true` only for the native +lane and false for executor-deadlock and after-commit lanes. The host then +observes loss, refuses to reuse the generation, reconnects, and health-checks a +new process. This is deliberate fail-stop recovery; Android does not +automatically make a deadlocked or native-blocked thread healthy. + +This witness proves that the same native PostgreSQL request emitted more than +4 MiB before its ordered sleeping child. It is not a direct stack sample of +entry into `pg_sleep`. The two-second delay is deliberate room for the native +callback to return and execution to reach that child. + +In the corrected worker, `nativeDispatchStarted` and the request lifecycle's +native-dispatch transition occur immediately before `execProtocolStream`. +Executor-deadlock admission therefore remains distinct and its fixture lane no +longer waits for a native-dispatch probe. The native `pg_sleep` lane still does. + +The third fault executes a counter upsert and then aborts before sending +`completed`. The corrected report publishes `ambiguousExecutionCount`; it +derives `replayCount = ambiguousExecutionCount - 1` and accepts only values one +and zero respectively. Unlike the retained baseline's primary-key insert, a +second execution increments the counter and cannot be hidden by a uniqueness +error. All ten exact-current runs observed `ambiguousExecutionCount=1`, +`replayCount=0`, and the previously committed marker after recovery. + +## 5. Exact retained results + +All ten exact-current runs passed the same 11 checks: + +```text +separateProcess, healthySql, outOfBandCancel, +executorDeadlockFailStop, nativePgSleepFailStop, +outcomeUnknownNoReplay, boundedSlowReader8MiB, +boundedSlowReader32MiB, persistentRecovery, binderDeath, +freshPidAndEpoch +``` + +Each cold-host run began with a cleared app container and produced four worker +generations. Within each run, the three faulted generations were replaced +without clearing or reinstalling the app, so the same database root was reopened +after every process abort. Across the series there were 10 unique host PIDs, +40 unique worker PIDs, and 40 unique UUID epochs. + +### Fault recovery + +| Fault lane | Samples | Terminal | Binder death | Fresh PID + epoch + healthy SQL | Request start → healthy replacement | +| --- | ---: | --- | --- | --- | ---: | +| Executor deadlock + independent fail-stop | 10/10 | `outcomeUnknown` | 10/10 | 10/10 | 2,594–5,240 ms; median 2,762 ms | +| Native output-then-`pg_sleep` + independent fail-stop | 10/10 | `outcomeUnknown` | 10/10 | 10/10 | 2,698–7,359 ms; median 2,739 ms | +| Commit before `completed` | 10/10 | `outcomeUnknown` | 10/10 | 10/10 | 597–823 ms; median 626.5 ms | + +Every faulted worker PID has a retained `Fatal signal 6` record in that run's +crash-filtered logcat. Binder-death events were scoped to the exact worker PID +and epoch, so a late event from a prior generation could not satisfy the next +lane. Every replacement passed Ready validation and a healthy SQL query before +its new PID and epoch were accepted. + +The native lane recorded the same generation- and request-scoped PostgreSQL +output witness in all ten runs: 4,202,496 backend bytes, followed by the +configured 2,000 ms fail-stop delay. The ordered query and callback placement +make `pg_sleep(60)` the next executor child after the witnessed output, but +there is no separate callback from inside `pg_sleep`; the claim is deliberately +source-backed sequencing, not a direct in-function observation. + +The report's `nativeDispatchObserved` field is currently derived from whether +that witness exists, so it is not independent dispatch evidence and is not used +for the claim above. The generation/request-scoped output witness, source +ordering, SIGABRT record, Binder death, and fresh healthy generation are the +relevant evidence chain. + +All 30 post-dispatch losses became `outcomeUnknown`; the client made no SQL +retry after first write. The ambiguous counter was exactly one and derived +`replayCount` was zero in all ten runs. This proves no replay for this +instrumented mutation. It is not a generic exactly-once protocol. + +Two earlier failures remain useful negative evidence and are excluded from the +ten-run result: + +- `final-witness-001-20260811T113240Z` showed that a small first-statement + `CommandComplete` stayed buffered until after `pg_sleep`; it could not serve + as a pre-hang witness. +- `final-output-witness-006-20260811T115345Z` showed that the first transient + `POLLOUT=false` write could advance while the client still had not read. The + final probe therefore keeps an explicit read gate closed and resets its + candidate until the same non-writable write remains unchanged for at least + 300 ms. + +### Slow-reader transport + +The final probe does not infer boundedness from PSS. It directly observes a +blocking socket with `POLLOUT=false`, the same synchronous `responseBytes` +write in progress for at least 300 ms, and unchanged completed-write and byte +counters while the host-controlled read gate remains closed. The client then +releases the gate and drains the complete response from the same generation. + +| Case | Samples | Response bytes / chunks | Full-drain elapsed | Stable blocked write | Conservative accepted-wire upper bound | +| --- | ---: | --- | ---: | ---: | ---: | +| 8 MiB | 10/10 | 8,399,927 / 1,026 | 432–453 ms; median 437.5 ms | 300–312 ms | 493,920 B in every run | +| 32 MiB | 10/10 | 33,599,543 / 4,102 | 639–766 ms; median 648 ms | 300–314 ms | 493,920 B in every run | + +The cross-size accepted-bound delta was zero in every run, while the response +size quadrupled. Each bound plus one maximum encoded OLPB frame remained below +the corresponding response. This is direct evidence of synchronous socket +backpressure and a workload-specific pre-read acceptance bound. It does not +measure the effective `SO_SNDBUF`, provide an indefinite-stall guarantee, or +establish a general memory SLA. + +PSS and RSS remain descriptive observations: + +| Case | PSS span across runs | RSS span across runs | Observed full-drain rate | +| --- | ---: | ---: | ---: | +| 8 MiB | 30,992,384–31,045,632 B | 33,157,120–33,964,032 B | 17.684–18.544 MiB/s | +| 32 MiB | 97,970,176–101,406,720 B | 98,533,376–101,945,344 B | 41.832–50.146 MiB/s | + +The large final response is accumulated and reported by the experimental host, +so process memory rises substantially even though the worker's socket write is +bounded. The drain rates are emulator fixture observations, not maximum SQL or +transport throughput. + +### Persistence and artifact provenance + +Within every run, a previously committed marker survived all three broker +process aborts and the ambiguous post-native-success counter remained exactly +one. This proves reopen and WAL-backed persistence for this root across these +injected process deaths. It does not prove persistence across `pm clear`, app +reinstall, reboot, power loss, disk-full, or filesystem corruption; the runner +cleared the app once before each independent run. + +All ten repeated runs retained identical executable inputs and source +manifests: + +- base source HEAD: `dcdaeaac6eaffb2bb20136719f6ce2e3ffcb708b` plus an exact + `source-files.sha256` manifest for the uncommitted spike inputs; +- Debug APK: + `9bd24c29e9a0899064466eca9dfa840151a07ce0c8babbd4f18c0876951cbe51`; +- `liboliphaunt.so`: + `a55f65a36553cf19c70fc72617f65ff6ac35c5dc6c77f141c8220402c889281b`; +- `libc++_shared.so`: + `46b51d661454b9cfaf42c1dc90893b5ad601b7a7ffc2e09d44bb3cc9d20e7ae2`. + +`pr-doc-sync-v1` retained the same APK and native-library hashes and its source +manifest recomputes against the final spike files. Its expected manifest +difference is the README update that names the final evidence directories. + +Before each launch, the runner required the canonical Android native +`--check-current` gate to pass. Each evidence directory retains the source +manifest, source status, native and APK hashes, device description, build and +install logs, process and Binder-service snapshots, meminfo, full and +crash-filtered logcat, DropBox tombstone output, and the machine-readable +report. This is exact-file evidence for the dirty experimental source, not an +exact-commit or reproducible-build attestation. + +## 6. Proven versus unproven + +| Capability or constraint | Status after the ten-run series | +| --- | --- | +| Host and broker use different Linux processes | Emulator-proven in all ten runs. | +| AIDL control and reliable socket-pair PostgreSQL data plane | Emulator-proven in all ten runs. | +| Cancellation bypasses the occupied database executor | Emulator-proven in all ten runs with SQLSTATE `57014` and same-generation health afterward. | +| Executor-deadlock fail-stop | Emulator-proven 10/10 with exact SIGABRT PID evidence. | +| Native output-then-`pg_sleep` fail-stop | Emulator-proven 10/10 with the >4 MiB PostgreSQL-output witness, ordered query, delayed fail-stop, exact SIGABRT PID evidence, and fresh recovery; direct entry inside `pg_sleep` is not separately instrumented. | +| Binder death notification | Emulator-proven 30/30, scoped by PID and epoch. | +| Fresh PID/epoch and healthy replacement | Emulator-proven after all 30 process deaths. | +| Loss after dispatch becomes `outcomeUnknown` with no replay | Emulator-proven for all 30 losses; no-replay is directly counter-checked for the ambiguous mutation, not a generic exactly-once guarantee. | +| Sustained synchronous socket backpressure | Emulator-proven for both 8 MiB and 32 MiB in all ten runs, with the same 493,920-byte conservative pre-read bound. | +| Persistent database reopen after process abort | Emulator-proven within each run across three aborts. | +| Security/UID isolation | Not provided by design; the private service shares the app UID. | +| Physical-device behavior | Unproven. | +| Broad dependability or an SLA | Unproven; these are serial trials on one emulator, API, ABI, image, build, and host machine. | +| Same long-lived host surviving many cycles | Unproven; each full run uses a cold host, though it performs three sequential worker recoveries. | +| Other API levels, ABIs, OEM builds, and 32-bit devices | Unproven. | +| Background/foreground lifecycle, Doze, app standby, and low-memory kill | Unproven. | +| Reboot, force-stop, update, reinstall, `pm clear`, and PID-reuse behavior | Unproven. | +| Power-loss, torn-write, disk-full, and filesystem-corruption durability | Unproven. | +| Multiple concurrent clients, sessions, roots, or databases | Unsupported by this single-root spike and unproven. | +| Release build, hardened progress-sensitive watchdog policy, telemetry, Play distribution, and SDK integration | Unproven; fault hooks are DEBUG-only. | +| General latency, maximum throughput, CPU, energy, memory, or LMK SLA | Unproven. | + +## 7. Next qualification steps + +The next high-value step is to run the same strict contract on at least one +physical arm64 device, then repeat it across Android/API/OEM combinations. A +shipping design would also need a progress-sensitive watchdog that can +distinguish a slow query from a stuck request, a long-lived multi-cycle soak, +post-recovery data-integrity and leak checks, lifecycle/Doze/LMK coverage, and +Release/Play qualification. None of those conditions is implied by the 10/10 +emulator result. + +The runnable spike and invocation are documented in +[`spikes/android-native-broker/README.md`](../../spikes/android-native-broker/README.md). diff --git a/docs/architecture/ios-native-broker-spike.md b/docs/architecture/ios-native-broker-spike.md new file mode 100644 index 00000000..467b7fa6 --- /dev/null +++ b/docs/architecture/ios-native-broker-spike.md @@ -0,0 +1,1060 @@ +# iOS NativeBroker Technical Feasibility Spike + +This report describes the implemented iOS 26 `NativeBroker` spike and its +authoritative exact-source simulator and physical-device qualification. Evidence +is classified as: + +- **Implemented**: present in the checked-in Swift/C code. +- **Build-proven**: inspected in an exact simulator or signed device artifact + used below. +- **Simulator-proven**: asserted by a deterministic fixture lane and accepted by + its strict report validator. +- **Physical-device-proven**: asserted by the final signed Debug or Release + fixture on the wired device and accepted by the v2 report validator. +- **Unqualified**: no current listed run proves the behavior. + +The authoritative aggregate is +`target/ios-native-broker-full-matrix/simulator-matrix.json` +(`oliphaunt-ios-broker-full-simulator-matrix-v1`). It completed with +`status=PASS` at `2026-08-10T05:35:14Z` on an iPhone 17 Pro simulator running +iOS 26.4. Its four independently built and launched lanes passed 55 checks: +33 semantic, 7 handshake-negative, 10 extended-fault, and 5 hang checks. + +The authoritative physical aggregate is +`target/ios-native-broker-device-spike/reports/device-runner-report.json` +(`oliphaunt-ios-broker-device-run-v2`). It completed with `status=PASS` at +`2026-08-10T05:57:28Z` on the wired `iPhone15,2` device named +`Sid Jain’s iPhone`, running iOS 26.5. It contains two signed Debug semantic +launches with 33 checks each and a separate signed Release lifecycle aggregate. +The canonical Release report is +`target/ios-native-broker-device-spike/reports/device-lifecycle-runner-report.json` +(`oliphaunt-ios-broker-device-lifecycle-run-v2`); it passed at +`2026-08-10T05:57:28Z` with two 30-check foreground/background/foreground +launches. No failed or superseded physical artifact is used as passing +qualification evidence in this report. + +A separate, final signed-Debug physical hang lane is recorded at +`target/ios-native-broker-device-hang/device-hang-20260810T095852Z-46512/reports/device-hang-runner-report.json` +(`oliphaunt-ios-broker-physical-hang-run-v1`). It completed at +`2026-08-10T09:59:20Z` on the same device. Its evidence contract passed, but +`recoveryProven=false`: it is negative capability evidence, not a passing +hang-recovery claim. + +A later DEBUG-only mechanism experiment is retained under +`target/ios-native-broker-recovery-experiments/20260811T060724Z`. It armed and +acknowledged a deadlock, proved the same worker still answered diagnostics, +then triggered the deadlock only after a normal native request registered its +cancellation target. On the physical device, immediate public recreation and a +50 ms delay both reached stale active-channel state; 100 ms and 250 ms delays +each obtained a different PID, fresh epoch, Ready generation, and healthy SQL +response. Private terminate and unique-instance SPI controls also recovered at +zero delay, as did a DEBUG extension-side one-second fail-stop watchdog. These +are causal experiments, not canonical capability qualification: the private +SPI cannot ship, the delay boundary was sampled only once per value, and the +fail-stop policy is not yet a production implementation. + +A final repeated physical experiment closes the native-execution mechanism +question without changing the advertised capability. Two 30-trial batches at +commit `adc8cf09` ran the one-second actor-block fail-stop path 60/60. Two more +30-trial batches at commit `b5b5177e` ran `pg_sleep(60)` while a DEBUG work item +outside `WorkerCore` called `abort()` during native execution; that path also +recovered 60/60. Every counted trial produced a different worker PID and epoch, +a new validated Ready generation, and healthy SQL. This is repeated mechanism +evidence on one iPhone15,2/iOS 26.5 beta stack, not a Release reliability or +shipping-policy claim. + +The exact simulator artifact manifest is +`oliphaunt-ios-broker-artifacts-v1`, arm64, iOS 26.0, PostgreSQL 18.4, C ABI 6, +`brokerDatabaseRole=oliphaunt_broker`, and +`selectedExtensions=vector,pg_trgm`. Its SHA-256 fingerprints are: + +- dylib: `dc25e809c67b93d7706f49f093857547eaa0daaf96997a90c12bb8303f358a81`; +- XCFramework: `8f87ce2bc10aa25dc9e7b35b9d6cb36bdc3bfea785fa466993e557087512cb48`; +- resources: `35f0803a95ab108a272fb9a3db32f71bf1b1f976b5c6518924dbbe26aa577ebf`; +- template/initdb: `2879a752803678cbf01f8dfad03d5b59e4bbc1d00ec9973fca2a7a7e99426906`. + +The semantic lane reported runtime root-manifest digest +`5488f9fa6f756b020c3ca57f92207bded78b6843a2da1beaf78c0af8bde96073`; +the handshake-negative and extended-fault lanes matched it. + +The static registry is complete for `vector,pg_trgm`. Build inspection shows no +`liboliphaunt` load command in the host and +`@rpath/liboliphaunt.dylib` only in the app-extension executable. The extension +also contains the runtime, template PGDATA, registry metadata, and matching +control/SQL files for both selected extensions. + +## 1. Implemented component diagram + +```mermaid +flowchart TD + subgraph Host["Host application: no liboliphaunt load command"] + DB["OliphauntDatabase query and transaction layer"] + Session["IOSBrokerSession"] + Manager["IOSBrokerManager application-scoped actor"] + XPCClient["OliphauntBrokerXPC control codec"] + Data["IOSBrokerDataChannel"] + DB --> Session --> Manager + Manager --> XPCClient + Manager --> Data + end + + subgraph Extension["Bundle-only BrokerAppExtension.appex"] + Entry["ExtensionFoundation AppExtension"] + XPCServer["Primitive-only XPC handler"] + Socket["BrokerSocketWorker"] + Core["WorkerCore actor"] + Cancel["CancellationController"] + Privacy["Bounded backend privacy filter"] + Direct["OliphauntNativeDirectEngine"] + Native["liboliphaunt and PostgreSQL 18.4"] + Root["one resident logical root: default"] + Entry --> XPCServer + XPCServer --> Core + XPCServer --> Cancel + Socket --> Core + Core --> Direct --> Native --> Root + Direct --> Privacy --> Socket + Cancel --> Direct + end + + XPCClient <-->|"Hello, cancel, lifecycle, diagnostics"| XPCServer + Data <-->|"AF_UNIX socketpair; 40-byte framed PostgreSQL bytes"| Socket +``` + +The host discovers a real `AppExtensionIdentity`, starts or attaches through +`AppExtensionProcess`, and retains that process for the generation. `Hello` +atomically transfers one socket endpoint. The extension owns one `WorkerCore`, +one root, and one native PostgreSQL session. There is no listener, loopback TCP, +daemon, downloaded code, `Process`/`NSTask`, private entitlement, or +`NativeServer`. + +## 2. Swift types and target/module boundaries + +| Module or target | Principal types | Dependency boundary | +| --- | --- | --- | +| `OliphauntBrokerProtocol` | protocol constants, `BrokerFrame`, `BrokerHello`, `BrokerReady`, capabilities, errors, epochs, request IDs, state machines | Pure Swift plus Foundation; no XPC, ExtensionFoundation, `Oliphaunt`, `COliphaunt`, or native runtime. | +| `OliphauntBrokerXPC` | `IOSBrokerXPC`, `IOSBrokerControlEnvelope`, `IOSBrokerWireDiagnostics`, `IOSBrokerOwnedFileDescriptor` | Primitive lightweight-XPC codec and explicit FD ownership. Depends only on the protocol module. It does not import the host manager or extension worker. | +| `OliphauntIOSBroker` | `IOSBrokerConfiguration`, `IOSBrokerEngine`, `IOSBrokerManager`, `IOSBrokerSession`, `IOSBrokerDataChannel` | Host adapter over `Oliphaunt`, the protocol module, and `OliphauntBrokerXPC`; it does not link the prepared native XCFramework. | +| `OliphauntBrokerExtension` | `WorkerCore`, `CancellationController`, `BrokerSocketWorker`, storage, response observer, privacy filter, DEBUG fault injector | Extension implementation over `Oliphaunt`, `COliphaunt`, and the protocol module. It does not depend on the host adapter or XPC codec. | +| `BrokerAppExtension` fixture target | ExtensionFoundation entry point and XPC server | Imports `OliphauntBrokerExtension` and `OliphauntBrokerXPC`, owns native resources, and is the only fixture target that loads `liboliphaunt`. | +| `OliphauntBrokerSpike` fixture target | semantic, handshake-negative, extended-fault, hang, and device-lifecycle fixtures | Uses the public host adapter and emits JSON plus exact PASS/FAIL markers. | + +The package exposes `OliphauntBrokerProtocol`, `OliphauntBrokerXPC`, +`OliphauntIOSBroker`, `OliphauntBrokerExtension`, and `Oliphaunt` as separate +products. The fixture is a non-UI custom ExtensionKit extension under +`Extensions/BrokerAppExtension.appex`. Swift 6 strict-concurrency checks cover +the shared state machines, host queueing, XPC wire values, extension recovery, +cancellation order, and exactly-once terminal behavior. + +The canonical top-level device report points through +`validations.semanticDebugRetainedProduct` to +`retained-semantic-debug-product.json` +(`oliphaunt-ios-broker-retained-semantic-debug-product-v1`, `status=PASS`). The +validated app is +`target/ios-native-broker-device-spike/retained-semantic-debug.3GeyQ0/OliphauntBrokerSpike.app`; +its extension is the nested +`Extensions/BrokerAppExtension.appex`, and its originating result bundle is +`reports/device-build-20260810T054111Z-58906.xcresult` under the device-spike +target. The runner copied the signed Debug app to a unique retained path before +the Release clean, verified the app and embedded-extension signatures and bundle +IDs, and +recorded executable SHA-256 values. Its lifecycle continuation then failed +closed unless the retained path stayed inside the device-build root, the +extension and result-bundle paths matched, both signatures remained valid, and +both executable hashes still matched. The retained host hash is +`b5ee74ec19b49670f3d94d9ee3af876cf3e43c5915e7fd77f7500e68db064d0f`; the +retained extension hash is +`ea28d943c448cde92e09baaab09ffd3ddd44870d6b06e5eb931f33c518a4a07c`. + +The final arm64 iPhoneOS Release artifact was built, archived, installed, and +launched from the archive. Recursive signing inspection found matching +development-team identities on the app, extension, and native framework. The +host has no `liboliphaunt` load command; the extension loads +`@rpath/liboliphaunt.framework/liboliphaunt`. The containing app embeds exactly +one framework, the extension embeds no duplicate framework, and no loose +`liboliphaunt.dylib` exists. Symbol audits found no host-only +`IOSBrokerManager`/`IOSBrokerEngine`/`IOSBrokerSession` implementation in the +extension. The Release app/extension audit excluded implementation symbols +matching `BrokerFaultInjector`, `WorkerCore.*injectFault`, +`IOSBrokerSession.*injectFault`, `ExtendedFaultMatrix`, and `HangFaultMatrix`. +Shared `BrokerWorkerFault` protocol enum types remain intentionally present. + +Exact Release product sizes were 58,549,340 bytes for the app bundle, 43,535,644 +for the extension bundle, 12,726,496 for the native framework, 41,943,411 for +runtime resources, 1,656,944 for the host executable, and 1,085,264 for the +extension executable. + +## 3. Exact XPC control-message schema + +Every control value is an `XPCDictionary`. Scalars are `String`, `UInt64`, +`Int64`, or `Bool`; structured values are ordinary `JSONEncoder` JSON strings. +`dataChannel` is a real `XPC_TYPE_FD`. PostgreSQL bytes never travel over XPC. + +### Handshake + +| Direction/kind | Required fields | Optional fields | +| --- | --- | --- | +| Host → extension, `hello` | `message`, `minimumProtocolVersion`, `maximumProtocolVersion`, `expectedABI`, `rootID`, `startupConfigurationDigest`, `requestedCapabilities`, `dataChannel` | `expectedRuntimeVersion` | +| Extension → host, `ready` | `message`, `selectedProtocolVersion`, `epoch`, `extensionPID`, `runtimeVersion`, `abiVersion`, `postgresMajorVersion`, `rootManifestDigest`, `actualCapabilities`, `actualRuntimeConfiguration` | none | +| Extension → host, `rejected` | `message`, `error`, `reason` | legacy `rejection` only as a compatibility fallback | + +The current fixture requests protocol `1...1`, ABI 6, root `default`, startup +digest `ios-native-broker-spike-v2-restricted-role`, and capabilities +`processIsolated`, `crashRestartable`, `sameRootLogicalReopen`, `protocolRaw`, +`protocolStream`, and `queryCancel`. Runtime version is not pinned. The worker +returns the actual root identity, startup digest, extensions, and +`smallMobile` footprint in `actualRuntimeConfiguration`. + +The host rejects protocol, ABI, optional runtime, root, digest, extension, +capability, and negative single-root/single-session mismatches. Before launch it +also rejects any broker database configuration that supplies a filesystem root, +unsafe durability, a non-`smallMobile` footprint, custom startup GUCs, or a +non-default public username/database. Unsupported public fields are not dropped. + +### Lifecycle and diagnostics + +Established-worker controls carry `message` and the expected `epoch`. +`requestID` is monotonic when supplied and is mandatory for `cancel`. + +| Request kind | Additional fields | Successful reply | +| --- | --- | --- | +| `cancel` | `requestID` | `message: "cancel"` or `"cancelObserved"`; `success: true` | +| `checkpoint` | none | same `message`; `success: true` | +| `prepareForBackground` | `deadlineUnixNanoseconds` | same `message`; `success`, `cancelledActiveWork`, `checkpointed` | +| `resumeFromBackground` | none | same `message`; `success: true` | +| `detach` | none | same `message`; `success: true` | +| `diagnostics` | none | fields below | +| `injectFault` | `fault` | same `message`; `success: true`; DEBUG only | + +A diagnostics reply contains `message`, `success`, `state`, `epoch`, +`extensionPID`, optional `manifestDigest`, optional `activeRequestID`, +`nativeDispatchStarted`, `transactionStatus`, `capabilities`, optional +`currentPhysFootprintBytes`, optional `currentResidentBytes`, optional +`availableMemoryBytes`, `checkpointInProgress`, optional +`storageProtectionEvidenceJSON`, and the four optional +`extensionEntryPreOpen*`/`openedIdle*` memory fields. A completed checkpoint may +also include the all-or-nothing historical tuple +`checkpointMemorySampleSequence`, +`checkpointMemorySampleStartedAtUptimeNanoseconds`, +`checkpointMemorySampledAtUptimeNanoseconds`, +`checkpointMemorySampleCompletedAtUptimeNanoseconds`, +`checkpointMemorySamplePhysFootprintBytes`, +`checkpointMemorySampleResidentBytes`, and +`checkpointMemorySampleAvailableMemoryBytes`. The public host diagnostic adds +manager-only `logicalHandleCount`, `queuedOperationCount`, +`launchAttemptCount`, `launchCount`, `interruptionCount`, and +`admissionsPaused`; those fields are not claimed as worker XPC fields. + +All boundary failures are encoded as structured `BrokerError` JSON. Path-bearing +or unconstrained internal errors are mapped to a small path-free reason set +before crossing XPC. `attachDataChannel` is reserved and rejected in v1 because +the FD is part of `Hello`. Host attempts to send `ready`, `rejected`, or +`cancelObserved`, malformed numeric values, invalid UUIDs, unknown faults, or +wrong-epoch controls are protocol errors. + +## 4. Exact 40-byte wire-frame definition + +All integers use network byte order; UUID bytes are the canonical 16 raw bytes. + +| Offset | Size | Field | Rule | +| ---: | ---: | --- | --- | +| 0 | 4 | magic | ASCII `OLPB` (`4f 4c 50 42`) | +| 4 | 2 | protocol version | UInt16; v1 is `1` | +| 6 | 2 | header length | UInt16; exactly `40` | +| 8 | 1 | frame type | UInt8 enumeration below | +| 9 | 1 | flags | UInt8; known mask is zero | +| 10 | 2 | reserved | UInt16; must be zero | +| 12 | 16 | epoch | current worker UUID | +| 28 | 8 | request ID | UInt64; nonzero for request frames, zero otherwise | +| 36 | 4 | payload length | UInt32; at most 256 KiB | + +| Value | Type | Request ID | Payload | +| ---: | --- | --- | --- | +| 1 | `requestBegin` | nonzero | empty | +| 2 | `requestBytes` | nonzero | PostgreSQL frontend bytes | +| 3 | `requestEnd` | nonzero | empty | +| 4 | `responseBytes` | nonzero | PostgreSQL backend bytes | +| 5 | `completed` | nonzero | empty | +| 6 | `rejected` | nonzero | encoded `BrokerRejectionReason` | +| 7 | `outcomeUnknown` | nonzero | optional path-free detail | +| 8 | `cancelRequested` | nonzero | empty; XPC is the normal path | +| 9 | `cancelObserved` | nonzero | empty | +| 10 | `ping` | zero | empty | +| 11 | `pong` | zero | empty | +| 12 | `protocolError` | zero | path-free UTF-8 detail | +| 13 | `channelClose` | zero | empty | + +The decoder rejects bad magic/version/header/type/flags/reserved values, stale +epochs, invalid request-ID domains, oversized payloads, overflow, truncation, +and illegal state transitions before allocating a declared payload. The maximum +frame payload is 256 KiB and the shared active-plus-queued request budget is +8 MiB. The default complete frontend request limit is also 8 MiB. + +The C API accepts one complete frontend-protocol request, so v1 validates and +assembles fragments before dispatch. It truthfully reports +`streamingRequestInput=false`. Backend bytes stream directly. The native stream +queue has a 4 MiB default hard ceiling and splits a larger backend write into +ordered pieces before allocation. A native smoke test forces a 1,024-byte +ceiling over a response larger than 64 KiB and validates exact PostgreSQL bytes. + +The nonstreaming `execProtocolRaw` path is bounded too: +`IOSBrokerConfiguration.maximumRawResponseBytes` defaults to 8 MiB, may not +exceed the transport bound, and its collector throws and discards its partial +buffer on overflow. Large results must use `execProtocolStream`. + +## 5. Request and worker state machines + +### Host generation + +```text +unavailable +idle -> launching -> binding -> ready(epoch) +interrupted(oldEpoch) -> recovering -> binding -> ready(newEpoch) +ready(epoch) -> quiescing(epoch) -> ready(epoch) +ready(epoch) -> closing -> idle +launch/discovery failure -> unavailable | idle | interrupted(oldEpoch) +``` + +`IOSBrokerManager` owns the retained process, XPC session, socket, epoch, +process-monotonic request-ID source, logical handles, FIFO, shared input budget, +in-flight registry, transaction owner, and recovery counters. SQL work is FIFO. +A `ReadyForQuery(T/E)` pins the transaction to its logical handle and +`ReadyForQuery(I)` releases it. + +### Worker + +```text +created -> starting -> ready +ready -> quiescing -> ready +ready | quiescing -> interrupted +ready -> detached +starting | ready | quiescing -> failed(reason) +interrupted | detached -> starting with a fresh epoch +``` + +`WorkerCore` has one active request and explicit reentrancy guards. Every start +owns a token revalidated after storage preparation, native open, restricted-role +bootstrap, capability validation, and health check. An interrupted suspended +open therefore cannot publish itself as the replacement generation. + +### Request + +```text +queued -> receiving -> readyToDispatch -> running -> terminal(completed) +queued | receiving | readyToDispatch -> terminal(canceled) +running -> cancelRequested -> terminal(completed | outcomeUnknown) +host-queued loss before any write -> terminal(notStarted) +loss after bytes may have reached the worker -> terminal(outcomeUnknown) +``` + +Each host operation has one checked continuation and one guarded terminal bit. +Terminal cleanup removes it from queue/in-flight state, cancels deadline tasks, +releases its reservation, and resumes exactly once. Request IDs never reset when +epochs change. + +## 6. Descriptor ownership rules + +1. The host creates `socketpair(AF_UNIX, SOCK_STREAM, 0, ...)` and initially owns + both descriptors. +2. It sets `FD_CLOEXEC`, `O_NONBLOCK`, and `SO_NOSIGPIPE` on both endpoints. +3. `IOSBrokerDataChannel` adopts the host endpoint; `DispatchIO` closes it from + its cleanup handler. +4. `xpc_fd_create` boxes a duplicate of the extension endpoint. No bare integer + FD crosses the control boundary. +5. The receiver calls `xpc_fd_dup`; `IOSBrokerOwnedFileDescriptor` owns the new + descriptor until it transfers ownership once to `BrokerSocketWorker`. +6. The host closes its original extension endpoint only after a worker reply + proves transfer. Every failure path closes what it still owns. +7. The extension reapplies `FD_CLOEXEC`/`SO_NOSIGPIPE`, clears `O_NONBLOCK`, and + uses bounded blocking I/O on its private queue. Its socket send-buffer target + is 512 KiB and there is no second unbounded Swift response queue. +8. Interruption shuts down the epoch socket, cancels XPC, and invalidates the + retained `AppExtensionProcess`; cleanup is idempotent. +9. The descriptor is bound to its `Hello` epoch. Header validation prevents it + from being attached to a later generation. +10. Graceful old-socket cleanup calls `detach(expectedEpoch:)`; channel tokens + and XPC session IDs prevent stale cleanup from detaching a replacement. + +## 7. Completion and OutcomeUnknown semantics + +`completed` proves only that framed transport ended normally. SQL success or +failure is carried in ordinary PostgreSQL backend bytes. A PostgreSQL +`ErrorResponse` is followed by `ReadyForQuery` and then `completed`; the existing +`OliphauntDatabase` parser turns those synchronized bytes into the typed SQL +error. + +The host does not trust `completed` alone. Its incremental observer requires a +structurally complete stream whose last completed backend message is +`ReadyForQuery(I/T/E)`. The worker independently requires a valid terminal +`ReadyForQuery`. Missing, malformed, truncated, or nonterminal output tears down +the epoch and returns `outcomeUnknown` after possible dispatch. + +`rejected` is restricted to proven pre-dispatch rejection. `notStarted` is used +only when the host proves no request bytes reached a worker. After the first +possible write or native dispatch, any loss before `completed` yields +`BrokerError.outcomeUnknown(epoch, requestID)`: crash, XPC interruption, EOF, +malformed output, consumer failure after chunks, native failure without proof, +or commit followed by loss of the terminal frame. + +There is no SQL classification and no automatic replay. A bounded raw collector +discards partial data when it throws. A streaming consumer may have seen chunks, +but a final `OutcomeUnknown` means those chunks are incomplete. + +The semantic lane deterministically crashed after a committed marker but before +`completed`, returned `OutcomeUnknown`, recovered, and found the marker exactly +once. It separately crashed with an uncommitted marker and found it absent after +WAL recovery. The extended-fault lane additionally observed 4,139 response bytes +in four chunks before a worker crash and still returned the partial-stream +outcome as unknown. + +Both final signed Debug device launches repeated the 33-check semantic workload. +Each returned `OutcomeUnknown` for the post-commit fault, recovered the committed +marker exactly once without replay, recovered a separate pre-commit crash, found +the uncommitted marker absent, streamed 2,119,735 bytes in 4,358 chunks, and +completed healthy SQL. That is current iPhoneOS PostgreSQL/WAL evidence, not a +reuse of the earlier device smoke. + +## 8. Cancellation race resolution + +Queued operations are removed before dispatch and finish once as canceled. +Running cancellation uses XPC. The extension invokes the lock-protected, +nonisolated `CancellationController` before scheduling actor-isolated +`WorkerCore` bookkeeping, so the native signal cannot queue behind the query. + +For running work: + +1. native cancellation is signaled out of band; +2. the lifecycle is recorded as `cancelRequested` without issuing a second + native cancel; +3. PostgreSQL produces ordinary backend output; +4. `ErrorResponse` SQLSTATE `57014` proves that PostgreSQL observed cancellation; +5. `ReadyForQuery` restores synchronization; and +6. the worker sends response bytes and exactly one `completed` terminal frame. + +The 64 KiB-bounded backend observer extracts only cancellation SQLSTATE and +terminal transaction state; it does not accumulate rows. Cancellation and +completion share one terminal transition. Duplicate or late controls are +idempotent. An acknowledgement proves only observation of the cancel request. + +The exact extended-fault close/cancel/completion race used a three-second +`pg_sleep`, below its six-second request deadline, waited for matching host and +worker active IDs plus `nativeDispatchStarted=true`, then closed and canceled +concurrently. Its raw response parsed as PostgreSQL cancellation SQLSTATE +`57014`, the transport still reached `completed`, and the report recorded: + +```text +closeCancelCompletionTerminal=postgresCanceledCompleted +closeCancelControlOutcome=acknowledged +``` + +The validator also permits the control side to lose only to an already closed +database, while still requiring the same PostgreSQL-canceled/completed terminal +result. Deadlines request cancellation, wait a bounded grace period, and +invalidate the epoch when synchronization cannot be recovered. + +Both signed Debug semantic launches and both signed Release lifecycle launches +also passed cancellation followed by post-cancel liveness on the physical +device. The exact raw close/cancel/`57014`/`completed` race remains the simulator +extended-fault proof; the device reports do not overstate their higher-level +cancellation checks as a second raw-race capture. + +## 9. Crash and hang recovery behavior + +Interruption atomically invalidates the launch ID and epoch, shuts down socket +and XPC, clears transaction ownership, completes every operation once, and moves +to `interrupted(oldEpoch)`. Recovery is demand-driven: discover again, create a +new process/XPC/socket generation, reopen root `default`, allow WAL recovery, +then require `ping`/`pong` before admission. No SQL is replayed. + +The semantic lane used host PID `55305` and these generations: + +| Launch | Cause | Epoch | Worker PID | Result | +| ---: | --- | --- | ---: | --- | +| 1 | Initial open | `fff65828-f5af-41d5-b155-8644580e3d31` | `55318` | Separate from host; normal workload. | +| 2 | Logical detach/reopen | `8e1711b6-4ae1-4dfc-80fe-2b885746dbb7` | `55318` | Fresh epoch in the same OS process. | +| 3 | Post-commit crash recovery | `facac562-1ceb-4fcc-9a85-c8fd43dd1856` | `55384` | Fresh process; committed marker present once. | +| 4 | Pre-commit crash recovery | `2ce33ccd-9a52-4647-9c65-bc3d1729f21d` | `55393` | Fresh process; uncommitted marker absent. | + +The extended-fault lane used host PID `56963`, initial worker PID `56972`, and +five fresh recovery epochs: +`730dc392-b58c-4ee0-ba73-887e02e978b7`, +`dd69f26c-1f7a-44b2-906c-befa936f146e`, +`1bfb141e-8a9a-4b28-af69-5d72afd0b1f8`, +`54317682-f6cf-4186-bec2-66ef9f06e663`, and +`dd8a3fe3-5657-40de-9837-317aa0b51c4e`; its final worker PID was `57024`. +It covered crashes before dispatch, after response chunks, during checkpoint, +and while idle via abort and SIGSEGV, plus different-root and archive-boundary +rejection. + +Hang behavior was measured conservatively. Host PID `57602` attached to worker +PID `57611` at epoch `9f3733c2-5f70-4e2c-98ef-7d4771891c4d`. The main actor +remained responsive, the old epoch was invalidated, calls failed with +`workerInterrupted`, and one replacement launch was attempted. The system did +not supply a successful new generation or fresh process: + +```text +replacementLaunchAttemptDelta=1 +successfulLaunchCountDelta=0 +freshProcessObtained=false +hangRestartableCapability=false +``` + +This is a passing conservative result, not a hang-restart claim. + +The dedicated physical lane then ran the same deliberate deadlock last, using +the retained signed Debug artifact. Host PID `7110` attached to worker PID +`7112` at epoch `69502c5d-eb5c-4f2c-842c-930060230881`. The host main actor +remained responsive, the request terminated as `workerInterrupted`, the old +epoch was invalidated, and the post-hang health operation caused one actual +replacement initializer attempt. The counters were again: + +```text +initialWorkerPID=7112 +replacementLaunchAttemptDelta=1 +successfulLaunchCountDelta=0 +freshProcessObtained=false +recoveredEpochs=[] +hangRestartableCapability=false +``` + +The console and app-container reports were semantically identical, the +`devicectl` host PID matched, and the exact signed app, installation URL, launch +executable, device, and process identities were cross-checked. Thus physical +deliberate-hang recovery is no longer merely untested: no fresh healthy worker +was obtained in this bounded iPhone15,2/iOS 26.5 observation. This verifies the +current v1 product limitation, but one immediate failed replacement attempt on +each tested stack does not prove that every retry policy or every iOS 26 device +must fail. + +The follow-up mechanism experiment removed that earlier observability gap. The +fault-control request now only arms the DEBUG deadlock and returns an +acknowledgement. Same-PID/same-epoch diagnostics then prove `WorkerCore` remains +responsive. The next ordinary query registers the native cancellation target +before entering the non-returning zero-count semaphore wait under actor +isolation. All public-delay trials observed the expected `outcomeUnknown` after +roughly six seconds and an interruption-count increase, proving the armed +deadlock—not merely the control request—caused the invalidation. + +The physical public-delay results were: + +| Configured / actual delay | Initial -> recovered PID | Ready delta | Result | +| ---: | --- | ---: | --- | +| 0 / 0 ms | `8976` -> none | 0 | Stale process/session rejected the recovery query because a broker data channel was still active. | +| 50 / 52 ms | `9006` -> none | 0 | Same stale active-channel result. | +| 100 / 106 ms | `9009` -> `9010` | 1 | Fresh epoch and healthy SQL response. | +| 250 / 265 ms | `8979` -> `8980` | 1 | Fresh epoch and healthy SQL response. | + +This local boundary is evidence of asynchronous teardown and process reuse, not +a documented or reliable 100 ms platform threshold. The zero-delay controls +make the causal distinction sharper: private unique-instance acquisition +recovered `8986` -> `8987`; corrected private termination recovered `8997` -> +`8998`; combining both recovered `9001` -> `9002`; and the independent +DEBUG fail-stop watchdog recovered `8993` -> `8994` after the original worker +exited. Every claimed recovery also changed epoch, incremented the Ready count, +and completed `SELECT 'healthy'`. + +The first private-termination probe used an incorrect dynamically cast Swift +method ABI and crashed the host before returning an SPI result. The corrected +instance-method bridge passed; the earlier SIGSEGV is therefore fixture error, +not evidence that the OS termination primitive failed. Both private mechanisms +remain unsupported implementation details and are retained only to diagnose +the public API gap. + +The final DEBUG-only repetition used the public manager/reconnect path and no +private ExtensionFoundation SPI. The actor-block control passed 60/60 trials: +the request became `outcomeUnknown` in 1.003-1.068 seconds, then a fresh PID, +epoch, Ready generation, and healthy SQL response were observed. The stronger +native control armed `.duringNativeExecution`, issued +`SELECT pg_sleep(60), 'must-not-complete'::text AS status`, and let a global +Dispatch work item call `abort()` while the worker actor/thread was inside the +native call. It also passed 60/60 trials. Its terminal was observed in 250-267 +ms because the fixture waits 250 ms to prove the main actor remains responsive; +the injected work item itself is scheduled after 50 ms. + +iOS retained three sampled crash reports for formal initial workers. Each is an +`EXC_CRASH` / `SIGABRT` with the faulting global-queue thread running +`BrokerFaultInjector.beginNativeExecution` -> `abort()`, while separate threads +are simultaneously inside `oliphaunt_exec_protocol_stream` and PostgreSQL +`pg_sleep`. Two other retained reports belong to recovered workers and record a +later RunningBoard `0xdead10cc` termination; they are excluded from watchdog +causality. Crash-log coalescing means these samples corroborate the mechanism +but are not a per-trial OS termination receipt for all 60 trials. + +All native-control trials had launch-attempt counters `1 -> 1 -> 2` and Ready +counters `1 -> 1 -> 2`. Their 120 worker epochs and 180 host/initial/recovered +PIDs were unique, the old worker was absent at report publication, per-trial +cleanup passed, and no fixture process remained afterward. Launch-to-report +latency was 3.191-3.541 seconds (nearest-rank p50 3.342 seconds; p95 3.465 +seconds). With 60/60 observed successes, the one-sided exact 95% IID lower +success bound is 95.13%. The IID assumption is weak for serial trials on one +device/build/install, so this establishes repeatability on that tested stack, +not a production SLA. The DEBUG timer is deliberately armed for a known fault; +it is not a progress-sensitive policy that can distinguish a valid slow query +from a native hang. + +The final signed Debug physical semantic launches recorded these generations: + +| App launch / host | Cause | Epoch | Worker PID | Result | +| --- | --- | --- | ---: | --- | +| 1 / `6615` | Initial open | `e7da4ad4-f4ae-41e6-a3ab-be9ef6e86d09` | `6617` | 33-check workload began in a separate process. | +| 1 / `6615` | Logical detach/reopen | `ee78d845-be19-4e3c-9557-83b25329f430` | `6617` | Fresh epoch, same process/root. | +| 1 / `6615` | Post-commit crash recovery | `043c8737-f142-4b50-ac07-4808ae1ea897` | `6619` | Fresh worker; committed marker present once. | +| 1 / `6615` | Pre-commit crash recovery | `82b1bdd6-b7af-47aa-8560-7929d4fa8cc5` | `6620` | Fresh worker; uncommitted marker absent. | +| 2 / `6621` | Initial open | `719ecba3-56cc-4cbc-9ae6-aee96baf30e5` | `6623` | New host launch without reinstall. | +| 2 / `6621` | Logical detach/reopen | `05d02deb-15c3-4b13-947c-95f1c71bd6c7` | `6623` | Fresh epoch, same process/root. | +| 2 / `6621` | Post-commit crash recovery | `8fc2289c-d34d-42ca-a1f0-0a980899bdf3` | `6624` | Fresh worker; committed marker present once. | +| 2 / `6621` | Pre-commit crash recovery | `0ef5db95-6508-4e73-bfba-97ab8926be02` | `6625` | Fresh worker; uncommitted marker absent. | + +The signed Release lifecycle artifact then ran twice: + +| Release launch / host | Phase | Epoch | Worker PID | External observation | +| --- | --- | --- | ---: | --- | +| 1 / `6629` | Initial foreground | `c7ccd904-c4e9-4ef7-b88f-2adc44c7002e` | `6631` | Foreground inventory contained one host and one worker. | +| 1 / `6629` | Foreground resume | `7f745e57-09b4-413c-8c17-a9a9489db290` | `6637` | Fresh PID and epoch; health and persistence passed. | +| 2 / `6638` | Initial foreground | `fae15a8c-54b3-44ff-9fe5-ac6333b08674` | `6640` | Foreground inventory contained one host and one worker. | +| 2 / `6638` | Foreground resume | `726b89c8-351f-4b2a-bc67-a90b32aac829` | `6642` | Fresh PID and epoch; health and persistence passed. | + +Each Release launch passed 30 checks. Foregrounding Settings caused a real scene +transition; after the fixture quiesced and checkpointed, `devicectl` delivered +`SIGSTOP` (signal 17) to the background host. At the post-suspend inventory each +host was present and its initial worker was absent. The evidence supports only +`workerLossWindow=afterQuiescedEvidenceThroughPostSuspendInventory`: **after +quiesced evidence through post-suspend inventory**. The termination cause is +unattributed. No intentional `SIGKILL` was delivered in either launch. Launch 1 +requested no worker kill +(`workerTerminationMode=notRequested`); launch 2 intended that exercise, but the +worker was already absent and the validated mode was +`workerAbsentAtPostSuspendInventory`. Both launches nevertheless prove recovery +from an unavailable worker via a fresh worker PID and epoch after foregrounding. + +The fixture's idle-timer guard is foreground-only: +`UIApplication.shared.isIdleTimerDisabled` is true only while the scene is +active. It is disabled again while inactive/backgrounded, creates no background +task, and is not evidence of a keepalive. The capability therefore remains +`backgroundContinuable=false`. + +## 10. Storage/root ownership design + +The host sends only root ID `default`; it never sends, receives, logs, or exposes +a PGDATA URL. `BrokerExtensionStorage.extensionPrivate()` resolves and owns: + +```text +Library/Application Support/Oliphaunt/default/ +├── manifest.json +├── pgdata/ +├── runtime-cache/ +└── staging/ +``` + +The extension rejects symlinks, applies +`completeUntilFirstUserAuthentication`, atomically writes a canonical manifest, +and acquires the stable native filesystem lease. One process owns one resident +root and one physical session. Logical opens share it; reference count zero is a +detach, not proof of process death. A different logical root is rejected. An App +Group constructor remains an explicit fallback and is not enabled. + +Host-visible SQL is authenticated as `oliphaunt_broker`, not PostgreSQL's +bootstrap superuser. PostgreSQL patch +`0021-liboliphaunt-authenticate-embedded-role.patch` changes only the Oliphaunt +host-I/O `InitPostgres` path: it requires the supplied role to have LOGIN, +initializes authenticated/session identity from it, and monotonically latches a +catalog-observed non-superuser state so RESET, rollback, or `DISCARD ALL` cannot +restore bootstrap privilege. Ordinary PostgreSQL standalone startup is kept +unchanged. + +Worker bootstrap owns the database and selected extensions as `postgres`, owns +only schema `oliphaunt_broker` as the restricted role, fixes `search_path` to +`"$user", public`, rejects caller `search_path`, and limits membership to +`pg_checkpoint`. It removes database/public-schema CREATE, database-owner +assumption, path-function execution, file/config views and functions, and other +privileged memberships. It validates the role, ownership, ACLs, extensions, +tablespaces, and effective role graph before publishing `Ready`. + +The semantic lane proved 23 denial probes at SQLSTATE `42501`, including role +and session-authorization escalation, RESET and DISCARD re-escalation, data +directory/settings, server-file and external COPY access, tablespace, role, +native-function and library creation, `ALTER SYSTEM`, and selected-extension +ownership. It found zero visible data-directory/source/private-path settings, +zero restricted function/view privileges, no non-default tablespace, database +owner `postgres`, extension owners `pg_trgm:postgres,vector:postgres`, broker +schema owner `oliphaunt_broker`, and search path +`{oliphaunt_broker,public}` after `DISCARD ALL`. + +Before backend bytes leave the extension, an incremental privacy filter examines +only `ErrorResponse` and `NoticeResponse`. It replaces extension-private path +prefixes with `[redacted]`, buffers at most 64 KiB for such a message, and emits a +fixed path-free replacement when it is larger. Other messages stream without +response-wide accumulation. The semantic injected text-search error remained a +typed PostgreSQL error (`F0000`) without exposing the extension path and the +session stayed live. + +The final physical runs used `requiresAppGroup=false` and one manifest digest, +`2b16d1e5cb560837ab7e4d7e1a6fdb51a04edd87c86ba76b58ded60dbde45676`. +The second Debug host launch, without reinstall, found first-launch marker +`87bc03d0-bf57-4592-9f12-4fc585953102` before writing distinct marker +`95298002-82e2-4b2c-8b0c-73cc8c8de54f`. Under run token +`device-lifecycle-20260810T055023Z-64408`, Release lifecycle launch 2 likewise +found launch 1 marker `device-lifecycle-20260810T055023Z-64408:1` before writing +`device-lifecycle-20260810T055023Z-64408:2`. + +While quiesced, each Release launch recursively audited the worker-owned root. +Each found 2,001 entries: 72 directories and 1,929 regular files. Launch 1's +regular files totaled 185,697,132 bytes; launch 2's totaled 185,705,324 bytes. +All 2,001 entries in each launch reported Class-C +`NSFileProtectionCompleteUntilFirstUserAuthentication`; there were zero missing, +mismatched, unavailable, unreadable, symlink, or other entries. It found 924 +relation files and four WAL files. Launch 1's newest relation/WAL modification +times were `1786341320146572032`/`1786341322336570368` nanoseconds since the Unix +epoch; launch 2's were +`1786341436791105280`/`1786341439035651328`. Each launch also populated an +8,192-row, 46,014,464-byte relation before the audit. This proves recursive +Class-C metadata and fresh relation/WAL files after first unlock. It does not +prove access while locked before the device's first unlock after boot. + +Simulator limitation: CoreSimulator resolved `extensionPrivate` to the selected +device's global `data/Library/Application Support/Oliphaunt/default`, not a +plugin/app container. The simulator runner therefore has a simulator-only, +fail-closed quarantine helper: it validates the exact canonical simulator data +root, rejects symlink ancestry, terminates and observes both target processes, +takes a root lock, retains the recognized v2 root, and atomically quarantines +only the recognized v1 manifest. The current report says +`status=retained-current`. +Consequently, the simulator proves persistence and process ownership but does +**not** prove a genuinely extension-private container. The physical +`requiresAppGroup=false` persistence and recursive protection evidence closes +that simulator-only gap for the tested post-first-unlock device state; the +simulator quarantine remains an explicit test-harness limitation. + +CoreSimulator also leaves File Protection metadata unavailable to this code +path. The storage preflight still rejects traversal or enumeration failure, +symlinks, and unsupported entry types in simulator builds, but `WorkerCore` +enforces the strict recursive all-entries-match-Class-C postcondition only on a +physical iOS device. The signed physical recursive audit above supplies that +protection evidence; the simulator lanes do not. + +## 11. Memory measurements by extension state + +The extension samples `TASK_VM_INFO` for physical footprint and resident bytes. +Because the dylib is eagerly linked in the extension, the earliest truthful +label is `extensionEntryPreOpen`, not “before native load.” + +| Phase | Epoch | Worker PID | Entry pre-open phys / resident | Opened-idle phys / resident | Current phys / resident | +| --- | --- | ---: | ---: | ---: | ---: | +| `openedIdle` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 21,333,984 / 137,461,760 | +| `streaming` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,709,664 / 142,442,496 | +| `simultaneousHandles` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,037,920 / 143,818,752 | +| `fifoActive` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,070,688 / 143,851,520 | +| `fifoQueued` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,070,688 / 143,851,520 | +| `fifoDrained` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,103,456 / 143,769,600 | +| `executing` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,103,456 / 143,785,984 | +| `afterCheckpoint` | `fff65828…` | `55318` | 16,271,304 / 119,373,824 | 22,497,248 / 137,084,928 | 23,103,456 / 143,785,984 | +| `sameRootReopen` | `8e1711b6…` | `55318` | 23,103,480 / 144,605,184 | 23,119,864 / 144,588,800 | 23,169,016 / 144,637,952 | +| `postCommitRecovery` | `facac562…` | `55384` | 16,271,304 / 119,308,288 | 22,382,560 / 136,888,320 | 21,465,056 / 137,592,832 | +| `preCommitRecovery` | `2ce33ccd…` | `55393` | 16,304,072 / 119,341,056 | 22,530,016 / 136,986,624 | 21,530,592 / 137,691,136 | + +Initial open increased sampled physical footprint by 6,225,944 bytes and +resident memory by 17,711,104 bytes. Peak sampled current physical footprint was +23,709,664 bytes during streaming; peak sampled current resident memory was +144,637,952 bytes after same-root reopen. `availableMemoryBytes` was reported as +zero by CoreSimulator, so this run does not establish an iOS extension jetsam +limit or headroom. `logicalDetach` has no memory values. + +### Signed Debug device semantic runs + +| Launch | Initial entry phys / resident | Initial opened-idle phys / resident | Peak current phys / resident | Minimum available | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 3,195,560 / 15,384,576 | 12,960,472 / 38,715,392 | 11,911,896 / 44,417,024 | 24,788,264 | +| 2 | 3,244,736 / 15,400,960 | 10,207,960 / 32,342,016 | 11,731,672 / 39,239,680 | 24,968,488 | + +These are Debug semantic samples; the Release lifecycle measurements below are +the memory/headroom gate. + +### Signed Release device lifecycle, launch 1 + +Host PID was `6629`; initial worker/epoch were +`6631`/`c7ccd904-c4e9-4ef7-b88f-2adc44c7002e`; resumed worker/epoch were +`6637`/`7f745e57-09b4-413c-8c17-a9a9489db290`. + +| Phase | Worker | Entry phys / resident | Opened-idle phys / resident | Current phys / resident | Available | +| --- | ---: | ---: | ---: | ---: | ---: | +| `openedIdle` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 8,798,936 / 31,752,192 | 27,901,224 | +| `foregroundIdle` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,631,064 / 40,550,400 | 21,069,096 | +| `executingBeforeCancel` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,631,064 / 40,550,400 | 21,069,096 | +| `afterCancel` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,598,296 / 40,583,168 | 21,101,864 | +| `slowStreaming8MiB` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 19,792,600 / 44,924,928 | 16,907,560 | +| `slowStreaming32MiB` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,581,912 / 45,121,536 | 21,118,248 | +| `checkpointMemorySample` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,549,144 / 45,154,304 | 21,151,016 | +| `afterCheckpoint` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,549,144 / 45,154,304 | 21,151,016 | +| `quiesced` | 6631 | 3,179,176 / 14,942,208 | 10,093,272 / 31,604,736 | 15,565,528 / 45,187,072 | 21,134,632 | +| `resumed` | 6637 | 3,113,640 / 14,942,208 | 10,011,328 / 31,571,968 | 9,011,904 / 31,752,192 | 27,688,256 | + +### Signed Release device lifecycle, launch 2 + +Host PID was `6638`; initial worker/epoch were +`6640`/`fae15a8c-54b3-44ff-9fe5-ac6333b08674`; resumed worker/epoch were +`6642`/`726b89c8-351f-4b2a-bc67-a90b32aac829`. + +| Phase | Worker | Entry phys / resident | Opened-idle phys / resident | Current phys / resident | Available | +| --- | ---: | ---: | ---: | ---: | ---: | +| `openedIdle` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 8,979,136 / 32,423,936 | 27,721,024 | +| `foregroundIdle` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,221,440 / 40,681,472 | 21,478,720 | +| `executingBeforeCancel` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,221,440 / 40,681,472 | 21,478,720 | +| `afterCancel` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,205,056 / 40,730,624 | 21,495,104 | +| `slowStreaming8MiB` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 19,350,208 / 45,072,384 | 17,349,952 | +| `slowStreaming32MiB` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,237,824 / 45,236,224 | 21,462,336 | +| `checkpointMemorySample` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,237,824 / 45,268,992 | 21,462,336 | +| `afterCheckpoint` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,237,824 / 45,268,992 | 21,462,336 | +| `quiesced` | 6640 | 3,195,584 / 14,974,976 | 10,289,856 / 32,309,248 | 15,237,824 / 45,285,376 | 21,462,336 | +| `resumed` | 6642 | 3,113,640 / 14,942,208 | 10,011,352 / 31,588,352 | 8,782,552 / 31,784,960 | 27,917,608 | + +All checkpoint rows reported `checkpointInProgress=false` when sampled. The +exact checkpoint timing/control evidence was: + +| Launch | Sequence | Started uptime ns | Sampled uptime ns | Completed uptime ns | In progress after completion | Background prepare elapsed ns / checkpointed | +| ---: | ---: | ---: | ---: | ---: | --- | --- | +| 1 | 1 | 78524074856375 | 78524074859375 | 78524154186166 | `false` | 775394500 / `true` | +| 2 | 1 | 78640716478625 | 78640716481375 | 78640788016458 | `false` | 729781125 / `true` | + +The column is the exact `checkpointInProgressAfterCompletion` value; `false` +means no checkpoint remained in progress. + +Slow-reader and protocol performance were: + +| Launch | RTT median (20 samples) | 8 MiB bytes / chunks / active samples / elapsed ns | 32 MiB bytes / chunks / active samples / elapsed ns | Throughput B/s | Peak phys / minimum available | +| ---: | ---: | --- | --- | ---: | ---: | +| 1 | 2.264 ms | 8,399,927 / 3,078 / 2,783 / 19,948,094,708 | 33,599,543 / 12,294 / 11,669 / 79,851,108,584 | 420,777 | 19,792,600 / 16,907,560 | +| 2 | 2.424 ms | 8,399,927 / 3,078 / 2,786 / 19,957,112,500 | 33,599,543 / 12,294 / 11,695 / 79,936,360,542 | 420,328 | 19,350,208 / 17,349,952 | + +The declared queue ceiling and required available-memory headroom were both +8,388,608 bytes. The validator allowed at most 16,777,216 bytes of footprint +growth when response size increased by 25,199,616 bytes; both launches measured +`slowStreamFootprintDeltaBytes=0`. The minimum available values above remained +greater than the required 8 MiB. These are sampled values on one device, not a +universal jetsam limit. + +The `smallMobile` profile uses `shared_buffers=8MB`, `wal_buffers=256kB`, +`min_wal_size=32MB`, `max_wal_size=64MB`, `work_mem=1MB`, and +`maintenance_work_mem=16MB`. Independent transport bounds are 256 KiB frames, +an 8 MiB shared input budget, an 8 MiB default raw collector, a 4 MiB native +stream queue, and a 512 KiB target socket send buffer. The final Release run +passed its slow-reader sampling, OS available-memory, bounded-footprint, and +8 MiB headroom assertions. + +## 12. Capability JSON containing only proven values + +```json +{ + "mode": "nativeBroker", + "implementation": "iosExtensionBroker", + "minimumOS": "iOS 26", + "processIsolated": true, + "crashRestartable": true, + "hangRestartable": false, + "sameRootLogicalReopen": true, + "rootSwitchable": false, + "multiRoot": false, + "independentSessions": false, + "maxClientSessions": 1, + "backgroundContinuable": false, + "requiresAppGroup": false, + "protocolRaw": true, + "protocolStream": true, + "streamingRequestInput": false, + "queryCancel": true, + "backupRestore": false, + "connectionString": null, + "serverMode": false +} +``` + +| Positive field | Current evidence | Scope | +| --- | --- | --- | +| `processIsolated` | Simulator lanes had distinct PIDs; Debug hosts `6615`/`6621` and Release hosts `6629`/`6638` differed from every worker | Simulator- and device-proven | +| `crashRestartable` | Seven simulator crash recoveries; four Debug device crash recoveries; two Release resumes after unavailable workers, all with fresh epochs/PIDs and healthy state | Proven for the listed crash and physical lifecycle-loss cases | +| `sameRootLogicalReopen` | Simulator and both Debug device launches changed epoch while retaining worker PID and manifest/root state | Simulator- and device-proven | +| `protocolRaw` | SELECT, DDL, writes, parameters, transactions, SQL errors, restricted-role probes, vector, and pg_trgm in simulator and both Debug device launches | Simulator- and device-proven | +| `protocolStream` | Simulator and each Debug launch streamed 2,119,735 bytes in 4,358 chunks; each Release launch completed 8,399,927- and 33,599,543-byte slow-reader responses within bounds | Simulator- and device-proven | +| `queryCancel` | Raw simulator race proved `57014` + `completed`; both Debug and both Release device launches passed cancellation/liveness | Simulator- and device-proven | + +The false values are deliberate v1 limits. The canonical simulator and +dedicated physical hang lanes directly support advertising +`hangRestartable=false`: their immediate public replacement attempts did not +recover. The later DEBUG mechanism experiment proves recovery is possible once +teardown completes or the old process is made unambiguously unavailable, but +even 60/60 actor-block and 60/60 native-call fail-stop trials on one physical +stack do not establish a shipping reliability contract. The actual Release +background transitions quiesced and closed admission, used no keepalive, and +recovered on foreground, +directly supporting `backgroundContinuable=false`. No capability implies +indefinite background execution, root switching, multiple physical sessions, +backup/restore, or server semantics. + +## 13. Any required additive C ABI changes + +No C ABI change was needed for the implemented request/response broker. +`oliphaunt_exec_protocol_stream` accepts a complete frontend request and streams +backend chunks. The hard-ceiling queue split is internal and preserves ABI 6. + +The existing whole-archive backup/restore APIs remain unsuitable for an +extension memory envelope. Before `backupRestore=true`, add versioned callback +symbols while retaining ABI 6 and all existing entry points, conceptually: + +```c +typedef int32_t (*OliphauntArchiveWriteCallback)( + void *context, + const uint8_t *bytes, + size_t length +); + +typedef int32_t (*OliphauntArchiveReadCallback)( + void *context, + uint8_t *buffer, + size_t capacity, + size_t *bytes_read +); + +OLIPHAUNT_API int32_t oliphaunt_backup_stream( + OliphauntHandle *handle, + const OliphauntBackupStreamOptions *options, + OliphauntArchiveWriteCallback callback, + void *callback_context +); + +OLIPHAUNT_API int32_t oliphaunt_restore_stream( + const OliphauntRestoreStreamOptions *options, + OliphauntArchiveReadCallback callback, + void *callback_context +); +``` + +Backup should stream to a transferred temporary-file FD, fsync, and return byte +count/SHA-256/format/runtime metadata. Restore must validate incrementally into +staging, reject traversal, links, special files, duplicates, truncation, and +version mismatch, fsync, and replace the root only after full validation. The +current archive control boundary rejects the operation rather than pretending it +is bounded. + +## 14. Remaining technical blockers + +The exact-source simulator matrix, both canonical physical reports, and the +dedicated physical hang report are complete: + +| Lane | Checks | Host / initial worker | Initial epoch | Completed UTC | Result | +| --- | ---: | --- | --- | --- | --- | +| semantic | 33 | `55305` / `55318` | `fff65828-f5af-41d5-b155-8644580e3d31` | `2026-08-10T05:30:54Z` | PASS | +| handshake negatives | 7 | `56270` / `56279` | `7637aa73-66f4-4dc9-b613-6b5fc69d210c` | `2026-08-10T05:32:17Z` | PASS | +| extended faults | 10 | `56963` / `56972` | `d2700924-64d9-4f7e-b455-34d6dd83c35c` | `2026-08-10T05:33:45Z` | PASS | +| hang | 5 | `57602` / `57611` | `9f3733c2-5f70-4e2c-98ef-7d4771891c4d` | `2026-08-10T05:35:14Z` | PASS, conservative no-restart outcome | + +| Physical run | Checks | Host / initial worker | Initial epoch | Result | +| --- | ---: | --- | --- | --- | +| Debug semantic launch 1 | 33 | `6615` / `6617` | `e7da4ad4-f4ae-41e6-a3ab-be9ef6e86d09` | PASS | +| Debug semantic launch 2 | 33 | `6621` / `6623` | `719ecba3-56cc-4cbc-9ae6-aee96baf30e5` | PASS, prior launch marker present without reinstall | +| Release lifecycle launch 1 | 30 | `6629` / `6631` | `c7ccd904-c4e9-4ef7-b88f-2adc44c7002e` | PASS, resumed as worker `6637` at fresh epoch `7f745e57-09b4-413c-8c17-a9a9489db290` | +| Release lifecycle launch 2 | 30 | `6638` / `6640` | `fae15a8c-54b3-44ff-9fe5-ac6333b08674` | PASS, resumed as worker `6642` at fresh epoch `726b89c8-351f-4b2a-bc67-a90b32aac829` | +| Debug deliberate hang | 5 | `7110` / `7112` | `69502c5d-eb5c-4f2c-842c-930060230881` | Evidence PASS; replacement attempted, no fresh worker, recovery not proven | + +The subsequent physical mechanism matrix is deliberately separate from the +canonical qualification table. Public recreation failed at 0 and 50 ms and +recovered at 100 and 250 ms in one trial per value. DEBUG private-unique, +private-terminate, combined-private, and extension fail-stop controls each +obtained a fresh PID, epoch, Ready generation, and healthy query at zero delay. + +The remaining blockers and explicit limits are: + +1. **The suspension evidence does not identify why either worker disappeared.** + Each worker existed at foreground inventory and was absent at post-suspend + inventory. The exact loss window is after quiesced evidence through that + inventory. No intentional `SIGKILL` was delivered. Fresh PID-and-epoch + recovery is proven; OS termination cause and timing within that window are + not. +2. **Physical memory-warning injection was unavailable.** The CoreDevice request + returned POSIX `ENOENT` (2). Memory-warning injection is outside the canonical + lifecycle gate, and no synthetic in-process notification was substituted or + claimed. The passing physical gate instead covers real sampled available + memory, bounded slow-reader growth, and its required 8 MiB headroom. +3. **Class-C evidence has a lock-state boundary.** The recursive physical audit + proves protection metadata and newly written relation/WAL files after first + unlock. It does not prove locked access before the first unlock after boot. +4. **Immediate public hang replacement is unreliable; a shipping recovery + policy remains unproven.** `hangRestartable=false` is deliberately + conservative product behavior. The iOS 26.4 simulator and the first iOS + 26.5 physical lane both stayed responsive, failed closed, invalidated the old + epoch, and made an immediate replacement attempt without obtaining Ready. + The follow-up physical experiment then demonstrated stale active-channel + state at 0 and 50 ms, but a fresh healthy worker at 100 and 250 ms. This + strongly identifies asynchronous teardown/process reuse as the immediate + failure mechanism, while also proving it is not a universal inability to + restart. The measured boundary is not an API guarantee. The repeated DEBUG + fail-stop controls now cover clean-lifetime actor and native-call recovery on + this one stack, but a bounded progress-sensitive policy still needs Release, + suspension, data/resource-integrity, and broader device/OS testing before the + capability can truthfully become `true`. +5. **Backup/restore is unavailable.** The additive bounded archive ABI and its + interruption tests do not exist. +6. **Direct-mode comparison and broader lifecycle coverage remain.** The stable + native lease exists and different-root rejection passed, but no current run + compares direct and broker modes or exercises an end-to-end + direct-runtime-versus-broker collision. Reboot, update, eviction, + locked-before-first-unlock access, and long-duration durability also remain + untested. +7. **Distribution is outside this spike.** The Release archive was development + signed, installed, and launched directly. Archive export, distribution + signing, TestFlight, App Store review, production entitlements, and the full + supported device/OS matrix were not run. + +`NativeServer` is intentionally unavailable, not a missing broker feature. The +iOS 26 floor, one canonical root, one physical session, no root switching, no +background keepalive, and no connection string are explicit v1 scope. + +## 15. Verdict + +**VIABLE WITH LIMITATIONS** + +The exact-source iOS 26.4 simulator matrix establishes the core architecture: +a genuine host/extension process boundary; extension-only native linkage; +negotiated FD transport; bounded framing, input, raw collection, streaming, and +privacy filtering; restricted non-superuser SQL that survives RESET and DISCARD; +typed PostgreSQL errors; cancellation proven by raw SQLSTATE `57014` followed by +`ReadyForQuery` and `completed`; conservative ambiguous-outcome semantics with +no replay; same-root recovery after seven injected crash paths; and truthful +failure to restart a deliberately hung worker. All 55 assertions passed. + +The canonical wired `iPhone15,2` reports add two 33-check signed Debug semantic +launches, a retained-Debug signature-and-hash evidence chain, persistence without +reinstall, and two 30-check signed Release lifecycle launches. They prove the +inspected archive can install and launch; +separate host/worker processes; cancellation and liveness; exact bounded +slow-reader memory/headroom behavior; checkpoint and quiesce; recursive Class-C +protection plus fresh relation/WAL files; actual background transitions; worker +absence at post-suspend inventory; and healthy persistence recovery with fresh +worker PIDs and epochs. Both canonical physical reports passed. + +The separate final physical hang lane used that exact retained signed Debug +artifact and independently passed its evidence validator. It did not recover: +launch attempts advanced from one to two while successful Ready generations +stayed at one, and it published no recovered PID or epoch. This matches the +simulator's conservative immediate-retry result and supports keeping +`hangRestartable=false` without treating the generic fixture PASS as recovery. +The later direct-device mechanism matrix explains rather than contradicts that +result: public recreation reached stale active-channel state at 0 and 50 ms, +then produced fresh healthy workers at 100 and 250 ms. Unsupported private +terminate/unique-instance controls and a DEBUG extension fail-stop watchdog +also recovered immediately. Recovery is therefore technically possible, but +its public, production-safe reliability policy remains unqualified. + +The Release archive is development signed, not an exported distribution +artifact. The workers' disappearance is bounded but causally unattributed, and +no intentional `SIGKILL` was delivered. Physical memory-warning injection was +unavailable and is not claimed. The storage audit does not cover +locked-before-first-unlock access. Immediate deliberate-hang recovery was not +obtained in the canonical simulator or physical lane; the follow-up device +matrix obtained it after a short public teardown delay and through diagnostic +force-fresh controls. The final DEBUG controls then repeated actor-block and +native-call fail-stop recovery 60/60 times each on the same physical stack, +establishing mechanism repeatability but not a production policy or SLA. +Backup/restore, direct-mode comparison and contention, extended +lifecycle/device coverage, TestFlight, and App Store qualification also remain +unproved. CoreSimulator's global root and quarantine helper remain +simulator-only limitations. The evidence therefore supports the stated +feasibility verdict, not production readiness. diff --git a/docs/internal/OLIPHAUNT_PATCH_STACK.md b/docs/internal/OLIPHAUNT_PATCH_STACK.md index 0e2e1f66..5bf0d405 100644 --- a/docs/internal/OLIPHAUNT_PATCH_STACK.md +++ b/docs/internal/OLIPHAUNT_PATCH_STACK.md @@ -40,6 +40,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write | 18 | `0018-liboliphaunt-contain-embedded-proc-signals.patch` | liboliphaunt | liboliphaunt: contain embedded process signals | | 19 | `0019-liboliphaunt-link-windows-embedded-modules-to-host.patch` | liboliphaunt | liboliphaunt: link Windows embedded modules to host | | 20 | `0020-liboliphaunt-enforce-embedded-signal-boundary.patch` | liboliphaunt | liboliphaunt: enforce embedded signal boundary | +| 21 | `0021-liboliphaunt-authenticate-embedded-role.patch` | liboliphaunt | liboliphaunt: authenticate embedded role | ## Changed Upstream Files @@ -48,6 +49,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write - `src/backend/access/transam/xlogarchive.c` (`0007-liboliphaunt-disable-shell-commands-on-apple-mobile.patch`) - `src/backend/archive/shell_archive.c` (`0007-liboliphaunt-disable-shell-commands-on-apple-mobile.patch`) - `src/backend/commands/event_trigger.c` (`0012-liboliphaunt-enable-event-triggers-in-embedded-backend.patch`) +- `src/backend/commands/variable.c` (`0021-liboliphaunt-authenticate-embedded-role.patch`) - `src/backend/libpq/be-secure.c` (`0001-liboliphaunt-add-backend-host-io.patch`) - `src/backend/libpq/pqcomm.c` (`0001-liboliphaunt-add-backend-host-io.patch`) - `src/backend/meson.build` (`0019-liboliphaunt-link-windows-embedded-modules-to-host.patch`) @@ -59,6 +61,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write - `src/backend/storage/ipc/procsignal.c` (`0018-liboliphaunt-contain-embedded-proc-signals.patch`) - `src/backend/tcop/postgres.c` (`0002-liboliphaunt-add-embedded-entrypoint.patch`, `0003-liboliphaunt-return-from-embedded-frontend-terminate.patch`, `0004-liboliphaunt-run-embedded-exit-cleanup.patch`, `0005-liboliphaunt-restore-host-cwd.patch`, `0009-liboliphaunt-guard-embedded-proc-exit.patch`, `0010-liboliphaunt-use-host-runtime-paths.patch`, `0014-liboliphaunt-use-portable-embedded-socketpair.patch`, `0018-liboliphaunt-contain-embedded-proc-signals.patch`) - `src/backend/utils/fmgr/dfmgr.c` (`0006-liboliphaunt-add-static-extension-loader.patch`, `0008-liboliphaunt-clean-embedded-symbols.patch`) +- `src/backend/utils/init/postinit.c` (`0021-liboliphaunt-authenticate-embedded-role.patch`) - `src/bin/initdb/initdb.c` (`0016-liboliphaunt-skip-icu-collation-version-without-icu-data.patch`) - `src/include/libpq/libpq-be.h` (`0001-liboliphaunt-add-backend-host-io.patch`) - `src/include/port.h` (`0011-liboliphaunt-add-android-embedded-shared-memory.patch`, `0020-liboliphaunt-enforce-embedded-signal-boundary.patch`) @@ -66,6 +69,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write - `src/include/storage/ipc.h` (`0004-liboliphaunt-run-embedded-exit-cleanup.patch`, `0009-liboliphaunt-guard-embedded-proc-exit.patch`) - `src/include/tcop/backend_startup.h` (`0013-liboliphaunt-fix-embedded-backend-main-return-contract.patch`) - `src/include/tcop/tcopprot.h` (`0003-liboliphaunt-return-from-embedded-frontend-terminate.patch`, `0008-liboliphaunt-clean-embedded-symbols.patch`) +- `src/include/utils/guc_hooks.h` (`0021-liboliphaunt-authenticate-embedded-role.patch`) - `src/include/utils/hsearch.h` (`0017-liboliphaunt-namespace-dynahash-host-collisions.patch`) - `src/port/chklocale.c` (`0011-liboliphaunt-add-android-embedded-shared-memory.patch`) - `src/port/pqsignal.c` (`0020-liboliphaunt-enforce-embedded-signal-boundary.patch`) @@ -79,6 +83,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write | `src/backend/access/transam/xlogarchive.c` | Apple mobile embedded builds compile out optional archive shell commands. | | `src/backend/archive/shell_archive.c` | Apple mobile embedded builds compile out optional archive shell commands. | | `src/backend/commands/event_trigger.c` | Embedded FE/BE protocol sessions can run event triggers without changing standalone recovery behavior. | +| `src/backend/commands/variable.c` | Oliphaunt session-authorization assignments monotonically latch an observed authenticated-role demotion without catalog access during transaction cleanup. | | `src/backend/libpq/be-secure.c` | Backend secure read/write path delegates to a host I/O vtable only when OLIPHAUNT_EMBEDDED is set. | | `src/backend/libpq/pqcomm.c` | Standalone embedded sessions avoid waiting on a non-existent postmaster death latch. | | `src/backend/meson.build` | Embedded MSVC extension modules link to the oliphaunt host import library instead of the standalone postgres executable. | @@ -90,6 +95,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write | `src/backend/storage/ipc/procsignal.c` | The one-backend embedded runtime dispatches ProcSignal flags without sending process-directed host signals. | | `src/backend/tcop/postgres.c` | Embedded backend entrypoint, protocol lifecycle, cwd restoration, host runtime paths, and host-owned SIGUSR1 disposition. | | `src/backend/utils/fmgr/dfmgr.c` | Static extension lookup reuses PostgreSQL dynamic function manager semantics. | +| `src/backend/utils/init/postinit.c` | Oliphaunt host-I/O sessions initialize the immutable authenticated identity from the configured role while ordinary standalone startup remains unchanged. | | `src/bin/initdb/initdb.c` | Base runtimes skip ICU-backed collation setup until optional ICU data is present. | | `src/include/libpq/libpq-be.h` | Host I/O vtable is attached to PostgreSQL Port state under OLIPHAUNT_EMBEDDED. | | `src/include/port.h` | Embedded mobile builds avoid POSIX shared memory declarations and route embedded backend signal calls through the host-safe provider boundary. | @@ -97,6 +103,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write | `src/include/storage/ipc.h` | Embedded cleanup and proc_exit guard declarations. | | `src/include/tcop/backend_startup.h` | Embedded BackendMain may return after its returning PostgresMain call without retaining an invalid pg_noreturn declaration. | | `src/include/tcop/tcopprot.h` | Embedded entrypoint and returning PostgresMain declarations. | +| `src/include/utils/guc_hooks.h` | Declares the Oliphaunt-only per-session authenticated-role latch reset used by InitPostgres. | | `src/include/utils/hsearch.h` | Apple builds namespace PostgreSQL dynahash symbols that otherwise bind to unrelated libSystem exports. | | `src/port/chklocale.c` | Android embedded builds avoid unsupported locale-environment mutation. | | `src/port/pqsignal.c` | Embedded backend signal registration and emission preserve the host-owned SIGUSR1 disposition while delegating other signals. | @@ -141,6 +148,7 @@ src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs --write | Embedded ProcSignal delivery cannot escape into the host process | `0018-liboliphaunt-contain-embedded-proc-signals.patch` | `oliphaunt_send_proc_signal`, `pid != MyProcPid`, `procsignal_sigusr1_handler(SIGUSR1)`, `host owns SIGUSR1` | The one-backend embedded runtime dispatches ProcSignal flags synchronously, rejects foreign PIDs, and leaves the host SIGUSR1 disposition untouched; normal PostgreSQL server builds retain upstream signal delivery. | | Windows embedded extension modules link to the host DLL provider | `0019-liboliphaunt-link-windows-embedded-modules-to-host.patch` | `oliphaunt_embedded_module_provider`, `requires an embedded MSVC Windows build`, `pg_mod_link_args += oliphaunt_embedded_module_provider`, `oliphaunt_embedded_module_provider == ''` | Embedded MSVC extension modules resolve PostgreSQL backend symbols from the oliphaunt host import library; ordinary PostgreSQL modules retain the upstream postgres executable link contract. | | Embedded backend and extension signal calls preserve host SIGUSR1 ownership | `0020-liboliphaunt-enforce-embedded-signal-boundary.patch` | `oliphaunt_embedded_kill`, `oliphaunt_embedded_raise`, `!defined(FRONTEND)`, `if (signo == SIGUSR1)` | Embedded backend and extension calls cannot replace or emit host-owned SIGUSR1; other signals delegate to the platform implementation, while frontend tools and normal PostgreSQL builds retain upstream behavior. | +| Embedded sessions authenticate as the configured database role | `0021-liboliphaunt-authenticate-embedded-role.patch` | `MyProcPort->oliphaunt_io != NULL`, `InitializeSessionUserId(username, useroid, false)`, `!role_form->rolcanlogin`, `is not permitted to log in`, `assign_session_authorization`, `ResetOliphauntAuthenticatedRoleLatch`, `oliphaunt_authenticated_role_is_superuser = false`, `InitializeSessionUserIdStandalone` | Only the Oliphaunt host-I/O backend resolves its immutable authenticated identity from the configured LOGIN role and latches an observed demotion for that session so stale RESET, rollback, and DISCARD state fail closed without catalog work in GUC cleanup; ordinary standalone PostgreSQL keeps bootstrap-superuser recovery semantics. | ## Guardrails diff --git a/spikes/android-native-broker/.gitignore b/spikes/android-native-broker/.gitignore new file mode 100644 index 00000000..d1744b72 --- /dev/null +++ b/spikes/android-native-broker/.gitignore @@ -0,0 +1,3 @@ +.gradle/ +app/build/ +reports/ diff --git a/spikes/android-native-broker/README.md b/spikes/android-native-broker/README.md new file mode 100644 index 00000000..9cdd68aa --- /dev/null +++ b/spikes/android-native-broker/README.md @@ -0,0 +1,136 @@ +# Android native broker spike + +This DEBUG-only spike mirrors the iOS broker experiment on Android with a +private `:broker` service process, an AIDL/Binder control plane, and a reliable +Unix socket-pair file-descriptor data plane. + +The `full` fixture exercises: + +- distinct host and broker PIDs plus a random worker epoch; +- PostgreSQL protocol bytes over the socket data plane; +- cancellation from a Binder thread outside the occupied database executor; +- executor-deadlock and native-output-gated `pg_sleep` fail-stop paths; +- generation-scoped Binder death, `outcomeUnknown`, no SQL replay, and recovery + to a fresh PID and epoch; +- controlled zero-read 8 MiB and 32 MiB streams that directly observe a + blocked synchronous socket write; and +- persistent data and an ambiguous committed counter across injected worker + deaths. + +It is an experiment, not a production broker. Fault hooks are DEBUG-only, and +an emulator result is not physical-device evidence. + +## Retained evidence + +The final ten-run behavior series is: + +```text +target/android-native-broker-spike/runs/pr-final-01-20260811T121603Z/ +through +target/android-native-broker-spike/runs/pr-final-10-20260811T121854Z/ +``` + +`target/android-native-broker-spike/runs/pr-doc-sync-v1/` is the post-document +single-run confirmation whose source manifest includes this final README. The +ten-run series used identical executable inputs and differs only in the README +evidence-path text. + +All ten API 34 arm64 emulator runs passed 11 checks. They produced 30 injected +worker deaths, 30 generation-scoped Binder-death observations, 30 +`outcomeUnknown` terminals, and 30 fresh PID/epoch recoveries followed by +healthy SQL. All 10 host PIDs and all 40 worker PIDs and epochs were unique in +the retained series. + +Each native fault recorded 4,202,496 bytes from the native PostgreSQL stream +for an ordered output-then-`pg_sleep(60)` query before arming a two-second +fail-stop watchdog. This is strong source-backed sequencing evidence, not a +callback from inside `pg_sleep`. + +The ambiguous counter was one after recovery and derived `replayCount` was zero +in every run. That proves no replay for the instrumented mutation, not a +generic exactly-once protocol. + +For both stream sizes, a host-controlled gate prevented response reads while +two diagnostics samples observed the same blocking `responseBytes` write with +`POLLOUT=false` and unchanged completion counters for at least 300 ms. The +conservative pre-read accepted-wire upper bound was 493,920 bytes for every +8 MiB and 32 MiB trial; after releasing the gate, the same generation drained +the full response. This proves synchronous socket backpressure for this +workload. It does not establish a process-memory bound, effective `SO_SNDBUF`, +or throughput SLA. + +Two earlier failed attempts are retained as negative evidence and are not part +of the passing series: + +- `final-witness-001-20260811T113240Z` showed that a small first-statement + `CommandComplete` remained buffered until after the sleeping statement, so it + was not a usable pre-hang witness. +- `final-output-witness-006-20260811T115345Z` showed that the first transient + non-writable socket write could advance before the client read. The final + probe therefore keeps an explicit read gate closed and resets its candidate + until one write remains unchanged for at least 300 ms. + +See [the architecture and evidence report](../../docs/architecture/android-native-broker-spike.md) +for exact ranges, artifact hashes, and the proven/unproven split. + +## Prerequisites + +- Android SDK with the API 34 `Pixel_9_API_34_Google_API` arm64 AVD; +- JDK 17, NDK `27.0.12077973`, and CMake `3.22.1`; +- a current Android arm64 `liboliphaunt.so`; and +- prepared mobile runtime resources containing a PostgreSQL 18 template + `PGDATA`. + +The defaults are: + +```text +target/android-native-broker-spike/native/out/liboliphaunt.so +target/android-native-broker-spike/runtime-resources +``` + +Override them with `OLIPHAUNT_ANDROID_BROKER_LIBOLIPHAUNT_SO` and +`OLIPHAUNT_ANDROID_BROKER_RUNTIME_RESOURCES_DIR`. + +## Run + +```sh +bash spikes/android-native-broker/run-emulator.sh +``` + +The runner: + +1. requires the canonical Android native `--check-current` gate; +2. records exact source, APK, and native-library hashes; +3. builds the Debug APK; +4. starts or reuses only the API 34 arm64 AVD; +5. installs and clears the app once; +6. launches the full fixture and rejects stale reports by run nonce; and +7. validates the JSON contract, exact worker crash PIDs, process transitions, + Binder deaths, native-output witness, replay counter, persistence, and + socket-stall arithmetic. + +Evidence is written beneath: + +```text +target/android-native-broker-spike/runs// +``` + +## Claim boundaries + +- The service is a separate private app process, not an Android + `isolatedProcess`; it shares the app UID and private storage. +- Binder death is the explicit process-death signal. Reliable-socket EOF drives + the same interruption path but does not identify why the peer disappeared. +- Recovery requires a different PID and epoch plus healthy SQL. +- Once request bytes may have reached the worker, loss is `outcomeUnknown` and + the host does not retry SQL. +- The native output witness proves ordered PostgreSQL execution immediately + before the sleeping plan child, but not a direct stack observation from + inside `pg_sleep`. +- `pm clear` runs before each complete matrix, never between a fault and its + recovery check. +- PSS/RSS and drain rates are observations, not acceptance limits. +- The retained result covers one API 34 arm64 emulator image. It does not prove + physical-device behavior, other Android/OEM versions, broad reliability, + lifecycle/Doze/LMK behavior, power-loss durability, concurrency, + security isolation, a production watchdog policy, or Release/Play readiness. diff --git a/spikes/android-native-broker/app/build.gradle.kts b/spikes/android-native-broker/app/build.gradle.kts new file mode 100644 index 00000000..832044f1 --- /dev/null +++ b/spikes/android-native-broker/app/build.gradle.kts @@ -0,0 +1,46 @@ +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.kotlin.android) +} + +android { + namespace = "dev.oliphaunt.androidbrokerspike" + compileSdk = 36 + + defaultConfig { + applicationId = "dev.oliphaunt.androidbrokerspike" + minSdk = 24 + targetSdk = 36 + versionCode = 1 + versionName = "0.1" + + testInstrumentationRunner = "android.test.InstrumentationTestRunner" + } + + buildFeatures { + aidl = true + buildConfig = true + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + packaging { + jniLibs { + useLegacyPackaging = true + } + } +} + +kotlin { + jvmToolchain(17) +} + +dependencies { + implementation(project(":oliphaunt")) + implementation(libs.kotlinx.coroutines.android) + implementation(libs.kotlinx.coroutines.core) + testImplementation(kotlin("test")) +} diff --git a/spikes/android-native-broker/app/src/main/AndroidManifest.xml b/spikes/android-native-broker/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..bb40ec17 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/AndroidManifest.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + diff --git a/spikes/android-native-broker/app/src/main/aidl/dev/oliphaunt/androidbrokerspike/IOliphauntBroker.aidl b/spikes/android-native-broker/app/src/main/aidl/dev/oliphaunt/androidbrokerspike/IOliphauntBroker.aidl new file mode 100644 index 00000000..4f058beb --- /dev/null +++ b/spikes/android-native-broker/app/src/main/aidl/dev/oliphaunt/androidbrokerspike/IOliphauntBroker.aidl @@ -0,0 +1,10 @@ +package dev.oliphaunt.androidbrokerspike; + +import android.os.Bundle; +import android.os.ParcelFileDescriptor; + +/** Minimal experimental control plane. Bulk protocol bytes use dataChannel. */ +interface IOliphauntBroker { + Bundle hello(in Bundle request, in ParcelFileDescriptor dataChannel); + Bundle control(in Bundle request); +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerClient.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerClient.kt new file mode 100644 index 00000000..44471662 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerClient.kt @@ -0,0 +1,990 @@ +package dev.oliphaunt.androidbrokerspike + +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.content.ServiceConnection +import android.os.Bundle +import android.os.DeadObjectException +import android.os.IBinder +import android.os.ParcelFileDescriptor +import android.os.Process +import android.os.RemoteException +import android.os.SystemClock +import android.system.ErrnoException +import android.system.Os +import android.system.OsConstants +import java.io.ByteArrayOutputStream +import java.io.EOFException +import java.io.IOException +import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +internal data class BrokerReady( + val epoch: UUID, + val workerPid: Int, + val protocolVersion: Int, + val runtimeVersion: String?, + val abiVersion: Long?, + val postgresMajorVersion: Int?, + val rootManifestDigest: String?, +) + +internal data class BrokerDiagnostics( + val epoch: UUID, + val workerPid: Int, + val state: String?, + val activeRequestId: Long?, + val nativeDispatchStarted: Boolean, + val nativePostgresOutputWitnessObserved: Boolean, + val nativePostgresOutputWitnessRequestId: Long?, + val nativePostgresOutputWitnessBackendBytes: Long?, + val nativePostgresOutputWitnessElapsedRealtimeNanos: Long?, + val transactionStatus: String?, + val currentPssBytes: Long?, + val currentRssBytes: Long?, + val socketSendBufferBytes: Int?, + val sampleElapsedRealtimeNanos: Long?, + val socketNonBlockingProbeSucceeded: Boolean, + val socketNonBlocking: Boolean?, + val socketPollSucceeded: Boolean, + val socketWritableNow: Boolean?, + val socketWriteInProgress: Boolean, + val socketActiveWriteSequence: Long?, + val socketActiveWriteRequestId: Long?, + val socketActiveWriteFrameType: OlpbFrameType?, + val socketActiveWriteStartedElapsedRealtimeNanos: Long?, + val socketActiveWriteEncodedBytes: Int?, + val socketWritesCompleted: Long?, + val socketCompletedEncodedBytes: Long?, +) + +internal data class BrokerExecutionStats( + val epoch: UUID, + val workerPid: Int, + val requestId: Long, + val responseBytes: Long, + val responseChunks: Int, + val transactionStatus: Byte, +) + +internal data class BrokerDeath( + val epoch: UUID, + val workerPid: Int, + val reason: String, + val observedAtElapsedRealtimeNanos: Long, +) + +private data class BinderDeathEvent( + val epoch: UUID, + val workerPid: Int, + val observedAtElapsedRealtimeNanos: Long, +) + +internal open class BrokerClientException( + message: String, + cause: Throwable? = null, +) : Exception(message, cause) + +internal class BrokerRejectedException(message: String) : BrokerClientException(message) + +internal class BrokerOutcomeUnknownException( + val epoch: UUID, + val requestId: Long, + cause: Throwable? = null, +) : BrokerClientException( + "request outcome is unknown (epoch $epoch, request ${java.lang.Long.toUnsignedString(requestId)})", + cause, + ) + +/** + * Experimental host for one remote broker generation. + * + * Connection establishment may be retried. An individual SQL request never is: + * after the first socket write attempt, every transport or protocol loss is + * surfaced as [BrokerOutcomeUnknownException]. + */ +internal class BrokerClient(context: Context) { + private val appContext = context.applicationContext + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val connectionMutex = Mutex() + private val executionMutex = Mutex() + private val stateLock = Any() + private val deathEvents = Channel(Channel.UNLIMITED) + private val binderDeathEvents = Channel(Channel.UNLIMITED) + + private var connection: Connection? = null + private var interruptedIdentity: BrokerReady? = null + private var activeRequest: ActiveRequest? = null + private var nextRequestId = 1L + private var closed = false + + suspend fun connect(): BrokerReady = ensureConnection().ready + + suspend fun reconnect(): BrokerReady = ensureConnection().ready + + suspend fun execute( + request: ByteArray, + slowReadDelayMillis: Long = 0, + ): ByteArray { + val output = ByteArrayOutputStream() + executeStreaming( + request = request, + slowReadDelayMillis = slowReadDelayMillis, + onChunk = { chunk -> + if (output.size().toLong() + chunk.size > MAXIMUM_RAW_RESPONSE_BYTES) { + throw BrokerClientException( + "raw broker response exceeds $MAXIMUM_RAW_RESPONSE_BYTES bytes; " + + "use executeStreaming", + ) + } + output.write(chunk) + }, + ) + return output.toByteArray() + } + + suspend fun execute( + sql: String, + slowReadDelayMillis: Long = 0, + ): ByteArray = execute(BrokerContract.simpleQuery(sql), slowReadDelayMillis) + + suspend fun executeSlowReader( + sql: String, + firstReadRelease: Deferred, + perFrameDelayMillis: Long, + ): BrokerExecutionStats = + executeStreaming( + request = BrokerContract.simpleQuery(sql), + slowReadDelayMillis = perFrameDelayMillis, + firstReadRelease = firstReadRelease, + onChunk = {}, + ) + + suspend fun executeStreaming( + request: ByteArray, + slowReadDelayMillis: Long = 0, + firstReadRelease: Deferred? = null, + onChunk: (ByteArray) -> Unit, + ): BrokerExecutionStats = executionMutex.withLock { + require(slowReadDelayMillis >= 0) { "slow-read delay must not be negative" } + validateFrontendRequest(request) + + // Recovery may happen before admission. Nothing below this point retries. + val generation = ensureConnection() + val requestId = allocateRequestId() + val attempt = RequestAttempt(generation.ready.epoch, requestId) + synchronized(stateLock) { + check(!closed) { "broker client is closed" } + activeRequest = ActiveRequest(generation.token, generation.ready.epoch, requestId) + } + + try { + attempt.bytesMayHaveReachedWorker = true + generation.channel.write( + OlpbFrame( + protocolVersion = generation.ready.protocolVersion, + frameType = OlpbFrameType.REQUEST_BEGIN, + epoch = generation.ready.epoch, + requestId = requestId, + ), + ) + var offset = 0 + while (offset < request.size) { + val end = minOf(request.size, offset + OlpbProtocol.MAXIMUM_FRAME_PAYLOAD) + generation.channel.write( + OlpbFrame( + protocolVersion = generation.ready.protocolVersion, + frameType = OlpbFrameType.REQUEST_BYTES, + epoch = generation.ready.epoch, + requestId = requestId, + payload = request.copyOfRange(offset, end), + ), + ) + offset = end + } + generation.channel.write( + OlpbFrame( + protocolVersion = generation.ready.protocolVersion, + frameType = OlpbFrameType.REQUEST_END, + epoch = generation.ready.epoch, + requestId = requestId, + ), + ) + + val backend = BackendTerminalObserver() + var responseBytes = 0L + var responseChunks = 0 + firstReadRelease?.await() + while (true) { + if (slowReadDelayMillis > 0) delay(slowReadDelayMillis) + val frame = generation.channel.read(generation.ready.epoch) + if (frame.header.frameType == OlpbFrameType.PROTOCOL_ERROR) { + throw OlpbProtocolException( + frame.payload.toString(Charsets.UTF_8).ifEmpty { "worker protocol error" }, + ) + } + if (frame.header.requestId != requestId) { + throw OlpbProtocolException( + "response request ID ${frame.header.requestId} does not match $requestId", + ) + } + when (frame.header.frameType) { + OlpbFrameType.RESPONSE_BYTES -> { + backend.append(frame.payload) + responseBytes += frame.payload.size + responseChunks += 1 + onChunk(frame.payload) + } + + OlpbFrameType.CANCEL_OBSERVED -> { + requireEmptyPayload(frame) + } + + OlpbFrameType.COMPLETED -> { + requireEmptyPayload(frame) + return@withLock BrokerExecutionStats( + epoch = generation.ready.epoch, + workerPid = generation.ready.workerPid, + requestId = requestId, + responseBytes = responseBytes, + responseChunks = responseChunks, + transactionStatus = backend.finish(), + ) + } + + OlpbFrameType.REJECTED -> { + throw BrokerRejectedException( + frame.payload.toString(Charsets.UTF_8).ifEmpty { "worker rejected request" }, + ) + } + + OlpbFrameType.OUTCOME_UNKNOWN -> { + throw BrokerOutcomeUnknownException(generation.ready.epoch, requestId) + } + + else -> { + throw OlpbProtocolException( + "illegal worker frame ${frame.header.frameType} while request is active", + ) + } + } + } + @Suppress("UNREACHABLE_CODE") + throw OlpbProtocolException("response loop ended without a terminal frame") + } catch (error: Throwable) { + when (error) { + is BrokerRejectedException -> throw error + is BrokerOutcomeUnknownException -> { + interrupt(generation, "worker reported outcomeUnknown") + throw error + } + + else -> { + if (attempt.bytesMayHaveReachedWorker) { + interrupt(generation, "request transport failed: ${error.javaClass.simpleName}") + throw BrokerOutcomeUnknownException( + epoch = generation.ready.epoch, + requestId = requestId, + cause = error, + ) + } + throw error + } + } + } finally { + synchronized(stateLock) { + if (activeRequest?.generation == generation.token && + activeRequest?.requestId == requestId + ) { + activeRequest = null + } + } + } + } + + /** Returns true only when the worker says cancellation was already observed. */ + suspend fun cancel(requestId: Long? = null): Boolean { + val target = synchronized(stateLock) { activeRequest } + ?: throw BrokerClientException("there is no active broker request") + if (requestId != null && requestId != target.requestId) { + throw BrokerClientException("request $requestId is not active") + } + val current = currentConnection(target.generation) + ?: throw BrokerOutcomeUnknownException(target.epoch, target.requestId) + val reply = callControl( + current, + BrokerContract.control( + message = BrokerContract.CANCEL, + epoch = target.epoch, + requestId = target.requestId, + ), + ) + return when (val message = reply.getString(BrokerContract.MESSAGE)) { + BrokerContract.CANCEL_OBSERVED -> true + BrokerContract.CANCEL -> false + else -> throw OlpbProtocolException("invalid cancellation reply $message") + } + } + + suspend fun armFault(fault: BrokerFault) { + val current = ensureConnection() + val reply = callControl( + current, + BrokerContract.control( + message = BrokerContract.INJECT_FAULT, + epoch = current.ready.epoch, + fault = fault, + ), + ) + requireReply(reply, BrokerContract.INJECT_FAULT) + } + + suspend fun diagnostics(): BrokerDiagnostics { + val current = ensureConnection() + val reply = callControl( + current, + BrokerContract.control( + message = BrokerContract.DIAGNOSTICS, + epoch = current.ready.epoch, + ), + ) + requireReply(reply, BrokerContract.DIAGNOSTICS) + val epoch = requiredUuid(reply, BrokerContract.EPOCH) + val workerPid = requiredPositiveInt(reply, BrokerContract.WORKER_PID) + if (epoch != current.ready.epoch || workerPid != current.ready.workerPid) { + interrupt(current, "diagnostics identity changed within one generation") + throw OlpbProtocolException("diagnostics identity does not match Ready") + } + return BrokerDiagnostics( + epoch = epoch, + workerPid = workerPid, + state = reply.getString(BrokerContract.STATE), + activeRequestId = reply.optionalLong(BrokerContract.ACTIVE_REQUEST_ID), + nativeDispatchStarted = reply.getBoolean(BrokerContract.NATIVE_DISPATCH_STARTED, false), + nativePostgresOutputWitnessObserved = + reply.getBoolean( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_OBSERVED, + false, + ), + nativePostgresOutputWitnessRequestId = + reply.optionalLong(BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_REQUEST_ID), + nativePostgresOutputWitnessBackendBytes = + reply.optionalLong(BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_BACKEND_BYTES), + nativePostgresOutputWitnessElapsedRealtimeNanos = + reply.optionalLong( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_ELAPSED_REALTIME_NANOS, + ), + transactionStatus = reply.getString(BrokerContract.TRANSACTION_STATUS), + currentPssBytes = reply.optionalLong(BrokerContract.CURRENT_PSS_BYTES), + currentRssBytes = reply.optionalLong(BrokerContract.CURRENT_RSS_BYTES), + socketSendBufferBytes = reply.optionalInt(BrokerContract.SOCKET_SEND_BUFFER_BYTES), + sampleElapsedRealtimeNanos = + reply.optionalLong(BrokerContract.DIAGNOSTICS_SAMPLE_ELAPSED_REALTIME_NANOS), + socketNonBlockingProbeSucceeded = + reply.getBoolean(BrokerContract.SOCKET_NON_BLOCKING_PROBE_SUCCEEDED, false), + socketNonBlocking = reply.optionalBoolean(BrokerContract.SOCKET_NON_BLOCKING), + socketPollSucceeded = + reply.getBoolean(BrokerContract.SOCKET_POLL_SUCCEEDED, false), + socketWritableNow = reply.optionalBoolean(BrokerContract.SOCKET_WRITABLE_NOW), + socketWriteInProgress = + reply.getBoolean(BrokerContract.SOCKET_WRITE_IN_PROGRESS, false), + socketActiveWriteSequence = + reply.optionalLong(BrokerContract.SOCKET_ACTIVE_WRITE_SEQUENCE), + socketActiveWriteRequestId = + reply.optionalLong(BrokerContract.SOCKET_ACTIVE_WRITE_REQUEST_ID), + socketActiveWriteFrameType = + reply.optionalInt(BrokerContract.SOCKET_ACTIVE_WRITE_FRAME_TYPE)?.let { + OlpbFrameType.fromWireValue(it) + }, + socketActiveWriteStartedElapsedRealtimeNanos = + reply.optionalLong( + BrokerContract.SOCKET_ACTIVE_WRITE_STARTED_ELAPSED_REALTIME_NANOS, + ), + socketActiveWriteEncodedBytes = + reply.optionalInt(BrokerContract.SOCKET_ACTIVE_WRITE_ENCODED_BYTES), + socketWritesCompleted = + reply.optionalLong(BrokerContract.SOCKET_WRITES_COMPLETED), + socketCompletedEncodedBytes = + reply.optionalLong(BrokerContract.SOCKET_COMPLETED_ENCODED_BYTES), + ) + } + + suspend fun awaitDeath(timeoutMillis: Long): BrokerDeath { + require(timeoutMillis > 0) { "death timeout must be positive" } + return withTimeout(timeoutMillis) { deathEvents.receive() } + } + + suspend fun awaitBinderDeath(generation: BrokerReady, timeoutMillis: Long): Long { + require(timeoutMillis > 0) { "Binder death timeout must be positive" } + return withTimeout(timeoutMillis) { + while (true) { + val event = binderDeathEvents.receive() + if (event.epoch == generation.epoch && event.workerPid == generation.workerPid) { + return@withTimeout event.observedAtElapsedRealtimeNanos + } + } + error("unreachable") + } + } + + suspend fun close() { + val current = synchronized(stateLock) { + if (closed) return + closed = true + val value = connection + connection = null + activeRequest = null + value + } + if (current != null) { + try { + callControl( + current, + BrokerContract.control(BrokerContract.DETACH, current.ready.epoch), + ) + } catch (_: Throwable) { + // Closing is terminal for this host handle. + } + current.close(appContext) + } + deathEvents.close() + binderDeathEvents.close() + } + + private suspend fun ensureConnection(): Connection = connectionMutex.withLock { + synchronized(stateLock) { + check(!closed) { "broker client is closed" } + connection + }?.let { current -> + if (!current.interrupted.get() && + !current.bound.dead.get() && + current.binder.isBinderAlive + ) { + return@withLock current + } + interrupt(current, "binder was not alive during connection acquisition") + } + + val stale = synchronized(stateLock) { interruptedIdentity } + val token = UUID.randomUUID() + val bound = bind(token) + val sockets = ParcelFileDescriptor.createReliableSocketPair() + val hostEndpoint = sockets[0] + val workerEndpoint = sockets[1] + var channel: HostDataChannel? = null + try { + val reply = withContext(Dispatchers.IO) { + bound.remote.hello(BrokerContract.hello(), workerEndpoint) + } + workerEndpoint.close() + val ready = decodeReady(reply) + bound.setReady(ready) + if (ready.workerPid == Process.myPid()) { + throw OlpbProtocolException("broker service is not running in a separate process") + } + if (stale != null) { + if (ready.epoch == stale.epoch) { + throw OlpbProtocolException("recovery reused stale epoch ${stale.epoch}") + } + if (ready.workerPid == stale.workerPid) { + throw OlpbProtocolException("recovery reused stale worker PID ${stale.workerPid}") + } + } + if (bound.dead.get() || !bound.binder.isBinderAlive) { + throw DeadObjectException() + } + channel = HostDataChannel(hostEndpoint) + channel.healthCheck(ready) + if (bound.dead.get() || !bound.binder.isBinderAlive) { + throw DeadObjectException() + } + val established = Connection(token, bound, channel, ready) + synchronized(stateLock) { + check(!closed) { "broker client was closed during connection establishment" } + connection = established + } + return@withLock established + } catch (error: Throwable) { + try { + workerEndpoint.close() + } catch (_: Throwable) { + } + if (channel == null) { + try { + hostEndpoint.close() + } catch (_: Throwable) { + } + } else { + channel.close() + } + bound.close(appContext) + throw BrokerClientException("failed to establish Android broker generation", error) + } + } + + private suspend fun bind(token: UUID): BoundService = + suspendCancellableCoroutine { continuation -> + val delivered = AtomicBoolean(false) + lateinit var serviceConnection: ServiceConnection + serviceConnection = + object : ServiceConnection { + override fun onServiceConnected(name: ComponentName, binder: IBinder) { + if (!delivered.compareAndSet(false, true)) return + val dead = AtomicBoolean(false) + val ready = AtomicReference(null) + val deathRecipient = IBinder.DeathRecipient { + dead.set(true) + ready.get()?.let { identity -> + binderDeathEvents.trySend( + BinderDeathEvent( + epoch = identity.epoch, + workerPid = identity.workerPid, + observedAtElapsedRealtimeNanos = + SystemClock.elapsedRealtimeNanos(), + ), + ) + } + signalInterruption(token, "binderDied") + } + try { + // Register before Hello so process death cannot hide in the handshake race. + binder.linkToDeath(deathRecipient, 0) + continuation.resume( + BoundService( + remote = IOliphauntBroker.Stub.asInterface(binder), + binder = binder, + serviceConnection = serviceConnection, + deathRecipient = deathRecipient, + dead = dead, + ready = ready, + ), + ) + } catch (error: Throwable) { + safeUnbind(appContext, serviceConnection) + continuation.resumeWithException(error) + } + } + + override fun onServiceDisconnected(name: ComponentName) { + if (delivered.get()) { + signalInterruption(token, "onServiceDisconnected") + } else if (delivered.compareAndSet(false, true)) { + continuation.resumeWithException( + BrokerClientException("broker service disconnected before binding"), + ) + } + } + + override fun onBindingDied(name: ComponentName) { + if (delivered.get()) { + signalInterruption(token, "onBindingDied") + } else if (delivered.compareAndSet(false, true)) { + continuation.resumeWithException( + BrokerClientException("broker service binding died"), + ) + } + } + + override fun onNullBinding(name: ComponentName) { + if (delivered.compareAndSet(false, true)) { + safeUnbind(appContext, serviceConnection) + continuation.resumeWithException( + BrokerClientException("broker service returned a null binding"), + ) + } + } + } + + val intent = Intent(appContext, BrokerService::class.java) + val didBind = appContext.bindService(intent, serviceConnection, Context.BIND_AUTO_CREATE) + if (!didBind && delivered.compareAndSet(false, true)) { + continuation.resumeWithException(BrokerClientException("bindService returned false")) + } + continuation.invokeOnCancellation { + if (delivered.compareAndSet(false, true)) safeUnbind(appContext, serviceConnection) + } + } + + private suspend fun callControl(current: Connection, request: Bundle): Bundle = + try { + val reply = withContext(Dispatchers.IO) { current.remote.control(request) } + if (!reply.getBoolean(BrokerContract.SUCCESS, true) || + reply.getString(BrokerContract.MESSAGE) == BrokerContract.REJECTED + ) { + throw BrokerRejectedException( + reply.getString(BrokerContract.REASON) ?: "worker rejected control request", + ) + } + reply + } catch (error: Throwable) { + if (error is DeadObjectException || error is RemoteException) { + interrupt(current, "Binder control failed: ${error.javaClass.simpleName}") + } + throw error + } + + private fun signalInterruption(token: UUID, reason: String) { + scope.launch { + currentConnection(token)?.let { interrupt(it, reason) } + } + } + + private fun interrupt(current: Connection, reason: String) { + if (!current.interrupted.compareAndSet(false, true)) return + synchronized(stateLock) { + if (connection?.token == current.token) { + connection = null + interruptedIdentity = current.ready + } + } + current.channel.close() + current.bound.unbind(appContext) + if (current.bound.dead.get()) { + current.bound.unlinkDeathRecipient() + } else { + // EOF can beat Binder death notification. Keep the proxy and recipient + // linked briefly after unbinding so that death remains observable. + scope.launch { + delay(BINDER_DEATH_RETENTION_MILLIS) + current.bound.unlinkDeathRecipient() + } + } + deathEvents.trySend( + BrokerDeath( + epoch = current.ready.epoch, + workerPid = current.ready.workerPid, + reason = reason, + observedAtElapsedRealtimeNanos = SystemClock.elapsedRealtimeNanos(), + ), + ) + } + + private fun currentConnection(token: UUID): Connection? = + synchronized(stateLock) { connection?.takeIf { it.token == token } } + + private fun allocateRequestId(): Long = synchronized(stateLock) { + check(nextRequestId != 0L) { "broker request ID space exhausted" } + val value = nextRequestId + nextRequestId += 1 + value + } + + private fun decodeReady(reply: Bundle): BrokerReady { + val message = reply.getString(BrokerContract.MESSAGE) + if (message == BrokerContract.REJECTED || !reply.getBoolean(BrokerContract.SUCCESS, true)) { + throw BrokerRejectedException( + reply.getString(BrokerContract.REASON) ?: "worker rejected Hello", + ) + } + if (message != BrokerContract.READY) { + throw OlpbProtocolException("Hello expected ready, received $message") + } + val protocolVersion = reply.getInt(BrokerContract.SELECTED_PROTOCOL_VERSION, -1) + if (protocolVersion !in OlpbProtocol.MINIMUM_VERSION..OlpbProtocol.MAXIMUM_VERSION) { + throw OlpbProtocolException("worker selected unsupported protocol $protocolVersion") + } + return BrokerReady( + epoch = requiredUuid(reply, BrokerContract.EPOCH), + workerPid = requiredPositiveInt(reply, BrokerContract.WORKER_PID), + protocolVersion = protocolVersion, + runtimeVersion = reply.getString(BrokerContract.RUNTIME_VERSION), + abiVersion = reply.optionalLong(BrokerContract.ABI_VERSION), + postgresMajorVersion = reply.optionalInt(BrokerContract.POSTGRES_MAJOR_VERSION), + rootManifestDigest = reply.getString(BrokerContract.ROOT_MANIFEST_DIGEST), + ) + } + + private class RequestAttempt( + val epoch: UUID, + val requestId: Long, + var bytesMayHaveReachedWorker: Boolean = false, + ) + + private data class ActiveRequest( + val generation: UUID, + val epoch: UUID, + val requestId: Long, + ) + + private class BoundService( + val remote: IOliphauntBroker, + val binder: IBinder, + val serviceConnection: ServiceConnection, + val deathRecipient: IBinder.DeathRecipient, + val dead: AtomicBoolean, + private val ready: AtomicReference, + ) { + private val unbound = AtomicBoolean(false) + private val unlinked = AtomicBoolean(false) + + fun setReady(value: BrokerReady) { + check(ready.compareAndSet(null, value)) { "broker identity was already assigned" } + } + + fun unbind(context: Context) { + if (unbound.compareAndSet(false, true)) safeUnbind(context, serviceConnection) + } + + fun unlinkDeathRecipient() { + if (!unlinked.compareAndSet(false, true)) return + try { + binder.unlinkToDeath(deathRecipient, 0) + } catch (_: Throwable) { + } + } + + fun close(context: Context) { + unlinkDeathRecipient() + unbind(context) + } + } + + private class Connection( + val token: UUID, + val bound: BoundService, + val channel: HostDataChannel, + val ready: BrokerReady, + ) { + val interrupted = AtomicBoolean(false) + val remote: IOliphauntBroker + get() = bound.remote + val binder: IBinder + get() = bound.binder + + fun close(context: Context) { + channel.close() + bound.close(context) + } + } + + private companion object { + const val BINDER_DEATH_RETENTION_MILLIS = 2_000L + const val MAXIMUM_RAW_RESPONSE_BYTES = 8 * 1024 * 1024 + } +} + +private class HostDataChannel(private val descriptor: ParcelFileDescriptor) { + private val closed = AtomicBoolean(false) + + suspend fun healthCheck(ready: BrokerReady) { + write( + OlpbFrame( + protocolVersion = ready.protocolVersion, + frameType = OlpbFrameType.PING, + epoch = ready.epoch, + requestId = 0, + ), + ) + val pong = read(ready.epoch) + if (pong.header.frameType != OlpbFrameType.PONG || + pong.header.requestId != 0L || + pong.payload.isNotEmpty() + ) { + throw OlpbProtocolException("broker health check did not receive an empty Pong") + } + } + + suspend fun write(frame: OlpbFrame) = withContext(Dispatchers.IO) { + val bytes = OlpbFrameCodec.encode(frame) + var offset = 0 + while (offset < bytes.size) { + try { + val count = Os.write(descriptor.fileDescriptor, bytes, offset, bytes.size - offset) + if (count <= 0) throw EOFException("broker socket write returned $count") + offset += count + } catch (error: ErrnoException) { + if (error.errno == OsConstants.EINTR) continue + throw IOException("broker socket write failed", error) + } + } + } + + suspend fun read(expectedEpoch: UUID): OlpbFrame = withContext(Dispatchers.IO) { + val headerBytes = readExactly(OlpbProtocol.HEADER_LENGTH) + val header = OlpbFrameCodec.decodeHeader(headerBytes, expectedEpoch) + val payload = readExactly(header.payloadLength) + OlpbFrame(header, payload) + } + + fun close() { + if (!closed.compareAndSet(false, true)) return + try { + Os.shutdown(descriptor.fileDescriptor, OsConstants.SHUT_RDWR) + } catch (_: Throwable) { + } + try { + descriptor.close() + } catch (_: Throwable) { + } + } + + private fun readExactly(count: Int): ByteArray { + if (count == 0) return ByteArray(0) + val bytes = ByteArray(count) + var offset = 0 + while (offset < count) { + val read = + try { + Os.read(descriptor.fileDescriptor, bytes, offset, count - offset) + } catch (error: ErrnoException) { + if (error.errno == OsConstants.EINTR) continue + throw IOException("broker socket read failed", error) + } + if (read == 0) { + try { + descriptor.checkError() + } catch (error: IOException) { + throw IOException("reliable broker socket reported peer failure", error) + } + throw EOFException("broker socket reached EOF") + } + offset += read + } + return bytes + } +} + +/** Requires a structurally complete backend stream ending in ReadyForQuery. */ +private class BackendTerminalObserver { + private val header = ByteArray(5) + private var headerBytes = 0 + private var messageType: Byte = 0 + private var remainingBodyBytes: Int? = null + private var bodyOffset = 0 + private var lastCompletedMessageType: Byte? = null + private var lastReadyStatus: Byte? = null + + fun append(bytes: ByteArray) { + var offset = 0 + while (offset < bytes.size) { + if (remainingBodyBytes == null) { + val count = minOf(5 - headerBytes, bytes.size - offset) + bytes.copyInto(header, destinationOffset = headerBytes, startIndex = offset, endIndex = offset + count) + headerBytes += count + offset += count + if (headerBytes != 5) continue + + messageType = header[0] + val length = + ((header[1].toLong() and 0xff) shl 24) or + ((header[2].toLong() and 0xff) shl 16) or + ((header[3].toLong() and 0xff) shl 8) or + (header[4].toLong() and 0xff) + if (length < 4 || length - 4 > Int.MAX_VALUE) { + throw OlpbProtocolException("invalid PostgreSQL backend message length $length") + } + val bodyLength = (length - 4).toInt() + if (messageType == READY_FOR_QUERY && bodyLength != 1) { + throw OlpbProtocolException("ReadyForQuery has invalid body length $bodyLength") + } + headerBytes = 0 + remainingBodyBytes = bodyLength + bodyOffset = 0 + if (bodyLength == 0) { + lastCompletedMessageType = messageType + remainingBodyBytes = null + } + continue + } + + val remaining = remainingBodyBytes ?: continue + val count = minOf(remaining, bytes.size - offset) + if (messageType == READY_FOR_QUERY && bodyOffset == 0 && count > 0) { + val status = bytes[offset] + if (status != IDLE && status != IN_TRANSACTION && status != FAILED_TRANSACTION) { + throw OlpbProtocolException("ReadyForQuery has unknown transaction status") + } + lastReadyStatus = status + } + offset += count + bodyOffset += count + val next = remaining - count + if (next == 0) lastCompletedMessageType = messageType + remainingBodyBytes = next.takeIf { it != 0 } + } + } + + fun finish(): Byte { + if (headerBytes != 0 || remainingBodyBytes != null) { + throw OlpbProtocolException("Completed arrived inside a PostgreSQL backend message") + } + if (lastCompletedMessageType != READY_FOR_QUERY || lastReadyStatus == null) { + throw OlpbProtocolException("Completed arrived without terminal ReadyForQuery") + } + return lastReadyStatus!! + } + + private companion object { + const val READY_FOR_QUERY: Byte = 0x5a + const val IDLE: Byte = 0x49 + const val IN_TRANSACTION: Byte = 0x54 + const val FAILED_TRANSACTION: Byte = 0x45 + } +} + +private fun validateFrontendRequest(bytes: ByteArray) { + val assembler = OlpbFrontendRequestAssembler() + assembler.append(bytes) + assembler.finish() +} + +private fun requireEmptyPayload(frame: OlpbFrame) { + if (frame.payload.isNotEmpty()) { + throw OlpbProtocolException("${frame.header.frameType} must not contain a payload") + } +} + +private fun requireReply(reply: Bundle, expected: String) { + val message = reply.getString(BrokerContract.MESSAGE) + if (message != expected) { + throw OlpbProtocolException("expected $expected control reply, received $message") + } +} + +private fun requiredUuid(bundle: Bundle, key: String): UUID { + val raw = bundle.getString(key) ?: throw OlpbProtocolException("missing $key") + return try { + UUID.fromString(raw) + } catch (error: IllegalArgumentException) { + throw OlpbProtocolException("invalid $key UUID") + } +} + +private fun requiredPositiveInt(bundle: Bundle, key: String): Int { + if (!bundle.containsKey(key)) throw OlpbProtocolException("missing $key") + val value = bundle.getInt(key) + if (value <= 0) throw OlpbProtocolException("invalid $key $value") + return value +} + +private fun Bundle.optionalLong(key: String): Long? = + if (containsKey(key)) getLong(key) else null + +private fun Bundle.optionalInt(key: String): Int? = + if (containsKey(key)) getInt(key) else null + +private fun Bundle.optionalBoolean(key: String): Boolean? = + if (containsKey(key)) getBoolean(key) else null + +private fun safeUnbind(context: Context, connection: ServiceConnection) { + try { + context.unbindService(connection) + } catch (_: IllegalArgumentException) { + } +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerContract.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerContract.kt new file mode 100644 index 00000000..f5214945 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerContract.kt @@ -0,0 +1,139 @@ +package dev.oliphaunt.androidbrokerspike + +import android.os.Bundle +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.UUID + +/** Primitive-only Binder schema shared by the spike host and private service. */ +internal object BrokerContract { + const val EXPECTED_ABI = 6L + const val ROOT_ID = "default" + const val STARTUP_CONFIGURATION_DIGEST = "android-native-broker-spike-v1" + + const val MESSAGE = "message" + const val MINIMUM_PROTOCOL_VERSION = "minimumProtocolVersion" + const val MAXIMUM_PROTOCOL_VERSION = "maximumProtocolVersion" + const val SELECTED_PROTOCOL_VERSION = "selectedProtocolVersion" + const val EXPECTED_ABI_KEY = "expectedABI" + const val EXPECTED_RUNTIME_VERSION = "expectedRuntimeVersion" + const val ROOT_ID_KEY = "rootID" + const val STARTUP_CONFIGURATION_DIGEST_KEY = "startupConfigurationDigest" + const val REQUESTED_CAPABILITIES = "requestedCapabilities" + const val EPOCH = "epoch" + const val REQUEST_ID = "requestID" + const val WORKER_PID = "workerPID" + const val RUNTIME_VERSION = "runtimeVersion" + const val ABI_VERSION = "abiVersion" + const val POSTGRES_MAJOR_VERSION = "postgresMajorVersion" + const val ROOT_MANIFEST_DIGEST = "rootManifestDigest" + const val ACTUAL_CAPABILITIES = "actualCapabilities" + const val ACTUAL_RUNTIME_CONFIGURATION = "actualRuntimeConfiguration" + const val SUCCESS = "success" + const val REASON = "reason" + const val FAULT = "fault" + const val STATE = "state" + const val ACTIVE_REQUEST_ID = "activeRequestID" + const val NATIVE_DISPATCH_STARTED = "nativeDispatchStarted" + const val NATIVE_POSTGRES_OUTPUT_WITNESS_OBSERVED = + "nativePostgresOutputWitnessObserved" + const val NATIVE_POSTGRES_OUTPUT_WITNESS_REQUEST_ID = + "nativePostgresOutputWitnessRequestID" + const val NATIVE_POSTGRES_OUTPUT_WITNESS_BACKEND_BYTES = + "nativePostgresOutputWitnessBackendBytes" + const val NATIVE_POSTGRES_OUTPUT_WITNESS_ELAPSED_REALTIME_NANOS = + "nativePostgresOutputWitnessElapsedRealtimeNanos" + const val TRANSACTION_STATUS = "transactionStatus" + const val CURRENT_PSS_BYTES = "currentPssBytes" + const val CURRENT_RSS_BYTES = "currentRssBytes" + const val REQUESTED_SOCKET_SEND_BUFFER_BYTES = "requestedSocketSendBufferBytes" + const val DIAGNOSTICS_SAMPLE_ELAPSED_REALTIME_NANOS = + "diagnosticsSampleElapsedRealtimeNanos" + const val SOCKET_NON_BLOCKING_PROBE_SUCCEEDED = "socketNonBlockingProbeSucceeded" + const val SOCKET_NON_BLOCKING = "socketNonBlocking" + const val SOCKET_POLL_SUCCEEDED = "socketPollSucceeded" + const val SOCKET_WRITABLE_NOW = "socketWritableNow" + const val SOCKET_WRITE_IN_PROGRESS = "socketWriteInProgress" + const val SOCKET_ACTIVE_WRITE_SEQUENCE = "socketActiveWriteSequence" + const val SOCKET_ACTIVE_WRITE_REQUEST_ID = "socketActiveWriteRequestID" + const val SOCKET_ACTIVE_WRITE_FRAME_TYPE = "socketActiveWriteFrameType" + const val SOCKET_ACTIVE_WRITE_STARTED_ELAPSED_REALTIME_NANOS = + "socketActiveWriteStartedElapsedRealtimeNanos" + const val SOCKET_ACTIVE_WRITE_ENCODED_BYTES = "socketActiveWriteEncodedBytes" + const val SOCKET_WRITES_COMPLETED = "socketWritesCompleted" + const val SOCKET_COMPLETED_ENCODED_BYTES = "socketCompletedEncodedBytes" + // BrokerClient keeps its experimental model name while the wire key and + // published evidence state honestly that this is the requested value. + const val SOCKET_SEND_BUFFER_BYTES = REQUESTED_SOCKET_SEND_BUFFER_BYTES + + const val HELLO = "hello" + const val READY = "ready" + const val REJECTED = "rejected" + const val CANCEL = "cancel" + const val CANCEL_OBSERVED = "cancelObserved" + const val DIAGNOSTICS = "diagnostics" + const val INJECT_FAULT = "injectFault" + const val DETACH = "detach" + + const val NATIVE_POSTGRES_OUTPUT_WITNESS_THRESHOLD_BYTES = 4L * 1_024 * 1_024 + const val NATIVE_POSTGRES_OUTPUT_WATCHDOG_DELAY_MILLIS = 2_000L + const val NATIVE_POSTGRES_OUTPUT_WITNESS_SQL = + "SELECT repeat('w', 8192) AS witness FROM generate_series(1, 513) " + + "UNION ALL SELECT ''::text FROM pg_sleep(60) AS blocker(ignored)" + + val requestedCapabilities = + arrayOf( + "processIsolated", + "crashRestartable", + "sameRootLogicalReopen", + "protocolRaw", + "protocolStream", + "queryCancel", + ) + + fun hello(): Bundle = + Bundle().apply { + putString(MESSAGE, HELLO) + putInt(MINIMUM_PROTOCOL_VERSION, OlpbProtocol.MINIMUM_VERSION) + putInt(MAXIMUM_PROTOCOL_VERSION, OlpbProtocol.MAXIMUM_VERSION) + putLong(EXPECTED_ABI_KEY, EXPECTED_ABI) + putString(ROOT_ID_KEY, ROOT_ID) + putString(STARTUP_CONFIGURATION_DIGEST_KEY, STARTUP_CONFIGURATION_DIGEST) + putStringArray(REQUESTED_CAPABILITIES, requestedCapabilities) + } + + fun control( + message: String, + epoch: UUID, + requestId: Long? = null, + fault: BrokerFault? = null, + ): Bundle = + Bundle().apply { + putString(MESSAGE, message) + putString(EPOCH, epoch.toString()) + requestId?.let { putLong(REQUEST_ID, it) } + fault?.let { putString(FAULT, it.wireValue) } + } + + fun simpleQuery(sql: String): ByteArray { + require('\u0000' !in sql) { "SQL must not contain NUL bytes" } + val body = sql.toByteArray(Charsets.UTF_8) + byteArrayOf(0) + val messageLength = body.size + 4 + return ByteBuffer + .allocate(1 + 4 + body.size) + .order(ByteOrder.BIG_ENDIAN) + .apply { + put('Q'.code.toByte()) + putInt(messageLength) + put(body) + }.array() + } +} + +internal enum class BrokerFault( + val wireValue: String, +) { + EXECUTOR_DEADLOCK_WITH_FAIL_STOP("executorDeadlockWithFailStop"), + NATIVE_FAIL_STOP_WATCHDOG("nativeFailStopWatchdog"), + AFTER_NATIVE_SUCCESS_BEFORE_COMPLETED("afterNativeSuccessBeforeCompleted"), +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerExperiment.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerExperiment.kt new file mode 100644 index 00000000..6dc5341a --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerExperiment.kt @@ -0,0 +1,703 @@ +package dev.oliphaunt.androidbrokerspike + +import android.content.Context +import android.os.Process +import android.os.SystemClock +import dev.oliphaunt.PostgresException +import dev.oliphaunt.parseQueryResponse +import java.util.UUID +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.withTimeout +import org.json.JSONArray +import org.json.JSONObject +import kotlin.math.abs + +internal class BrokerExperiment(private val context: Context) { + private data class SustainedSocketStall( + val first: BrokerDiagnostics, + val second: BrokerDiagnostics, + val transientCandidatesRejected: Int, + ) + + private val checks = linkedSetOf() + private val workerPids = mutableListOf() + private val workerEpochs = mutableListOf() + private val faultEvidence = JSONArray() + + suspend fun run(runNonce: String, strategy: String): JSONObject { + require(strategy == "full") { "unsupported strategy $strategy" } + require(runNonce.matches(Regex("[A-Za-z0-9._-]{1,128}"))) { "run nonce is not portable" } + + val started = SystemClock.elapsedRealtimeNanos() + val client = BrokerClient(context) + try { + val initial = client.connect() + recordGeneration(initial) + require(initial.workerPid != Process.myPid()) { "broker reused the host process" } + checks += "separateProcess" + + require(queryText(client, "SELECT 'healthy'::text AS status", "status") == "healthy") + checks += "healthySql" + + runCancellation(client) + checks += "outOfBandCancel" + + client.execute( + "CREATE TABLE IF NOT EXISTS android_broker_markers " + + "(marker text PRIMARY KEY, execution_count bigint NOT NULL DEFAULT 0, " + + "created_at timestamptz NOT NULL DEFAULT now())", + ) + // Preserve old experimental PGDATA while making repeated ambiguous + // mutations observable instead of hiding replay behind a key error. + client.execute( + "ALTER TABLE android_broker_markers ADD COLUMN IF NOT EXISTS " + + "execution_count bigint NOT NULL DEFAULT 0", + ) + val stableMarker = "stable-$runNonce" + client.execute( + "INSERT INTO android_broker_markers(marker) VALUES ('$stableMarker') " + + "ON CONFLICT (marker) DO NOTHING", + ) + + runFailStop( + client = client, + fault = BrokerFault.EXECUTOR_DEADLOCK_WITH_FAIL_STOP, + sql = "SELECT 'executor-must-not-complete'::text AS status", + label = "executorDeadlock", + requireNativePostgresOutputWitness = false, + ) + checks += "executorDeadlockFailStop" + + runFailStop( + client = client, + fault = BrokerFault.NATIVE_FAIL_STOP_WATCHDOG, + sql = BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_SQL, + label = "nativePgSleep", + requireNativePostgresOutputWitness = true, + ) + checks += "nativePgSleepFailStop" + + val ambiguousMarker = "ambiguous-$runNonce" + runFailStop( + client = client, + fault = BrokerFault.AFTER_NATIVE_SUCCESS_BEFORE_COMPLETED, + sql = + "INSERT INTO android_broker_markers(marker, execution_count) " + + "VALUES ('$ambiguousMarker', 1) " + + "ON CONFLICT (marker) DO UPDATE SET execution_count = " + + "android_broker_markers.execution_count + 1", + label = "afterCommitBeforeCompleted", + requireNativePostgresOutputWitness = false, + ) + val ambiguousExecutionCount = + queryText( + client, + "SELECT execution_count::text AS count FROM android_broker_markers " + + "WHERE marker = '$ambiguousMarker'", + "count", + ).toLong() + val replayCount = ambiguousExecutionCount - 1L + require(replayCount == 0L) { + "ambiguous mutation executed $ambiguousExecutionCount times; replay count is $replayCount" + } + checks += "outcomeUnknownNoReplay" + + require( + queryText( + client, + "SELECT count(*)::text AS count FROM android_broker_markers " + + "WHERE marker = '$stableMarker'", + "count", + ) == "1", + ) { "stable marker did not survive worker replacement" } + + val slow8 = runSlowReader(client, rows = 1_024, expectedMinimumBytes = 8L * 1_024 * 1_024) + checks += "boundedSlowReader8MiB" + val slow32 = runSlowReader(client, rows = 4_096, expectedMinimumBytes = 32L * 1_024 * 1_024) + val acceptedWireBoundDeltaBytes = + abs( + slow32.getLong("acceptedWireBytesUpperBound") - + slow8.getLong("acceptedWireBytesUpperBound"), + ) + require(acceptedWireBoundDeltaBytes <= MAXIMUM_ENCODED_FRAME_BYTES) { + "32 MiB and 8 MiB pre-read socket bounds differ by " + + "$acceptedWireBoundDeltaBytes bytes" + } + checks += "boundedSlowReader32MiB" + checks += "persistentRecovery" + checks += "binderDeath" + checks += "freshPidAndEpoch" + + require(workerPids.distinct().size == workerPids.size) { "adjacent worker PIDs were not fresh" } + require(workerEpochs.distinct().size == workerEpochs.size) { "worker epochs were not fresh" } + + return JSONObject() + .put("schema", "oliphaunt-android-native-broker-spike-v1") + .put("status", "PASS") + .put("runNonce", runNonce) + .put("strategy", strategy) + .put("hostPid", Process.myPid()) + .put("workerPids", JSONArray(workerPids)) + .put("workerEpochs", JSONArray(workerEpochs.map(UUID::toString))) + .put("checks", JSONArray(checks.toList())) + .put("faultEvidence", faultEvidence) + .put("slowReader8MiB", slow8) + .put("slowReader32MiB", slow32) + .put("acceptedWireBoundDeltaBytes", acceptedWireBoundDeltaBytes) + .put("maximumAcceptedWireBoundDeltaBytes", MAXIMUM_ENCODED_FRAME_BYTES) + .put("replayCount", replayCount) + .put("persistentMarkerSurvived", true) + .put("ambiguousExecutionCount", ambiguousExecutionCount) + .put("elapsedMilliseconds", nanosToMillis(SystemClock.elapsedRealtimeNanos() - started)) + .put("environment", "android-emulator") + .put("physicalDeviceEvidence", false) + } finally { + client.close() + } + } + + private suspend fun runCancellation(client: BrokerClient) = coroutineScope { + val before = client.diagnostics() + val query = async { client.execute("SELECT pg_sleep(60)") } + val active = awaitNativeDispatch(client) + require(active.workerPid == before.workerPid && active.epoch == before.epoch) + // nativeDispatchStarted is published immediately before the JNI call. + // Give PostgreSQL the same short settle window used by the native ABI + // cancellation smoke so the cancel targets the in-flight pg_sleep. + delay(250) + require(client.cancel()) { "worker did not acknowledge cancellation" } + val bytes = withTimeout(10_000) { query.await() } + val sqlstate = + try { + parseQueryResponse(bytes) + error("canceled pg_sleep unexpectedly completed") + } catch (error: PostgresException) { + error.postgresError.sqlstate + } + require(sqlstate == "57014") { "cancel SQLSTATE is $sqlstate" } + val after = client.diagnostics() + require(after.workerPid == before.workerPid && after.epoch == before.epoch) + require(queryText(client, "SELECT 'healthy'::text AS status", "status") == "healthy") + } + + private suspend fun runFailStop( + client: BrokerClient, + fault: BrokerFault, + sql: String, + label: String, + requireNativePostgresOutputWitness: Boolean, + ) = coroutineScope { + val before = client.connect() + val binderDeath = async { client.awaitBinderDeath(before, 10_000) } + val interruption = async { client.awaitDeath(10_000) } + client.armFault(fault) + val requestStarted = SystemClock.elapsedRealtimeNanos() + val request: Deferred> = async { runCatching { client.execute(sql) } } + val witness = + if (requireNativePostgresOutputWitness) { + awaitNativePostgresOutputWitness(client, before) + } else { + null + } + witness?.let { + require(it.workerPid == before.workerPid && it.epoch == before.epoch) + } + val terminal = withTimeout(10_000) { request.await() } + val failure = terminal.exceptionOrNull() + require(failure is BrokerOutcomeUnknownException) { + "$label did not terminate as outcomeUnknown: ${failure?.javaClass?.simpleName}" + } + witness?.let { + require(it.nativePostgresOutputWitnessRequestId == failure.requestId) { + "native PostgreSQL-output witness request " + + "${it.nativePostgresOutputWitnessRequestId} " + + "does not match terminal request ${failure.requestId}" + } + } + val binderDeathAt = withTimeout(10_000) { binderDeath.await() } + val death = withTimeout(10_000) { interruption.await() } + require(death.workerPid == before.workerPid && death.epoch == before.epoch) + val recovered = reconnectEventually(client) + require(recovered.workerPid != before.workerPid) { "$label reused worker PID" } + require(recovered.epoch != before.epoch) { "$label reused worker epoch" } + recordGeneration(recovered) + require(queryText(client, "SELECT 'healthy'::text AS status", "status") == "healthy") + faultEvidence.put( + JSONObject() + .put("label", label) + .put("fault", fault.wireValue) + .put("initialWorkerPid", before.workerPid) + .put("initialEpoch", before.epoch.toString()) + .put("recoveredWorkerPid", recovered.workerPid) + .put("recoveredEpoch", recovered.epoch.toString()) + .put("requestId", failure.requestId) + .put("terminal", "outcomeUnknown") + .put("nativeDispatchObserved", witness != null) + .put("nativePostgresOutputWitnessObserved", witness != null) + .apply { + witness?.let { + put( + "nativePostgresOutputWitnessRequestId", + it.nativePostgresOutputWitnessRequestId, + ) + put( + "nativePostgresOutputWitnessBackendBytes", + it.nativePostgresOutputWitnessBackendBytes, + ) + put( + "nativePostgresOutputWitnessElapsedRealtimeNanos", + it.nativePostgresOutputWitnessElapsedRealtimeNanos, + ) + put( + "nativePostgresOutputWatchdogDelayMilliseconds", + BrokerContract.NATIVE_POSTGRES_OUTPUT_WATCHDOG_DELAY_MILLIS, + ) + } + } + .put("binderDeathObserved", true) + .put("binderDeathElapsedRealtimeNanos", binderDeathAt) + .put("interruptionReason", death.reason) + .put( + "requestToRecoveryMilliseconds", + nanosToMillis(SystemClock.elapsedRealtimeNanos() - requestStarted), + ), + ) + } + + private suspend fun runSlowReader( + client: BrokerClient, + rows: Int, + expectedMinimumBytes: Long, + ): JSONObject = coroutineScope { + val pssSamples = mutableListOf() + val rssSamples = mutableListOf() + val started = SystemClock.elapsedRealtimeNanos() + val baseline = client.diagnostics() + captureMemorySample(baseline, pssSamples, rssSamples) + require(baseline.socketNonBlockingProbeSucceeded) { + "socket blocking-mode probe failed before the slow-reader request" + } + require(baseline.socketNonBlocking == false) { "broker data socket is nonblocking" } + require(baseline.socketPollSucceeded) { + "socket POLLOUT probe failed before the slow-reader request" + } + val baselineWritesCompleted = + baseline.socketWritesCompleted + ?: error("missing baseline completed-write count") + val baselineCompletedEncodedBytes = + baseline.socketCompletedEncodedBytes + ?: error("missing baseline completed-byte count") + val readGateCreatedNanos = SystemClock.elapsedRealtimeNanos() + val firstReadRelease = CompletableDeferred() + val stream = + async { + client.executeSlowReader( + sql = "SELECT repeat('s', 8192) FROM generate_series(1, $rows)", + firstReadRelease = firstReadRelease, + perFrameDelayMillis = 0, + ) + } + + var readGateReleasedNanos = 0L + val sustainedStall = + try { + val evidence = awaitSustainedBlockedSocketWrite(client) + captureMemorySample(evidence.first, pssSamples, rssSamples) + captureMemorySample(evidence.second, pssSamples, rssSamples) + require(!stream.isCompleted) { + "stream completed while the host-controlled read gate was closed" + } + evidence + } finally { + readGateReleasedNanos = SystemClock.elapsedRealtimeNanos() + firstReadRelease.complete(Unit) + } + val firstStall = sustainedStall.first + val secondStall = sustainedStall.second + val firstSampleNanos = + firstStall.sampleElapsedRealtimeNanos + ?: error("first stall sample omitted its timestamp") + val activeWriteStartedNanos = + firstStall.socketActiveWriteStartedElapsedRealtimeNanos + ?: error("first stall sample omitted its write start") + val activeWriteSequence = + firstStall.socketActiveWriteSequence + ?: error("first stall sample omitted its write sequence") + val activeWriteRequestId = + firstStall.socketActiveWriteRequestId + ?: error("first stall sample omitted its request ID") + val activeWriteEncodedBytes = + firstStall.socketActiveWriteEncodedBytes + ?: error("first stall sample omitted its encoded size") + val firstWritesCompleted = + firstStall.socketWritesCompleted + ?: error("first stall sample omitted its completed-write count") + val firstCompletedEncodedBytes = + firstStall.socketCompletedEncodedBytes + ?: error("first stall sample omitted its completed-byte count") + val secondSampleNanos = + secondStall.sampleElapsedRealtimeNanos + ?: error("second stall sample omitted its timestamp") + require(readGateReleasedNanos >= secondSampleNanos) { + "host read gate was released before the second stall sample" + } + require(secondStall.workerPid == baseline.workerPid && secondStall.epoch == baseline.epoch) { + "worker identity changed during the no-read window" + } + require(secondStall.state == "running" && secondStall.nativeDispatchStarted) { + "worker was not still executing during the no-read window" + } + require(secondStall.socketNonBlockingProbeSucceeded) { + "socket blocking-mode probe failed during the sustained stall" + } + require(secondStall.socketNonBlocking == false) { + "broker data socket became nonblocking" + } + require(secondStall.socketPollSucceeded && secondStall.socketWritableNow == false) { + "socket became writable while the host was not reading" + } + require(secondStall.socketWriteInProgress) { + "synchronous socket write was no longer in progress" + } + require(secondStall.socketActiveWriteFrameType == OlpbFrameType.RESPONSE_BYTES) { + "blocked write was not a responseBytes frame" + } + require(secondStall.socketActiveWriteRequestId == activeWriteRequestId) { + "active request changed during the socket stall" + } + require(secondStall.socketActiveWriteSequence == activeWriteSequence) { + "socket writer advanced during the no-read interval" + } + require(secondStall.socketActiveWriteStartedElapsedRealtimeNanos == activeWriteStartedNanos) { + "socket write start changed during the no-read interval" + } + require(secondStall.socketActiveWriteEncodedBytes == activeWriteEncodedBytes) { + "socket write size changed during the no-read interval" + } + require(secondStall.socketWritesCompleted == firstWritesCompleted) { + "completed-write count advanced during the no-read interval" + } + require(secondStall.socketCompletedEncodedBytes == firstCompletedEncodedBytes) { + "completed socket bytes advanced during the no-read interval" + } + require(secondSampleNanos - firstSampleNanos >= REQUIRED_STALL_NANOS) { + "same-write socket stall lasted less than $REQUIRED_STALL_MILLIS ms" + } + require(secondSampleNanos - activeWriteStartedNanos >= REQUIRED_STALL_NANOS) { + "active synchronous write was younger than the required stall" + } + + val completedEncodedDeltaBeforeRead = + firstCompletedEncodedBytes - baselineCompletedEncodedBytes + require(completedEncodedDeltaBeforeRead >= 0) { + "completed socket-byte counter regressed" + } + require(firstWritesCompleted >= baselineWritesCompleted) { + "completed socket-write counter regressed" + } + val acceptedWireBytesUpperBound = + Math.addExact(completedEncodedDeltaBeforeRead, activeWriteEncodedBytes.toLong()) + require( + Math.addExact(acceptedWireBytesUpperBound, MAXIMUM_ENCODED_FRAME_BYTES) < + expectedMinimumBytes, + ) { + "pre-read accepted-wire bound $acceptedWireBytesUpperBound is not below the " + + "$expectedMinimumBytes-byte response by one maximum frame" + } + + val stats = + withTimeout(SLOW_READER_DRAIN_TIMEOUT_MILLIS) { + while (!stream.isCompleted) { + runCatching { client.diagnostics() }.getOrNull()?.let { + captureMemorySample(it, pssSamples, rssSamples) + } + delay(100) + } + stream.await() + } + val afterDrain = client.diagnostics() + captureMemorySample(afterDrain, pssSamples, rssSamples) + require(stats.responseBytes >= expectedMinimumBytes) { + "slow reader returned only ${stats.responseBytes} bytes" + } + require(stats.responseChunks > 1) { "slow reader did not observe multiple chunks" } + require(stats.workerPid == baseline.workerPid && stats.epoch == baseline.epoch) { + "slow reader completed in a different worker generation" + } + require(stats.requestId == activeWriteRequestId) { + "completed request does not match the blocked write" + } + val afterDrainWritesCompleted = + afterDrain.socketWritesCompleted + ?: error("post-drain diagnostics omitted the completed-write count") + val afterDrainCompletedEncodedBytes = + afterDrain.socketCompletedEncodedBytes + ?: error("post-drain diagnostics omitted the completed-byte count") + require(afterDrainWritesCompleted >= activeWriteSequence) { + "the blocked socket write did not complete after reads resumed" + } + require( + afterDrainCompletedEncodedBytes - baselineCompletedEncodedBytes >= stats.responseBytes, + ) { + "post-drain encoded-byte count is smaller than the response payload" + } + require(pssSamples.isNotEmpty()) { "slow reader did not capture a PSS sample" } + require(rssSamples.isNotEmpty()) { "slow reader did not capture an RSS sample" } + val minimumPssBytes = pssSamples.minOrNull() ?: error("PSS sample invariant failed") + val maximumPssBytes = pssSamples.maxOrNull() ?: error("PSS sample invariant failed") + val minimumRssBytes = rssSamples.minOrNull() ?: error("RSS sample invariant failed") + val maximumRssBytes = rssSamples.maxOrNull() ?: error("RSS sample invariant failed") + JSONObject() + .put("rows", rows) + .put("responseBytes", stats.responseBytes) + .put("responseChunks", stats.responseChunks) + .put("elapsedMilliseconds", nanosToMillis(SystemClock.elapsedRealtimeNanos() - started)) + .put("sampleCount", pssSamples.size) + .put("maximumPssBytes", maximumPssBytes) + .put("minimumPssBytes", minimumPssBytes) + .put("pssSpanBytes", maximumPssBytes - minimumPssBytes) + .put("maximumRssBytes", maximumRssBytes) + .put("minimumRssBytes", minimumRssBytes) + .put("rssSpanBytes", maximumRssBytes - minimumRssBytes) + .put("readReleaseMode", "hostControlledGate") + .put("readGateReleasedAfterSecondSample", true) + .put("readGateCreatedElapsedRealtimeNanos", readGateCreatedNanos) + .put("readGateReleasedElapsedRealtimeNanos", readGateReleasedNanos) + .put( + "readGateHeldMilliseconds", + nanosToMillis(readGateReleasedNanos - readGateCreatedNanos), + ) + .put("stableStallSearchTimeoutMilliseconds", STALL_DISCOVERY_TIMEOUT_MILLIS) + .put("stableStallPollIntervalMilliseconds", STALL_POLL_INTERVAL_MILLIS) + .put( + "transientStallCandidatesRejected", + sustainedStall.transientCandidatesRejected, + ) + .put("slowReaderDrainTimeoutMilliseconds", SLOW_READER_DRAIN_TIMEOUT_MILLIS) + .put("requiredStallMilliseconds", REQUIRED_STALL_MILLIS) + .put( + "observedSameWriteStallMilliseconds", + nanosToMillis(secondSampleNanos - firstSampleNanos), + ) + .put( + "activeWriteAgeAtSecondSampleMilliseconds", + nanosToMillis(secondSampleNanos - activeWriteStartedNanos), + ) + .put("socketNonBlockingProbeSucceeded", true) + .put("socketNonBlocking", false) + .put("firstSocketPollSucceeded", firstStall.socketPollSucceeded) + .put("secondSocketPollSucceeded", secondStall.socketPollSucceeded) + .put("firstSocketWritableNow", firstStall.socketWritableNow) + .put("secondSocketWritableNow", secondStall.socketWritableNow) + .put("firstSocketWriteInProgress", firstStall.socketWriteInProgress) + .put("secondSocketWriteInProgress", secondStall.socketWriteInProgress) + .put("firstSampleElapsedRealtimeNanos", firstSampleNanos) + .put("secondSampleElapsedRealtimeNanos", secondSampleNanos) + .put("activeWriteStartedElapsedRealtimeNanos", activeWriteStartedNanos) + .put("activeWriteRequestId", activeWriteRequestId) + .put("activeWriteFrameType", OlpbFrameType.RESPONSE_BYTES.name) + .put("firstActiveWriteSequence", activeWriteSequence) + .put("secondActiveWriteSequence", secondStall.socketActiveWriteSequence) + .put("activeWriteEncodedBytes", activeWriteEncodedBytes) + .put("baselineWritesCompleted", baselineWritesCompleted) + .put("firstWritesCompleted", firstWritesCompleted) + .put("secondWritesCompleted", secondStall.socketWritesCompleted) + .put("afterDrainWritesCompleted", afterDrainWritesCompleted) + .put("baselineCompletedEncodedBytes", baselineCompletedEncodedBytes) + .put("firstCompletedEncodedBytes", firstCompletedEncodedBytes) + .put("secondCompletedEncodedBytes", secondStall.socketCompletedEncodedBytes) + .put("afterDrainCompletedEncodedBytes", afterDrainCompletedEncodedBytes) + .put("completedEncodedDeltaBeforeRead", completedEncodedDeltaBeforeRead) + .put("acceptedWireBytesUpperBound", acceptedWireBytesUpperBound) + .put("maximumEncodedFrameBytes", MAXIMUM_ENCODED_FRAME_BYTES) + .put( + "requestedSocketSendBufferBytes", + afterDrain.socketSendBufferBytes ?: -1, + ) + } + + private suspend fun awaitSustainedBlockedSocketWrite( + client: BrokerClient, + ): SustainedSocketStall = + withTimeout(STALL_DISCOVERY_TIMEOUT_MILLIS) { + var candidate: BrokerDiagnostics? = null + var transientCandidatesRejected = 0 + while (true) { + val diagnostics = client.diagnostics() + if (!diagnostics.isBlockedResponseWriteCandidate()) { + if (candidate != null) transientCandidatesRejected += 1 + candidate = null + delay(STALL_POLL_INTERVAL_MILLIS) + continue + } + + val first = candidate + if (first == null || !first.isSameBlockedWriteAs(diagnostics)) { + if (first != null) transientCandidatesRejected += 1 + candidate = diagnostics + delay(STALL_POLL_INTERVAL_MILLIS) + continue + } + + val firstSample = first.sampleElapsedRealtimeNanos + ?: error("blocked-write candidate omitted its timestamp") + val currentSample = diagnostics.sampleElapsedRealtimeNanos + ?: error("blocked-write sample omitted its timestamp") + if (currentSample - firstSample >= REQUIRED_STALL_NANOS) { + return@withTimeout SustainedSocketStall( + first = first, + second = diagnostics, + transientCandidatesRejected = transientCandidatesRejected, + ) + } + delay(STALL_POLL_INTERVAL_MILLIS) + } + error("unreachable") + } + + private fun BrokerDiagnostics.isBlockedResponseWriteCandidate(): Boolean = + state == "running" && + nativeDispatchStarted && + activeRequestId != null && + socketNonBlockingProbeSucceeded && + socketNonBlocking == false && + socketPollSucceeded && + socketWritableNow == false && + socketWriteInProgress && + socketActiveWriteFrameType == OlpbFrameType.RESPONSE_BYTES && + socketActiveWriteSequence != null && + socketActiveWriteRequestId == activeRequestId && + socketActiveWriteStartedElapsedRealtimeNanos != null && + socketActiveWriteEncodedBytes != null && + socketWritesCompleted != null && + socketCompletedEncodedBytes != null && + sampleElapsedRealtimeNanos != null + + private fun BrokerDiagnostics.isSameBlockedWriteAs(other: BrokerDiagnostics): Boolean = + workerPid == other.workerPid && + epoch == other.epoch && + activeRequestId == other.activeRequestId && + socketWritableNow == other.socketWritableNow && + socketActiveWriteSequence == other.socketActiveWriteSequence && + socketActiveWriteRequestId == other.socketActiveWriteRequestId && + socketActiveWriteFrameType == other.socketActiveWriteFrameType && + socketActiveWriteStartedElapsedRealtimeNanos == + other.socketActiveWriteStartedElapsedRealtimeNanos && + socketActiveWriteEncodedBytes == other.socketActiveWriteEncodedBytes && + socketWritesCompleted == other.socketWritesCompleted && + socketCompletedEncodedBytes == other.socketCompletedEncodedBytes + + private fun captureMemorySample( + diagnostics: BrokerDiagnostics, + pssSamples: MutableList, + rssSamples: MutableList, + ) { + diagnostics.currentPssBytes?.takeIf { it > 0 }?.let(pssSamples::add) + diagnostics.currentRssBytes?.takeIf { it > 0 }?.let(rssSamples::add) + } + + private suspend fun awaitNativeDispatch(client: BrokerClient): BrokerDiagnostics = + withTimeout(5_000) { + while (true) { + val diagnostics = client.diagnostics() + if (diagnostics.activeRequestId != null && + diagnostics.activeRequestId != 0L && + diagnostics.nativeDispatchStarted + ) { + return@withTimeout diagnostics + } + delay(10) + } + error("unreachable") + } + + private suspend fun awaitNativePostgresOutputWitness( + client: BrokerClient, + expectedGeneration: BrokerReady, + ): BrokerDiagnostics = + withTimeout(10_000) { + while (true) { + val diagnostics = client.diagnostics() + require( + diagnostics.workerPid == expectedGeneration.workerPid && + diagnostics.epoch == expectedGeneration.epoch, + ) { "native PostgreSQL-output witness moved to a different worker generation" } + if (diagnostics.nativePostgresOutputWitnessObserved) { + require(diagnostics.nativeDispatchStarted) { + "native PostgreSQL-output witness was published before native dispatch" + } + val activeRequestId = + requireNotNull(diagnostics.activeRequestId) { + "native PostgreSQL-output witness has no active request" + } + val witnessRequestId = + requireNotNull(diagnostics.nativePostgresOutputWitnessRequestId) { + "native PostgreSQL-output witness has no request marker" + } + require(witnessRequestId == activeRequestId) { + "native PostgreSQL-output witness request $witnessRequestId " + + "is not active request $activeRequestId" + } + val backendBytes = + requireNotNull(diagnostics.nativePostgresOutputWitnessBackendBytes) { + "native PostgreSQL-output witness has no backend-byte count" + } + require( + backendBytes > + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_THRESHOLD_BYTES, + ) { + "native PostgreSQL-output witness followed only $backendBytes backend bytes" + } + requireNotNull(diagnostics.nativePostgresOutputWitnessElapsedRealtimeNanos) { + "native PostgreSQL-output witness has no monotonic timestamp" + } + return@withTimeout diagnostics + } + delay(10) + } + error("unreachable") + } + + private suspend fun reconnectEventually(client: BrokerClient): BrokerReady = + withTimeout(15_000) { + var lastError: Throwable? = null + while (true) { + try { + return@withTimeout client.reconnect() + } catch (error: Throwable) { + lastError = error + delay(100) + } + } + throw lastError ?: IllegalStateException("recovery did not run") + } + + private suspend fun queryText(client: BrokerClient, sql: String, column: String): String { + val result = parseQueryResponse(client.execute(sql)) + return result.getText(0, column) ?: error("query returned NULL for $column") + } + + private fun recordGeneration(ready: BrokerReady) { + require(ready.workerPid > 0) + require(ready.workerPid !in workerPids) { "worker PID ${ready.workerPid} was already observed" } + require(ready.epoch !in workerEpochs) { "worker epoch ${ready.epoch} was already observed" } + workerPids += ready.workerPid + workerEpochs += ready.epoch + } + + private fun nanosToMillis(nanos: Long): Long = nanos / 1_000_000L + + private companion object { + const val REQUIRED_STALL_MILLIS = 300L + const val REQUIRED_STALL_NANOS = REQUIRED_STALL_MILLIS * 1_000_000L + const val STALL_DISCOVERY_TIMEOUT_MILLIS = 10_000L + const val STALL_POLL_INTERVAL_MILLIS = 10L + const val SLOW_READER_DRAIN_TIMEOUT_MILLIS = 30_000L + const val MAXIMUM_ENCODED_FRAME_BYTES = + (OlpbProtocol.HEADER_LENGTH + OlpbProtocol.MAXIMUM_FRAME_PAYLOAD).toLong() + } +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerService.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerService.kt new file mode 100644 index 00000000..f3e62d5b --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/BrokerService.kt @@ -0,0 +1,950 @@ +package dev.oliphaunt.androidbrokerspike + +import android.app.Service +import android.content.Intent +import android.content.pm.ApplicationInfo +import android.os.Bundle +import android.os.Debug +import android.os.IBinder +import android.os.ParcelFileDescriptor +import android.os.Process +import android.os.SystemClock +import android.system.Os +import android.system.OsConstants +import android.system.StructPollfd +import android.util.Log +import dev.oliphaunt.AndroidNativeDirectEngine +import dev.oliphaunt.DurabilityProfile +import dev.oliphaunt.EngineMode +import dev.oliphaunt.OliphauntConfig +import dev.oliphaunt.OliphauntSession +import dev.oliphaunt.ProtocolRequest +import kotlinx.coroutines.runBlocking +import java.io.Closeable +import java.io.File +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.util.UUID +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference + +/** + * Experimental worker intended to run as an unexported `android:process=":broker"` service. + * Binder is the out-of-band control plane; the reliable socket is the bounded data plane. + */ +internal class BrokerService : Service() { + private enum class WorkerState { + CREATED, + OPENING, + READY, + RUNNING, + DETACHING, + FAILED, + CLOSED, + } + + private data class ActiveRequest( + val lifecycle: OlpbRequestLifecycle, + val assembler: OlpbFrontendRequestAssembler, + ) + + private data class NativePostgresOutputWitness( + val requestId: Long, + val backendBytes: Long, + val observedAtElapsedRealtimeNanos: Long, + ) + + private val epoch = UUID.randomUUID() + private val workerPid = Process.myPid() + private val attachStarted = AtomicBoolean(false) + private val workerState = AtomicReference(WorkerState.CREATED) + private val armedFault = AtomicReference(null) + private val nativePostgresOutputWitness = + AtomicReference(null) + private val requestLock = Any() + private val requestsStarted = AtomicLong(0) + private val requestsCompleted = AtomicLong(0) + private val cancellationRequests = AtomicLong(0) + private val databaseExecutor = + Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "oliphaunt-broker-database").apply { isDaemon = true } + } + // This must never share the deliberately blockable database executor. + private val watchdogExecutor = + Executors.newSingleThreadScheduledExecutor { runnable -> + Thread(runnable, "oliphaunt-broker-watchdog").apply { isDaemon = true } + } + + @Volatile + private var selectedProtocolVersion = OlpbProtocol.MAXIMUM_VERSION + + @Volatile + private var endpoint: BrokerSocketEndpoint? = null + + @Volatile + private var session: OliphauntSession? = null + + @Volatile + private var activeRequestId = 0L + + @Volatile + private var nativeDispatchStarted = false + + @Volatile + private var lastError: String? = null + + @Volatile + private var lastRequestId = 0L + + private var activeRequest: ActiveRequest? = null + + private val binder = + object : IOliphauntBroker.Stub() { + override fun hello(request: Bundle, dataChannel: ParcelFileDescriptor): Bundle = + handleHello(request, dataChannel) + + override fun control(request: Bundle): Bundle = handleControl(request) + } + + override fun onBind(intent: Intent?): IBinder = binder + + override fun onDestroy() { + workerState.set(WorkerState.CLOSED) + endpoint?.requestStop() + endpoint = null + watchdogExecutor.shutdownNow() + databaseExecutor.shutdownNow() + super.onDestroy() + } + + private fun handleHello(request: Bundle, dataChannel: ParcelFileDescriptor): Bundle { + if (!attachStarted.compareAndSet(false, true)) { + dataChannel.closeQuietly() + return rejected("this worker epoch already accepted a data channel") + } + try { + validateHello(request) + selectedProtocolVersion = + minOf( + request.getInt( + BrokerContract.MAXIMUM_PROTOCOL_VERSION, + OlpbProtocol.MAXIMUM_VERSION, + ), + OlpbProtocol.MAXIMUM_VERSION, + ) + val ownedEndpoint = BrokerSocketEndpoint.takeOwnership(dataChannel) + endpoint = ownedEndpoint + workerState.set(WorkerState.OPENING) + + val openFuture = + databaseExecutor.submit { + val root = + File( + noBackupFilesDir, + "oliphaunt-android-broker/${BrokerContract.ROOT_ID}", + ) + val engine = AndroidNativeDirectEngine(applicationContext) + runBlocking { + engine.open( + OliphauntConfig( + mode = EngineMode.NativeDirect, + root = root.absolutePath, + durability = DurabilityProfile.Safe, + username = "oliphaunt_broker", + database = "postgres", + ), + ) + } + } + session = openFuture.get() + workerState.set(WorkerState.READY) + databaseExecutor.execute { runDataLoop(ownedEndpoint) } + return ready() + } catch (error: Throwable) { + val reason = safeError(error) + Log.e(TAG, "broker Hello/open failed: $reason", error) + lastError = reason + workerState.set(WorkerState.FAILED) + endpoint?.requestStop() + endpoint = null + dataChannel.closeQuietly() + return rejected(reason) + } + } + + private fun validateHello(request: Bundle) { + if (request.getString(BrokerContract.MESSAGE) != BrokerContract.HELLO) { + throw IllegalArgumentException("expected hello control message") + } + val minimum = + request.getInt( + BrokerContract.MINIMUM_PROTOCOL_VERSION, + OlpbProtocol.MINIMUM_VERSION, + ) + val maximum = + request.getInt( + BrokerContract.MAXIMUM_PROTOCOL_VERSION, + OlpbProtocol.MAXIMUM_VERSION, + ) + if (minimum > OlpbProtocol.MAXIMUM_VERSION || maximum < OlpbProtocol.MINIMUM_VERSION) { + throw IllegalArgumentException("no supported OLPB protocol version") + } + if (request.getLong(BrokerContract.EXPECTED_ABI_KEY) != BrokerContract.EXPECTED_ABI) { + throw IllegalArgumentException("liboliphaunt ABI mismatch") + } + if (request.getString(BrokerContract.ROOT_ID_KEY) != BrokerContract.ROOT_ID) { + throw IllegalArgumentException("this spike supports only the default root") + } + if ( + request.getString(BrokerContract.STARTUP_CONFIGURATION_DIGEST_KEY) != + BrokerContract.STARTUP_CONFIGURATION_DIGEST + ) { + throw IllegalArgumentException("startup configuration digest mismatch") + } + } + + private fun handleControl(request: Bundle): Bundle { + val message = request.getString(BrokerContract.MESSAGE) + ?: return rejected("control message is missing") + return when (message) { + BrokerContract.DIAGNOSTICS -> withValidEpoch(request) { diagnostics() } + BrokerContract.CANCEL -> withValidEpoch(request) { cancel(request) } + BrokerContract.INJECT_FAULT -> withValidEpoch(request) { injectFault(request) } + BrokerContract.DETACH -> withValidEpoch(request) { detach() } + else -> rejected("unsupported control message $message") + } + } + + private inline fun withValidEpoch(request: Bundle, operation: () -> Bundle): Bundle { + val expected = request.getString(BrokerContract.EPOCH) + if (expected != epoch.toString()) { + return rejected("stale worker epoch") + } + return operation() + } + + /** Runs directly on a Binder thread, outside both database executors. */ + private fun cancel(request: Bundle): Bundle { + val cancelStarted = android.os.SystemClock.elapsedRealtimeNanos() + val requestedId = + if (request.containsKey(BrokerContract.REQUEST_ID)) { + request.getLong(BrokerContract.REQUEST_ID) + } else { + activeRequestId + } + val shouldCallNative = + synchronized(requestLock) { + val active = activeRequest + ?: return rejected("there is no active request") + if (active.lifecycle.requestId != requestedId) { + return rejected("request $requestedId is not active") + } + active.lifecycle.requestCancellation() && active.lifecycle.nativeDispatchStarted + } + cancellationRequests.incrementAndGet() + Log.i( + TAG, + "cancel request=$requestedId state=${workerState.get()} nativeDispatch=$nativeDispatchStarted " + + "callNative=$shouldCallNative", + ) + if (shouldCallNative) { + val currentSession = session ?: return rejected("database session is unavailable") + try { + runBlocking { currentSession.cancel() } + } catch (error: Throwable) { + return rejected("native cancel failed: ${safeError(error)}") + } + } + Log.i( + TAG, + "cancel acknowledged request=$requestedId elapsedMillis=" + + ((android.os.SystemClock.elapsedRealtimeNanos() - cancelStarted) / 1_000_000L), + ) + return success(BrokerContract.CANCEL_OBSERVED).apply { + putLong(BrokerContract.REQUEST_ID, requestedId) + } + } + + /** Uses cached request state and process APIs only; it remains live if the DB executor wedges. */ + private fun diagnostics(): Bundle { + val socket = endpoint?.diagnostics() + val witness = nativePostgresOutputWitness.get() + return success(BrokerContract.DIAGNOSTICS).apply { + putString(BrokerContract.STATE, workerState.get().name.lowercase()) + activeRequestId.takeIf { it != 0L }?.let { + putLong(BrokerContract.ACTIVE_REQUEST_ID, it) + } + putBoolean(BrokerContract.NATIVE_DISPATCH_STARTED, nativeDispatchStarted) + putBoolean( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_OBSERVED, + witness != null, + ) + witness?.let { + putLong( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_REQUEST_ID, + it.requestId, + ) + putLong( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_BACKEND_BYTES, + it.backendBytes, + ) + putLong( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_ELAPSED_REALTIME_NANOS, + it.observedAtElapsedRealtimeNanos, + ) + } + putString(BrokerContract.TRANSACTION_STATUS, "unknown") + putLong(BrokerContract.CURRENT_PSS_BYTES, Debug.getPss().toLong() * 1024L) + putLong(BrokerContract.CURRENT_RSS_BYTES, currentRssBytes()) + putInt( + BrokerContract.REQUESTED_SOCKET_SEND_BUFFER_BYTES, + endpoint?.requestedSocketSendBufferBytes ?: -1, + ) + socket?.let { + putLong( + BrokerContract.DIAGNOSTICS_SAMPLE_ELAPSED_REALTIME_NANOS, + it.sampleElapsedRealtimeNanos, + ) + putBoolean( + BrokerContract.SOCKET_NON_BLOCKING_PROBE_SUCCEEDED, + it.nonBlockingProbeSucceeded, + ) + if (it.nonBlockingProbeSucceeded) { + putBoolean(BrokerContract.SOCKET_NON_BLOCKING, it.nonBlocking) + } + putBoolean(BrokerContract.SOCKET_POLL_SUCCEEDED, it.pollSucceeded) + if (it.pollSucceeded) { + putBoolean(BrokerContract.SOCKET_WRITABLE_NOW, it.writableNow) + } + putBoolean(BrokerContract.SOCKET_WRITE_IN_PROGRESS, it.activeWrite != null) + it.activeWrite?.let { activeWrite -> + putLong( + BrokerContract.SOCKET_ACTIVE_WRITE_SEQUENCE, + activeWrite.sequence, + ) + putLong( + BrokerContract.SOCKET_ACTIVE_WRITE_REQUEST_ID, + activeWrite.requestId, + ) + putInt( + BrokerContract.SOCKET_ACTIVE_WRITE_FRAME_TYPE, + activeWrite.frameType.wireValue, + ) + putLong( + BrokerContract.SOCKET_ACTIVE_WRITE_STARTED_ELAPSED_REALTIME_NANOS, + activeWrite.startedElapsedRealtimeNanos, + ) + putInt( + BrokerContract.SOCKET_ACTIVE_WRITE_ENCODED_BYTES, + activeWrite.encodedBytes, + ) + } + putLong(BrokerContract.SOCKET_WRITES_COMPLETED, it.writesCompleted) + putLong( + BrokerContract.SOCKET_COMPLETED_ENCODED_BYTES, + it.completedEncodedBytes, + ) + } + putLong("requestsStarted", requestsStarted.get()) + putLong("requestsCompleted", requestsCompleted.get()) + putLong("cancellationRequests", cancellationRequests.get()) + lastError?.let { putString(BrokerContract.REASON, it) } + } + } + + private fun injectFault(request: Bundle): Bundle { + if (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE == 0) { + return rejected("fault injection requires a debuggable build") + } + val wireValue = request.getString(BrokerContract.FAULT) + ?: return rejected("fault is missing") + val fault = BrokerFault.entries.firstOrNull { it.wireValue == wireValue } + ?: return rejected("unknown fault $wireValue") + if (!armedFault.compareAndSet(null, fault)) { + return rejected("another fault is already armed") + } + return success(BrokerContract.INJECT_FAULT).apply { + putString(BrokerContract.FAULT, fault.wireValue) + } + } + + private fun detach(): Bundle { + workerState.set(WorkerState.DETACHING) + endpoint?.requestStop() + return success(BrokerContract.DETACH) + } + + private fun runDataLoop(channel: BrokerSocketEndpoint) { + try { + while (workerState.get() != WorkerState.DETACHING) { + val frame = channel.readFrame(epoch) + ?: throw IOException("broker data channel closed without channelClose") + if (frame.header.protocolVersion != selectedProtocolVersion) { + throw OlpbProtocolException( + "frame version ${frame.header.protocolVersion} does not match $selectedProtocolVersion", + ) + } + when (frame.header.frameType) { + OlpbFrameType.REQUEST_BEGIN -> beginRequest(frame, channel) + OlpbFrameType.REQUEST_BYTES -> appendRequestBytes(frame, channel) + OlpbFrameType.REQUEST_END -> finishAndExecuteRequest(frame, channel) + OlpbFrameType.PING -> + channel.writeFrame( + OlpbFrame( + frameType = OlpbFrameType.PONG, + epoch = epoch, + requestId = 0, + ), + ) + OlpbFrameType.CHANNEL_CLOSE -> { + workerState.set(WorkerState.DETACHING) + break + } + else -> + throw OlpbProtocolException( + "host sent illegal ${frame.header.frameType} frame", + ) + } + } + } catch (error: Throwable) { + if (workerState.get() != WorkerState.DETACHING) { + lastError = safeError(error) + workerState.set(WorkerState.FAILED) + try { + channel.writeFrame( + OlpbFrame( + frameType = OlpbFrameType.PROTOCOL_ERROR, + epoch = epoch, + requestId = 0, + payload = safeError(error).toByteArray(Charsets.UTF_8), + ), + ) + } catch (_: Throwable) { + // The socket itself is commonly the failure source. + } + } + } finally { + channel.requestStop() + endpoint = null + closeSession() + synchronized(requestLock) { activeRequest = null } + activeRequestId = 0 + nativeDispatchStarted = false + if (workerState.get() == WorkerState.DETACHING) { + workerState.set(WorkerState.CLOSED) + } + stopSelf() + } + } + + private fun beginRequest(frame: OlpbFrame, channel: BrokerSocketEndpoint) { + if (frame.payload.isNotEmpty()) { + channel.writeRejected(frame.header.requestId, epoch, "requestBegin payload must be empty") + return + } + synchronized(requestLock) { + if (activeRequest != null) { + throw OlpbProtocolException("requestBegin while another request is active") + } + if ( + lastRequestId != 0L && + java.lang.Long.compareUnsigned(frame.header.requestId, lastRequestId) <= 0 + ) { + throw OlpbProtocolException("request IDs must be strictly increasing") + } + val lifecycle = OlpbRequestLifecycle(epoch, frame.header.requestId) + lifecycle.beginReceiving() + nativePostgresOutputWitness.set(null) + nativeDispatchStarted = false + activeRequest = ActiveRequest(lifecycle, OlpbFrontendRequestAssembler()) + activeRequestId = frame.header.requestId + lastRequestId = frame.header.requestId + requestsStarted.incrementAndGet() + } + } + + private fun appendRequestBytes(frame: OlpbFrame, channel: BrokerSocketEndpoint) { + val active = synchronized(requestLock) { + activeRequest?.takeIf { it.lifecycle.requestId == frame.header.requestId } + ?: throw OlpbProtocolException("requestBytes does not match the active request") + } + val canceled = synchronized(requestLock) { + active.lifecycle.state == OlpbRequestState.TERMINAL && + active.lifecycle.terminalResult == OlpbTerminalResult.CANCELED + } + if (canceled) { + return + } + try { + active.assembler.append(frame.payload) + } catch (error: Throwable) { + synchronized(requestLock) { + active.lifecycle.establishTerminal(OlpbTerminalResult.REJECTED) + activeRequest = null + } + activeRequestId = 0 + channel.writeRejected(frame.header.requestId, epoch, safeError(error)) + } + } + + private fun finishAndExecuteRequest(frame: OlpbFrame, channel: BrokerSocketEndpoint) { + if (frame.payload.isNotEmpty()) { + channel.writeRejected(frame.header.requestId, epoch, "requestEnd payload must be empty") + clearActiveRequest(frame.header.requestId) + return + } + val active = synchronized(requestLock) { + activeRequest?.takeIf { it.lifecycle.requestId == frame.header.requestId } + ?: throw OlpbProtocolException("requestEnd does not match the active request") + } + if ( + active.lifecycle.state == OlpbRequestState.TERMINAL && + active.lifecycle.terminalResult == OlpbTerminalResult.CANCELED + ) { + channel.writeRejected(frame.header.requestId, epoch, "request canceled before dispatch") + clearActiveRequest(frame.header.requestId) + return + } + + val requestBytes = + try { + active.assembler.finish() + } catch (error: Throwable) { + synchronized(requestLock) { + active.lifecycle.establishTerminal(OlpbTerminalResult.REJECTED) + } + channel.writeRejected(frame.header.requestId, epoch, safeError(error)) + clearActiveRequest(frame.header.requestId) + return + } + synchronized(requestLock) { active.lifecycle.finishReceiving() } + workerState.set(WorkerState.RUNNING) + val fault = armedFault.getAndSet(null) + + try { + when (fault) { + BrokerFault.EXECUTOR_DEADLOCK_WITH_FAIL_STOP -> triggerExecutorDeadlockFailStop() + else -> Unit + } + val nativeRequest = ProtocolRequest(requestBytes) + val nativeOutputWitnessCounter = + if (fault == BrokerFault.NATIVE_FAIL_STOP_WATCHDOG) { + NativePostgresOutputWitnessCounter( + BrokerContract.NATIVE_POSTGRES_OUTPUT_WITNESS_THRESHOLD_BYTES, + ) + } else { + null + } + val currentSession = session ?: throw IllegalStateException("database session is unavailable") + runBlocking { + synchronized(requestLock) { + active.lifecycle.beginNativeDispatch() + nativeDispatchStarted = true + } + currentSession.execProtocolStream(nativeRequest) { response -> + val pendingOutputWitness = + nativeOutputWitnessCounter?.consume(response.bytes) + channel.writeResponseBytes( + requestId = frame.header.requestId, + epoch = epoch, + bytes = response.bytes, + ) + pendingOutputWitness?.let { outputWitness -> + val witness = + NativePostgresOutputWitness( + requestId = frame.header.requestId, + backendBytes = outputWitness.backendBytes, + observedAtElapsedRealtimeNanos = + android.os.SystemClock.elapsedRealtimeNanos(), + ) + check(nativePostgresOutputWitness.compareAndSet(null, witness)) { + "native PostgreSQL-output witness was already published" + } + armFailStopWatchdog( + reason = "native pg_sleep after >4 MiB PostgreSQL output", + delayMillis = + BrokerContract.NATIVE_POSTGRES_OUTPUT_WATCHDOG_DELAY_MILLIS, + ) + } + } + } + if (fault == BrokerFault.AFTER_NATIVE_SUCCESS_BEFORE_COMPLETED) { + armFailStopWatchdog("after native success", delayMillis = 1) + CountDownLatch(1).await() + } + synchronized(requestLock) { + active.lifecycle.establishTerminal(OlpbTerminalResult.COMPLETED) + } + channel.writeFrame( + OlpbFrame( + frameType = OlpbFrameType.COMPLETED, + epoch = epoch, + requestId = frame.header.requestId, + ), + ) + requestsCompleted.incrementAndGet() + clearActiveRequest(frame.header.requestId) + workerState.set(WorkerState.READY) + } catch (error: Throwable) { + synchronized(requestLock) { + active.lifecycle.establishTerminal(OlpbTerminalResult.OUTCOME_UNKNOWN) + } + try { + channel.writeFrame( + OlpbFrame( + frameType = OlpbFrameType.OUTCOME_UNKNOWN, + epoch = epoch, + requestId = frame.header.requestId, + ), + ) + } catch (_: Throwable) { + // A failed or killed transport cannot carry the terminal marker. + } + clearActiveRequest(frame.header.requestId) + throw error + } + } + + private fun triggerExecutorDeadlockFailStop(): Nothing { + armFailStopWatchdog("database executor deadlock") + CountDownLatch(1).await() + error("unreachable after executor deadlock") + } + + private fun armFailStopWatchdog( + reason: String, + delayMillis: Long = FAIL_STOP_DELAY_MILLIS, + ) { + lastError = "fail-stop watchdog armed: $reason" + watchdogExecutor.schedule( + { + lastError = "fail-stop watchdog fired: $reason" + try { + Os.kill(workerPid, OsConstants.SIGABRT) + } catch (_: Throwable) { + Process.killProcess(workerPid) + } + }, + delayMillis, + TimeUnit.MILLISECONDS, + ) + } + + private fun clearActiveRequest(requestId: Long) { + synchronized(requestLock) { + if (activeRequest?.lifecycle?.requestId == requestId) { + activeRequest = null + } + } + if (activeRequestId == requestId) activeRequestId = 0 + nativeDispatchStarted = false + } + + private fun closeSession() { + val current = session ?: return + session = null + try { + runBlocking { current.close() } + } catch (error: Throwable) { + Log.w(TAG, "session close failed", error) + } + } + + private fun ready(): Bundle = + success(BrokerContract.READY).apply { + putInt(BrokerContract.SELECTED_PROTOCOL_VERSION, selectedProtocolVersion) + putLong(BrokerContract.ABI_VERSION, BrokerContract.EXPECTED_ABI) + putString(BrokerContract.RUNTIME_VERSION, "local-android-spike") + putInt(BrokerContract.POSTGRES_MAJOR_VERSION, 18) + putString(BrokerContract.ROOT_MANIFEST_DIGEST, BrokerContract.STARTUP_CONFIGURATION_DIGEST) + putStringArray(BrokerContract.ACTUAL_CAPABILITIES, BrokerContract.requestedCapabilities) + putString(BrokerContract.ACTUAL_RUNTIME_CONFIGURATION, BrokerContract.STARTUP_CONFIGURATION_DIGEST) + } + + private fun success(message: String): Bundle = + Bundle().apply { + putString(BrokerContract.MESSAGE, message) + putBoolean(BrokerContract.SUCCESS, true) + putString(BrokerContract.EPOCH, epoch.toString()) + putInt(BrokerContract.WORKER_PID, workerPid) + } + + private fun rejected(reason: String): Bundle = + Bundle().apply { + putString(BrokerContract.MESSAGE, BrokerContract.REJECTED) + putBoolean(BrokerContract.SUCCESS, false) + putString(BrokerContract.REASON, reason) + putString(BrokerContract.EPOCH, epoch.toString()) + putInt(BrokerContract.WORKER_PID, workerPid) + } + + private fun currentRssBytes(): Long = + try { + File("/proc/self/status").useLines { lines -> + lines + .firstOrNull { it.startsWith("VmRSS:") } + ?.trim() + ?.split(Regex("\\s+")) + ?.getOrNull(1) + ?.toLongOrNull() + ?.times(1024L) + ?: -1L + } + } catch (_: Throwable) { + -1L + } + + private fun safeError(error: Throwable): String { + val cause = error.cause ?: error + return "${cause::class.java.simpleName}: ${cause.message ?: "unknown failure"}".take(512) + } + + private companion object { + const val TAG = "OliphauntBroker" + const val FAIL_STOP_DELAY_MILLIS = 1_000L + } +} + +internal data class NativePostgresOutputWitnessSample( + val backendBytes: Long, +) + +/** Reports once when cumulative bytes emitted by the native PostgreSQL stream exceed the threshold. */ +internal class NativePostgresOutputWitnessCounter( + private val thresholdBytes: Long, +) { + private var backendBytes = 0L + private var witnessed = false + + init { + require(thresholdBytes >= 0) { "native output threshold must not be negative" } + } + + fun consume(bytes: ByteArray): NativePostgresOutputWitnessSample? { + if (witnessed) return null + backendBytes = Math.addExact(backendBytes, bytes.size.toLong()) + if (backendBytes <= thresholdBytes) return null + witnessed = true + return NativePostgresOutputWitnessSample(backendBytes) + } +} + +private data class ActiveSocketWrite( + val sequence: Long, + val requestId: Long, + val frameType: OlpbFrameType, + val startedElapsedRealtimeNanos: Long, + val encodedBytes: Int, +) + +private data class BrokerSocketDiagnostics( + val sampleElapsedRealtimeNanos: Long, + val nonBlockingProbeSucceeded: Boolean, + val nonBlocking: Boolean, + val pollSucceeded: Boolean, + val writableNow: Boolean, + val activeWrite: ActiveSocketWrite?, + val writesCompleted: Long, + val completedEncodedBytes: Long, +) + +/** Owns two dup'd descriptors for one reliable full-duplex socket endpoint. */ +private class BrokerSocketEndpoint private constructor( + private val inputDescriptor: ParcelFileDescriptor, + private val outputDescriptor: ParcelFileDescriptor, + private val input: InputStream, + private val output: OutputStream, + val requestedSocketSendBufferBytes: Int, +) : Closeable { + private val closed = AtomicBoolean(false) + private val writeLock = Any() + private val nextWriteSequence = AtomicLong(0) + private val activeWrite = AtomicReference(null) + private val writesCompleted = AtomicLong(0) + private val completedEncodedBytes = AtomicLong(0) + + fun readFrame(expectedEpoch: UUID): OlpbFrame? { + val headerBytes = input.readExactlyOrNull(OlpbProtocol.HEADER_LENGTH) ?: run { + outputDescriptor.checkError() + return null + } + val header = OlpbFrameCodec.decodeHeader(headerBytes, expectedEpoch) + val payload = input.readExactly(header.payloadLength) + return OlpbFrame(header, payload) + } + + fun writeFrame(frame: OlpbFrame) { + val encoded = OlpbFrameCodec.encode(frame) + synchronized(writeLock) { + check(!closed.get()) { "broker socket is closed" } + val write = + ActiveSocketWrite( + sequence = nextWriteSequence.incrementAndGet(), + requestId = frame.header.requestId, + frameType = frame.header.frameType, + startedElapsedRealtimeNanos = SystemClock.elapsedRealtimeNanos(), + encodedBytes = encoded.size, + ) + check(activeWrite.compareAndSet(null, write)) { "socket write probe is already active" } + try { + output.write(encoded) + output.flush() + completedEncodedBytes.addAndGet(encoded.size.toLong()) + writesCompleted.incrementAndGet() + } finally { + activeWrite.compareAndSet(write, null) + } + } + } + + /** Samples the descriptor and atomic write counters without waiting for [writeLock]. */ + fun diagnostics(): BrokerSocketDiagnostics { + val descriptor = outputDescriptor.fileDescriptor + val flags = + try { + Os.fcntlInt(descriptor, OsConstants.F_GETFL, 0) + } catch (_: Throwable) { + null + } + val pollDescriptor = + StructPollfd().apply { + fd = descriptor + events = OsConstants.POLLOUT.toShort() + } + var pollSucceeded = false + var writableNow = false + try { + Os.poll(arrayOf(pollDescriptor), 0) + pollSucceeded = true + writableNow = + pollDescriptor.revents.toInt() and OsConstants.POLLOUT != 0 + } catch (_: Throwable) { + // The success bit keeps a probe failure distinct from backpressure. + } + return BrokerSocketDiagnostics( + sampleElapsedRealtimeNanos = SystemClock.elapsedRealtimeNanos(), + nonBlockingProbeSucceeded = flags != null, + nonBlocking = flags?.let { it and OsConstants.O_NONBLOCK != 0 } ?: false, + pollSucceeded = pollSucceeded, + writableNow = writableNow, + activeWrite = activeWrite.get(), + writesCompleted = writesCompleted.get(), + completedEncodedBytes = completedEncodedBytes.get(), + ) + } + + fun writeResponseBytes(requestId: Long, epoch: UUID, bytes: ByteArray) { + if (bytes.isEmpty()) return + var offset = 0 + while (offset < bytes.size) { + val end = minOf(bytes.size, offset + OlpbProtocol.MAXIMUM_FRAME_PAYLOAD) + writeFrame( + OlpbFrame( + frameType = OlpbFrameType.RESPONSE_BYTES, + epoch = epoch, + requestId = requestId, + payload = bytes.copyOfRange(offset, end), + ), + ) + offset = end + } + } + + fun writeRejected(requestId: Long, epoch: UUID, reason: String) { + writeFrame( + OlpbFrame( + frameType = OlpbFrameType.REJECTED, + epoch = epoch, + requestId = requestId, + payload = reason.take(512).toByteArray(Charsets.UTF_8), + ), + ) + } + + fun requestStop() = close() + + override fun close() { + if (!closed.compareAndSet(false, true)) return + try { + Os.shutdown(outputDescriptor.fileDescriptor, OsConstants.SHUT_RDWR) + } catch (_: Throwable) { + // Closing both owned descriptors is the fallback wakeup. + } + input.closeQuietly() + output.closeQuietly() + inputDescriptor.closeQuietly() + outputDescriptor.closeQuietly() + } + + companion object { + private const val REQUESTED_SEND_BUFFER_BYTES = 512 * 1024 + + fun takeOwnership(descriptor: ParcelFileDescriptor): BrokerSocketEndpoint { + val inputDescriptor = + try { + ParcelFileDescriptor.dup(descriptor.fileDescriptor) + } catch (error: Throwable) { + descriptor.closeQuietly() + throw error + } + try { + try { + Os.setsockoptInt( + descriptor.fileDescriptor, + OsConstants.SOL_SOCKET, + OsConstants.SO_SNDBUF, + REQUESTED_SEND_BUFFER_BYTES, + ) + } catch (_: Throwable) { + // The socket keeps its kernel default when the request fails. + } + return BrokerSocketEndpoint( + inputDescriptor = inputDescriptor, + outputDescriptor = descriptor, + input = ParcelFileDescriptor.AutoCloseInputStream(inputDescriptor), + output = ParcelFileDescriptor.AutoCloseOutputStream(descriptor), + // Android's public Os facade can set but not query SO_SNDBUF. + // This is the requested value, not a queried effective value. + // Slow-reader PSS/RSS spans are reported independently. + requestedSocketSendBufferBytes = REQUESTED_SEND_BUFFER_BYTES, + ) + } catch (error: Throwable) { + inputDescriptor.closeQuietly() + descriptor.closeQuietly() + throw error + } + } + } +} + +private fun InputStream.readExactlyOrNull(length: Int): ByteArray? { + if (length == 0) return ByteArray(0) + val result = ByteArray(length) + val first = read() + if (first < 0) return null + result[0] = first.toByte() + var offset = 1 + while (offset < length) { + val count = read(result, offset, length - offset) + if (count < 0) throw IOException("truncated broker frame") + if (count == 0) continue + offset += count + } + return result +} + +private fun InputStream.readExactly(length: Int): ByteArray = + readExactlyOrNull(length) ?: throw IOException("truncated broker frame") + +private fun Closeable.closeQuietly() { + try { + close() + } catch (_: Throwable) { + // Best-effort experimental cleanup. + } +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/MainActivity.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/MainActivity.kt new file mode 100644 index 00000000..f2fdd018 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/MainActivity.kt @@ -0,0 +1,90 @@ +package dev.oliphaunt.androidbrokerspike + +import android.app.Activity +import android.os.Bundle +import android.util.Log +import android.widget.TextView +import java.io.File +import java.io.FileOutputStream +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.json.JSONObject + +internal class MainActivity : Activity() { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val output = TextView(this).apply { text = "Android broker experiment running…" } + setContentView(output) + + val runNonce = intent.getStringExtra("runNonce") ?: "manual-${System.currentTimeMillis()}" + val strategy = intent.getStringExtra("strategy") ?: "full" + scope.launch { + val report = + try { + withContext(Dispatchers.IO) { + BrokerExperiment(applicationContext).run(runNonce, strategy) + } + } catch (error: Throwable) { + Log.e(TAG, "Android broker experiment failed", error) + JSONObject() + .put("schema", "oliphaunt-android-native-broker-spike-v1") + .put("status", "FAIL") + .put("runNonce", runNonce) + .put("strategy", strategy) + .put("error", error.causeChain()) + } + publishReport(report) + val encoded = report.toString() + if (report.optString("status") == "PASS") { + Log.i(TAG, "$JSON_MARKER$encoded") + Log.i(TAG, PASS_MARKER) + output.text = "PASS\n$encoded" + } else { + Log.e(TAG, "$JSON_MARKER$encoded") + Log.e(TAG, FAIL_MARKER) + output.text = "FAIL\n$encoded" + } + } + } + + override fun onDestroy() { + scope.cancel() + super.onDestroy() + } + + private fun publishReport(report: JSONObject) { + val destination = File(filesDir, REPORT_NAME) + val temporary = File(filesDir, "$REPORT_NAME.tmp") + FileOutputStream(temporary).use { stream -> + stream.write(report.toString(2).toByteArray(Charsets.UTF_8)) + stream.write('\n'.code) + stream.fd.sync() + } + check(temporary.renameTo(destination)) { "failed to publish $REPORT_NAME atomically" } + } + + private companion object { + const val TAG = "OliphauntBrokerSpike" + const val REPORT_NAME = "android-broker-report.json" + const val JSON_MARKER = "OLIPHAUNT_ANDROID_BROKER_JSON " + const val PASS_MARKER = "OLIPHAUNT_ANDROID_BROKER_PASS" + const val FAIL_MARKER = "OLIPHAUNT_ANDROID_BROKER_FAIL" + } +} + +private fun Throwable.causeChain(): String { + val seen = mutableSetOf() + val parts = mutableListOf() + var current: Throwable? = this + while (current != null && seen.add(current) && parts.size < 8) { + parts += "${current.javaClass.simpleName}: ${current.message ?: "unknown failure"}" + current = current.cause + } + return parts.joinToString(" <- ").take(2_048) +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/OlpbProtocol.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/OlpbProtocol.kt new file mode 100644 index 00000000..7bb8b153 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/OlpbProtocol.kt @@ -0,0 +1,254 @@ +package dev.oliphaunt.androidbrokerspike + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.UUID + +internal object OlpbProtocol { + val magic: ByteArray = byteArrayOf(0x4f, 0x4c, 0x50, 0x42) + const val HEADER_LENGTH = 40 + const val MINIMUM_VERSION = 1 + const val MAXIMUM_VERSION = 1 + const val MAXIMUM_FRAME_PAYLOAD = 256 * 1024 + const val MAXIMUM_QUEUED_BYTES_PER_DIRECTION = 8 * 1024 * 1024 + const val DEFAULT_MAXIMUM_REQUEST_BYTES = 8 * 1024 * 1024 + const val KNOWN_FLAGS_MASK = 0 +} + +internal enum class OlpbFrameType( + val wireValue: Int, + val requiresRequestId: Boolean, +) { + REQUEST_BEGIN(1, true), + REQUEST_BYTES(2, true), + REQUEST_END(3, true), + RESPONSE_BYTES(4, true), + COMPLETED(5, true), + REJECTED(6, true), + OUTCOME_UNKNOWN(7, true), + CANCEL_REQUESTED(8, true), + CANCEL_OBSERVED(9, true), + PING(10, false), + PONG(11, false), + PROTOCOL_ERROR(12, false), + CHANNEL_CLOSE(13, false), + ; + + companion object { + fun fromWireValue(value: Int): OlpbFrameType = + entries.firstOrNull { it.wireValue == value } + ?: throw OlpbProtocolException("unknown OLPB frame type $value") + } +} + +internal data class OlpbFrameHeader( + val protocolVersion: Int = OlpbProtocol.MAXIMUM_VERSION, + val frameType: OlpbFrameType, + val flags: Int = 0, + val epoch: UUID, + /** Raw unsigned 64-bit request ID bits. Zero is reserved. */ + val requestId: Long, + val payloadLength: Int, +) + +internal class OlpbFrame( + val header: OlpbFrameHeader, + val payload: ByteArray, +) { + constructor( + protocolVersion: Int = OlpbProtocol.MAXIMUM_VERSION, + frameType: OlpbFrameType, + flags: Int = 0, + epoch: UUID, + requestId: Long, + payload: ByteArray = ByteArray(0), + ) : this( + header = + OlpbFrameHeader( + protocolVersion = protocolVersion, + frameType = frameType, + flags = flags, + epoch = epoch, + requestId = requestId, + payloadLength = payload.size, + ), + payload = payload, + ) +} + +internal class OlpbProtocolException(message: String) : IllegalArgumentException(message) + +internal object OlpbFrameCodec { + fun encode(frame: OlpbFrame): ByteArray { + if (frame.payload.size != frame.header.payloadLength) { + throw OlpbProtocolException( + "OLPB payload length declared ${frame.header.payloadLength}, actual ${frame.payload.size}", + ) + } + val header = encodeHeader(frame.header) + return ByteArray(header.size + frame.payload.size).also { encoded -> + header.copyInto(encoded) + frame.payload.copyInto(encoded, destinationOffset = header.size) + } + } + + fun encodeHeader(header: OlpbFrameHeader): ByteArray { + validateHeader(header) + return ByteBuffer + .allocate(OlpbProtocol.HEADER_LENGTH) + .order(ByteOrder.BIG_ENDIAN) + .apply { + put(OlpbProtocol.magic) + putShort(header.protocolVersion.toShort()) + putShort(OlpbProtocol.HEADER_LENGTH.toShort()) + put(header.frameType.wireValue.toByte()) + put(header.flags.toByte()) + putShort(0) + putLong(header.epoch.mostSignificantBits) + putLong(header.epoch.leastSignificantBits) + putLong(header.requestId) + putInt(header.payloadLength) + }.array() + } + + fun decodeHeader( + bytes: ByteArray, + expectedEpoch: UUID? = null, + maximumPayloadLength: Int = OlpbProtocol.MAXIMUM_FRAME_PAYLOAD, + ): OlpbFrameHeader { + if (bytes.size < OlpbProtocol.HEADER_LENGTH) { + throw OlpbProtocolException("truncated OLPB frame header") + } + if (!bytes.copyOfRange(0, OlpbProtocol.magic.size).contentEquals(OlpbProtocol.magic)) { + throw OlpbProtocolException("invalid OLPB magic") + } + val source = ByteBuffer.wrap(bytes, 4, OlpbProtocol.HEADER_LENGTH - 4).order(ByteOrder.BIG_ENDIAN) + val protocolVersion = source.short.toInt() and 0xffff + val headerLength = source.short.toInt() and 0xffff + if (headerLength != OlpbProtocol.HEADER_LENGTH) { + throw OlpbProtocolException("invalid OLPB header length $headerLength") + } + val frameType = OlpbFrameType.fromWireValue(source.get().toInt() and 0xff) + val flags = source.get().toInt() and 0xff + val reserved = source.short.toInt() and 0xffff + if (reserved != 0) { + throw OlpbProtocolException("nonzero OLPB reserved field $reserved") + } + val epoch = UUID(source.long, source.long) + if (expectedEpoch != null && epoch != expectedEpoch) { + throw OlpbProtocolException("stale OLPB epoch $epoch; expected $expectedEpoch") + } + val requestId = source.long + val payloadLengthUnsigned = source.int.toLong() and 0xffff_ffffL + if (payloadLengthUnsigned > Int.MAX_VALUE.toLong()) { + throw OlpbProtocolException("OLPB payload length $payloadLengthUnsigned is not representable") + } + val header = + OlpbFrameHeader( + protocolVersion = protocolVersion, + frameType = frameType, + flags = flags, + epoch = epoch, + requestId = requestId, + payloadLength = payloadLengthUnsigned.toInt(), + ) + validateHeader(header, maximumPayloadLength) + return header + } + + fun decode( + bytes: ByteArray, + expectedEpoch: UUID? = null, + maximumPayloadLength: Int = OlpbProtocol.MAXIMUM_FRAME_PAYLOAD, + ): OlpbFrame { + val header = decodeHeader(bytes, expectedEpoch, maximumPayloadLength) + val expectedLength = OlpbProtocol.HEADER_LENGTH + header.payloadLength + if (bytes.size != expectedLength) { + throw OlpbProtocolException("OLPB frame length ${bytes.size}; expected $expectedLength") + } + return OlpbFrame(header, bytes.copyOfRange(OlpbProtocol.HEADER_LENGTH, expectedLength)) + } + + fun validateHeader( + header: OlpbFrameHeader, + maximumPayloadLength: Int = OlpbProtocol.MAXIMUM_FRAME_PAYLOAD, + ) { + if (header.protocolVersion !in OlpbProtocol.MINIMUM_VERSION..OlpbProtocol.MAXIMUM_VERSION) { + throw OlpbProtocolException("unsupported OLPB protocol version ${header.protocolVersion}") + } + if (header.flags and OlpbProtocol.KNOWN_FLAGS_MASK.inv() != 0) { + throw OlpbProtocolException("unknown OLPB flags ${header.flags}") + } + if (header.frameType.requiresRequestId == (header.requestId == 0L)) { + throw OlpbProtocolException( + "invalid request ID ${header.requestId} for ${header.frameType}", + ) + } + if (maximumPayloadLength < 0 || header.payloadLength !in 0..maximumPayloadLength) { + throw OlpbProtocolException( + "OLPB payload length ${header.payloadLength} exceeds $maximumPayloadLength", + ) + } + } +} + +/** Incremental decoder with an aggregate unread-byte bound. */ +internal class OlpbFrameDecoder( + var expectedEpoch: UUID? = null, + private val maximumPayloadLength: Int = OlpbProtocol.MAXIMUM_FRAME_PAYLOAD, + private val maximumBufferedBytes: Int = OlpbProtocol.MAXIMUM_QUEUED_BYTES_PER_DIRECTION, +) { + private var buffered = ByteArray(0) + + fun append(bytes: ByteArray): List { + val combinedSize = buffered.size.toLong() + bytes.size.toLong() + if (combinedSize > maximumBufferedBytes) { + throw OlpbProtocolException( + "OLPB buffered bytes $combinedSize exceed $maximumBufferedBytes", + ) + } + if (bytes.isNotEmpty()) { + buffered += bytes + } + val frames = mutableListOf() + var offset = 0 + while (buffered.size - offset >= OlpbProtocol.HEADER_LENGTH) { + val header = + OlpbFrameCodec.decodeHeader( + buffered.copyOfRange(offset, offset + OlpbProtocol.HEADER_LENGTH), + expectedEpoch, + maximumPayloadLength, + ) + val frameLength = OlpbProtocol.HEADER_LENGTH + header.payloadLength + if (buffered.size - offset < frameLength) { + break + } + frames += + OlpbFrame( + header, + buffered.copyOfRange( + offset + OlpbProtocol.HEADER_LENGTH, + offset + frameLength, + ), + ) + offset += frameLength + } + if (offset > 0) { + buffered = buffered.copyOfRange(offset, buffered.size) + } + return frames + } + + fun finish(): List { + val frames = append(ByteArray(0)) + if (buffered.isNotEmpty()) { + throw OlpbProtocolException("truncated OLPB frame") + } + return frames + } + + fun reset(expectedEpoch: UUID? = null) { + buffered = ByteArray(0) + this.expectedEpoch = expectedEpoch + } +} diff --git a/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/OlpbState.kt b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/OlpbState.kt new file mode 100644 index 00000000..780d88e2 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/kotlin/dev/oliphaunt/androidbrokerspike/OlpbState.kt @@ -0,0 +1,148 @@ +package dev.oliphaunt.androidbrokerspike + +import java.util.UUID + +internal enum class OlpbRequestState { + QUEUED, + RECEIVING, + READY_TO_DISPATCH, + RUNNING, + CANCEL_REQUESTED, + TERMINAL, +} + +internal enum class OlpbTerminalResult { + COMPLETED, + REJECTED, + OUTCOME_UNKNOWN, + CANCELED, + NOT_STARTED, +} + +internal class OlpbRequestLifecycle( + val epoch: UUID, + val requestId: Long, +) { + var state: OlpbRequestState = OlpbRequestState.QUEUED + private set + var terminalResult: OlpbTerminalResult? = null + private set + var nativeDispatchStarted: Boolean = false + private set + + fun beginReceiving() = transition(OlpbRequestState.QUEUED, OlpbRequestState.RECEIVING) + + fun finishReceiving() = + transition(OlpbRequestState.RECEIVING, OlpbRequestState.READY_TO_DISPATCH) + + fun beginNativeDispatch() { + transition(OlpbRequestState.READY_TO_DISPATCH, OlpbRequestState.RUNNING) + nativeDispatchStarted = true + } + + fun requestCancellation(): Boolean = + when (state) { + OlpbRequestState.QUEUED, + OlpbRequestState.RECEIVING, + OlpbRequestState.READY_TO_DISPATCH, + -> establishTerminal(OlpbTerminalResult.CANCELED) + OlpbRequestState.RUNNING -> { + state = OlpbRequestState.CANCEL_REQUESTED + true + } + OlpbRequestState.CANCEL_REQUESTED, + OlpbRequestState.TERMINAL, + -> false + } + + fun establishTerminal(result: OlpbTerminalResult): Boolean { + if (state == OlpbRequestState.TERMINAL) return false + state = OlpbRequestState.TERMINAL + terminalResult = result + return true + } + + fun lossResult(): OlpbTerminalResult = + if (nativeDispatchStarted) OlpbTerminalResult.OUTCOME_UNKNOWN else OlpbTerminalResult.NOT_STARTED + + private fun transition(expected: OlpbRequestState, next: OlpbRequestState) { + if (state != expected) { + throw OlpbProtocolException("illegal request transition $state -> $next") + } + state = next + } +} + +/** Bounded PostgreSQL frontend-message assembler used between requestBegin/requestEnd. */ +internal class OlpbFrontendRequestAssembler( + private val maximumRequestBytes: Int = OlpbProtocol.DEFAULT_MAXIMUM_REQUEST_BYTES, +) { + private var bytes = ByteArray(minOf(16 * 1024, maximumRequestBytes)) + private var size = 0 + private var scanOffset = 0 + + init { + require(maximumRequestBytes >= 5) + } + + val byteCount: Int + get() = size + + fun append(chunk: ByteArray) { + val newSize = size.toLong() + chunk.size.toLong() + if (newSize > maximumRequestBytes) { + throw OlpbProtocolException("request bytes $newSize exceed $maximumRequestBytes") + } + ensureCapacity(newSize.toInt()) + chunk.copyInto(bytes, destinationOffset = size) + size = newSize.toInt() + scanCompleteMessages() + } + + fun finish(): ByteArray { + scanCompleteMessages() + if (size == 0) throw OlpbProtocolException("empty PostgreSQL frontend request") + if (scanOffset != size) { + val remaining = size - scanOffset + throw OlpbProtocolException( + if (remaining < 5) "truncated PostgreSQL message header" + else "truncated PostgreSQL message body", + ) + } + return bytes.copyOf(size) + } + + fun reset() { + size = 0 + scanOffset = 0 + } + + private fun scanCompleteMessages() { + while (size - scanOffset >= 5) { + val lengthOffset = scanOffset + 1 + val messageLength = + ((bytes[lengthOffset].toLong() and 0xff) shl 24) or + ((bytes[lengthOffset + 1].toLong() and 0xff) shl 16) or + ((bytes[lengthOffset + 2].toLong() and 0xff) shl 8) or + (bytes[lengthOffset + 3].toLong() and 0xff) + if (messageLength < 4) { + throw OlpbProtocolException("PostgreSQL message length $messageLength is smaller than 4") + } + val totalLength = messageLength + 1 + if (totalLength > maximumRequestBytes) { + throw OlpbProtocolException("PostgreSQL message bytes $totalLength exceed $maximumRequestBytes") + } + if (totalLength > size - scanOffset) return + scanOffset += totalLength.toInt() + } + } + + private fun ensureCapacity(required: Int) { + if (required <= bytes.size) return + var capacity = maxOf(1, bytes.size) + while (capacity < required) { + capacity = minOf(maximumRequestBytes, capacity * 2) + } + bytes = bytes.copyOf(capacity) + } +} diff --git a/spikes/android-native-broker/app/src/main/res/values/styles.xml b/spikes/android-native-broker/app/src/main/res/values/styles.xml new file mode 100644 index 00000000..82adc578 --- /dev/null +++ b/spikes/android-native-broker/app/src/main/res/values/styles.xml @@ -0,0 +1,7 @@ + + + + diff --git a/spikes/android-native-broker/app/src/test/kotlin/dev/oliphaunt/androidbrokerspike/OlpbProtocolTest.kt b/spikes/android-native-broker/app/src/test/kotlin/dev/oliphaunt/androidbrokerspike/OlpbProtocolTest.kt new file mode 100644 index 00000000..8bc6cd7d --- /dev/null +++ b/spikes/android-native-broker/app/src/test/kotlin/dev/oliphaunt/androidbrokerspike/OlpbProtocolTest.kt @@ -0,0 +1,141 @@ +package dev.oliphaunt.androidbrokerspike + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.UUID +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class OlpbProtocolTest { + private val epoch = UUID.fromString("00112233-4455-6677-8899-aabbccddeeff") + + @Test + fun headerUsesCanonicalFortyByteNetworkLayout() { + val payload = byteArrayOf(0x10, 0x20, 0x30) + val encoded = + OlpbFrameCodec.encode( + OlpbFrame( + frameType = OlpbFrameType.REQUEST_BYTES, + epoch = epoch, + requestId = 0x0102030405060708L, + payload = payload, + ), + ) + + assertEquals(OlpbProtocol.HEADER_LENGTH + payload.size, encoded.size) + assertContentEquals(byteArrayOf(0x4f, 0x4c, 0x50, 0x42), encoded.copyOfRange(0, 4)) + assertContentEquals(byteArrayOf(0, 1, 0, 40, 2, 0, 0, 0), encoded.copyOfRange(4, 12)) + assertContentEquals( + byteArrayOf( + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88.toByte(), 0x99.toByte(), 0xaa.toByte(), 0xbb.toByte(), + 0xcc.toByte(), 0xdd.toByte(), 0xee.toByte(), 0xff.toByte(), + ), + encoded.copyOfRange(12, 28), + ) + assertContentEquals(payload, OlpbFrameCodec.decode(encoded, epoch).payload) + } + + @Test + fun incrementalDecoderHandlesFragmentedAndAdjacentFrames() { + val first = + OlpbFrameCodec.encode( + OlpbFrame(frameType = OlpbFrameType.PING, epoch = epoch, requestId = 0), + ) + val second = + OlpbFrameCodec.encode( + OlpbFrame( + frameType = OlpbFrameType.REQUEST_BYTES, + epoch = epoch, + requestId = 9, + payload = byteArrayOf(1, 2, 3), + ), + ) + val decoder = OlpbFrameDecoder(expectedEpoch = epoch) + + assertTrue(decoder.append(first.copyOfRange(0, 11)).isEmpty()) + val decoded = decoder.append(first.copyOfRange(11, first.size) + second) + + assertEquals(listOf(OlpbFrameType.PING, OlpbFrameType.REQUEST_BYTES), decoded.map { it.header.frameType }) + assertContentEquals(byteArrayOf(1, 2, 3), decoded[1].payload) + assertTrue(decoder.finish().isEmpty()) + } + + @Test + fun codecRejectsStaleEpochUnknownFlagsAndTruncation() { + val valid = + OlpbFrameCodec.encode( + OlpbFrame(frameType = OlpbFrameType.PING, epoch = epoch, requestId = 0), + ) + assertFailsWith { + OlpbFrameCodec.decode(valid, UUID.randomUUID()) + } + + val flagged = valid.copyOf().also { it[9] = 1 } + assertFailsWith { OlpbFrameCodec.decode(flagged, epoch) } + + val decoder = OlpbFrameDecoder(expectedEpoch = epoch) + decoder.append(valid.copyOf(valid.size - 1)) + assertFailsWith { decoder.finish() } + } + + @Test + fun frontendAssemblerAcceptsFragmentedCompleteMessages() { + val query = simpleQuery("SELECT 1") + val assembler = OlpbFrontendRequestAssembler() + assembler.append(query.copyOfRange(0, 3)) + assembler.append(query.copyOfRange(3, query.size)) + + assertContentEquals(query, assembler.finish()) + } + + @Test + fun lifecycleDistinguishesPreDispatchLossFromUnknownOutcome() { + val beforeDispatch = OlpbRequestLifecycle(epoch, 1) + beforeDispatch.beginReceiving() + assertEquals(OlpbTerminalResult.NOT_STARTED, beforeDispatch.lossResult()) + + val afterDispatch = OlpbRequestLifecycle(epoch, 2) + afterDispatch.beginReceiving() + afterDispatch.finishReceiving() + afterDispatch.beginNativeDispatch() + assertEquals(OlpbTerminalResult.OUTCOME_UNKNOWN, afterDispatch.lossResult()) + assertTrue(afterDispatch.requestCancellation()) + assertFalse(afterDispatch.requestCancellation()) + } + + @Test + fun nativePostgresOutputWitnessRequiresStrictlyMoreThanThresholdAcrossChunks() { + val counter = NativePostgresOutputWitnessCounter(thresholdBytes = 4) + + assertNull(counter.consume(byteArrayOf(1, 2, 3))) + assertNull(counter.consume(byteArrayOf(4))) + assertEquals(5, counter.consume(byteArrayOf(5))?.backendBytes) + assertNull(counter.consume(byteArrayOf(6))) + } + + @Test + fun nativePostgresOutputWitnessRejectsNegativeThreshold() { + assertFailsWith { + NativePostgresOutputWitnessCounter(thresholdBytes = -1) + } + } + + private fun simpleQuery(sql: String): ByteArray { + val body = sql.toByteArray(Charsets.UTF_8) + 0 + return ByteBuffer + .allocate(1 + 4 + body.size) + .order(ByteOrder.BIG_ENDIAN) + .apply { + put('Q'.code.toByte()) + putInt(body.size + 4) + put(body) + }.array() + } + +} diff --git a/spikes/android-native-broker/run-emulator.sh b/spikes/android-native-broker/run-emulator.sh new file mode 100755 index 00000000..35220863 --- /dev/null +++ b/spikes/android-native-broker/run-emulator.sh @@ -0,0 +1,425 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { + echo "android-native-broker: $*" >&2 + exit 1 +} + +need() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +root="$(git rev-parse --show-toplevel 2>/dev/null)" || fail "run inside the Oliphaunt checkout" +cd "$root" + +export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}" +export ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:-$ANDROID_HOME}" +adb="$ANDROID_HOME/platform-tools/adb" +emulator="$ANDROID_HOME/emulator/emulator" +avd="${OLIPHAUNT_ANDROID_BROKER_AVD:-Pixel_9_API_34_Google_API}" +package="dev.oliphaunt.androidbrokerspike" +strategy="${OLIPHAUNT_ANDROID_BROKER_STRATEGY:-full}" +timeout_seconds="${OLIPHAUNT_ANDROID_BROKER_TIMEOUT_SECONDS:-240}" +runtime_resources="${OLIPHAUNT_ANDROID_BROKER_RUNTIME_RESOURCES_DIR:-$root/target/android-native-broker-spike/runtime-resources}" +native_library="${OLIPHAUNT_ANDROID_BROKER_LIBOLIPHAUNT_SO:-$root/target/android-native-broker-spike/native/out/liboliphaunt.so}" +ndk_version="${OLIPHAUNT_ANDROID_BROKER_NDK_VERSION:-27.0.12077973}" +libcxx_shared="" +for ndk_host in darwin-arm64 darwin-x86_64 linux-x86_64; do + candidate="$ANDROID_HOME/ndk/$ndk_version/toolchains/llvm/prebuilt/$ndk_host/sysroot/usr/lib/aarch64-linux-android/libc++_shared.so" + if [ -f "$candidate" ]; then + libcxx_shared="$candidate" + break + fi +done +scratch="$root/target/android-native-broker-spike" +jni_root="$scratch/android-jni" +gradle_build_root="$scratch/gradle-build" +gradle_cxx_root="$scratch/gradle-cxx" +gradle_cache_root="$scratch/gradle-cache" +run_nonce="${OLIPHAUNT_ANDROID_BROKER_RUN_NONCE:-$(date -u +%Y%m%dT%H%M%SZ)-$$}" +run_dir="$scratch/runs/$run_nonce" + +case "$timeout_seconds" in + ''|*[!0-9]*) fail "OLIPHAUNT_ANDROID_BROKER_TIMEOUT_SECONDS must be a positive integer" ;; +esac +[ "$timeout_seconds" -gt 0 ] || fail "timeout must be positive" +[ -d "$runtime_resources/oliphaunt/runtime/files" ] || + fail "prepared runtime resources are missing: $runtime_resources" +[ -f "$runtime_resources/oliphaunt/template-pgdata/files/PG_VERSION" ] || + fail "prepared template PGDATA is missing under $runtime_resources" +[ -f "$native_library" ] || fail "prepared Android liboliphaunt is missing: $native_library" +[ -n "$libcxx_shared" ] || fail "Android NDK libc++_shared.so is missing for NDK $ndk_version" +[ -x "$adb" ] || fail "adb is missing: $adb" +[ -x "$emulator" ] || fail "emulator is missing: $emulator" +need git +need python3 +need shasum + +mkdir -p "$run_dir" "$jni_root/jniLibs/arm64-v8a" + +# Bind retained evidence to the exact dirty-tree inputs used for this spike. +# The APK/native-library hashes below identify outputs; this manifest identifies +# the source bytes that produced the Android host and exercised native ABI. +source_scope=( + spikes/android-native-broker + src/sdks/kotlin/build.gradle.kts + src/sdks/kotlin/gradle/libs.versions.toml + src/sdks/kotlin/settings.gradle.kts + src/sdks/kotlin/oliphaunt/src + src/runtimes/liboliphaunt/native/src/liboliphaunt_native.c +) +git rev-parse HEAD >"$run_dir/source-head.txt" +git status --short --untracked-files=all -- "${source_scope[@]}" \ + >"$run_dir/source-status.txt" +git ls-files --cached --others --exclude-standard -- "${source_scope[@]}" \ + | LC_ALL=C sort -u \ + | while IFS= read -r source_path; do + [ -f "$source_path" ] && shasum -a 256 "$source_path" + done \ + >"$run_dir/source-files.sha256" + +native_out_dir="$(cd "$(dirname "$native_library")" && pwd)" +[ "$(basename "$native_out_dir")" = out ] || + fail "liboliphaunt must be the canonical out/liboliphaunt.so artifact" +native_work_root="$(dirname "$native_out_dir")" +ANDROID_NDK_HOME="$ANDROID_HOME/ndk/$ndk_version" \ + ANDROID_NDK_ROOT="$ANDROID_HOME/ndk/$ndk_version" \ + OLIPHAUNT_ANDROID_WORK_ROOT="$native_work_root" \ + src/runtimes/liboliphaunt/native/bin/build-postgres18-android-arm64.sh \ + --check-current \ + | tee "$run_dir/native-current.log" + +install -m 0644 "$native_library" "$jni_root/jniLibs/arm64-v8a/liboliphaunt.so" +install -m 0644 "$libcxx_shared" "$jni_root/jniLibs/arm64-v8a/libc++_shared.so" +shasum -a 256 "$native_library" "$libcxx_shared" >"$run_dir/native-libraries.sha256" + +echo "==> Build Android broker spike" +src/sdks/kotlin/gradlew -p src/sdks/kotlin \ + :android-native-broker-spike:assembleDebug \ + -PoliphauntRuntimeResourcesDir="$runtime_resources" \ + -PoliphauntAndroidJniLibsDir="$jni_root" \ + -PoliphauntAndroidAbiFilters=arm64-v8a \ + -PoliphauntMobileStaticModules= \ + -PoliphauntBuildRoot="$gradle_build_root" \ + -PoliphauntCxxBuildRoot="$gradle_cxx_root" \ + --project-cache-dir "$gradle_cache_root" \ + --no-configuration-cache \ + | tee "$run_dir/gradle-build.log" + +apk="$gradle_build_root/android-native-broker-spike/outputs/apk/debug/android-native-broker-spike-debug.apk" +[ -f "$apk" ] || fail "Gradle did not produce the expected APK: $apk" +shasum -a 256 "$apk" >"$run_dir/apk.sha256" + +owned_emulator=0 +serial="${ANDROID_SERIAL:-}" +cleanup() { + status=$? + if [ "$owned_emulator" -eq 1 ] && [ -n "$serial" ]; then + "$adb" -s "$serial" emu kill >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +"$adb" start-server >/dev/null +if [ -z "$serial" ]; then + serial="$($adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" +fi +if [ -z "$serial" ]; then + "$emulator" -list-avds | grep -Fxq "$avd" || fail "Android AVD is unavailable: $avd" + echo "==> Start $avd" + "$emulator" \ + -avd "$avd" \ + -no-window \ + -no-audio \ + -no-boot-anim \ + -no-snapshot-load \ + -no-snapshot-save \ + -no-metrics \ + -gpu swiftshader_indirect \ + >"$run_dir/emulator.log" 2>&1 & + owned_emulator=1 + deadline=$((SECONDS + timeout_seconds)) + while [ "$SECONDS" -lt "$deadline" ]; do + serial="$($adb devices | awk 'NR > 1 && $2 == "device" && $1 ~ /^emulator-/ { print $1; exit }')" + [ -n "$serial" ] && break + sleep 2 + done + [ -n "$serial" ] || fail "emulator did not appear in adb within ${timeout_seconds}s" +fi + +deadline=$((SECONDS + timeout_seconds)) +while [ "$SECONDS" -lt "$deadline" ]; do + [ "$($adb -s "$serial" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r')" = 1 ] && break + sleep 2 +done +[ "$($adb -s "$serial" shell getprop sys.boot_completed | tr -d '\r')" = 1 ] || + fail "Android target did not finish booting" + +api="$($adb -s "$serial" shell getprop ro.build.version.sdk | tr -d '\r')" +abi="$($adb -s "$serial" shell getprop ro.product.cpu.abi | tr -d '\r')" +qemu="$($adb -s "$serial" shell getprop ro.boot.qemu | tr -d '\r')" +[ "$api" = 34 ] || fail "expected API 34, got $api" +[ "$abi" = arm64-v8a ] || fail "expected arm64-v8a, got $abi" +[ "$qemu" = 1 ] || fail "target is not an Android emulator" +printf 'serial=%s\napi=%s\nabi=%s\navd=%s\n' "$serial" "$api" "$abi" "$avd" >"$run_dir/device.txt" + +echo "==> Install and run strategy=$strategy nonce=$run_nonce" +"$adb" -s "$serial" install -r -t "$apk" | tee "$run_dir/install.log" +"$adb" -s "$serial" shell am force-stop "$package" >/dev/null 2>&1 || true +"$adb" -s "$serial" shell pm clear "$package" | tee "$run_dir/pm-clear.log" +"$adb" -s "$serial" logcat -c +"$adb" -s "$serial" shell am start -W \ + -n "$package/.MainActivity" \ + --es runNonce "$run_nonce" \ + --es strategy "$strategy" \ + | tee "$run_dir/launch.log" + +report="$run_dir/android-broker-report.json" +deadline=$((SECONDS + timeout_seconds)) +while [ "$SECONDS" -lt "$deadline" ]; do + if "$adb" -s "$serial" exec-out run-as "$package" \ + cat files/android-broker-report.json >"$report.candidate" 2>/dev/null; then + if [ -s "$report.candidate" ] && python3 - "$report.candidate" "$run_nonce" 2>/dev/null <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + report = json.load(source) +if report.get("runNonce") != sys.argv[2]: + raise SystemExit(1) +PY + then + mv "$report.candidate" "$report" + break + fi + fi + sleep 1 +done +[ -s "$report" ] || fail "fixture did not publish a report within ${timeout_seconds}s" + +"$adb" -s "$serial" shell ps -A -o USER,PID,PPID,NAME,ARGS >"$run_dir/processes.txt" 2>&1 || true +"$adb" -s "$serial" shell pidof "$package" >"$run_dir/host-pid.txt" 2>&1 || true +"$adb" -s "$serial" shell pidof "$package:broker" >"$run_dir/broker-pid.txt" 2>&1 || true +"$adb" -s "$serial" shell dumpsys activity services "$package" >"$run_dir/services.txt" 2>&1 || true +"$adb" -s "$serial" shell dumpsys meminfo "$package:broker" >"$run_dir/broker-meminfo.txt" 2>&1 || true +"$adb" -s "$serial" logcat -d -v epoch >"$run_dir/logcat.txt" 2>&1 || true +"$adb" -s "$serial" logcat -b crash -d -v epoch >"$run_dir/crash-logcat.txt" 2>&1 || true +"$adb" -s "$serial" shell dumpsys dropbox --print SYSTEM_TOMBSTONE >"$run_dir/dropbox-tombstones.txt" 2>&1 || true + +python3 - "$report" "$strategy" "$run_dir/processes.txt" "$run_dir/crash-logcat.txt" <<'PY' +import json +import re +import sys + +report_path, strategy, process_path, crash_path = sys.argv[1:] +with open(report_path, encoding="utf-8") as source: + report = json.load(source) + +def require(condition, message): + if not condition: + raise SystemExit(f"report validation failed: {message}") + +require(report.get("status") == "PASS", report.get("error", "status is not PASS")) +require(report.get("strategy") == strategy, "strategy mismatch") +checks = set(report.get("checks", [])) +required = { + "separateProcess", + "healthySql", + "outOfBandCancel", + "executorDeadlockFailStop", + "nativePgSleepFailStop", + "binderDeath", + "freshPidAndEpoch", + "outcomeUnknownNoReplay", + "boundedSlowReader8MiB", + "boundedSlowReader32MiB", + "persistentRecovery", +} +require(required <= checks, f"missing checks: {sorted(required - checks)}") +host_pid = int(report["hostPid"]) +worker_pids = [int(value) for value in report.get("workerPids", [])] +epochs = report.get("workerEpochs", []) +require(len(worker_pids) >= 3 and len(set(worker_pids)) == len(worker_pids), "worker PIDs are not fresh") +require(all(pid != host_pid for pid in worker_pids), "worker reused host PID") +require(len(epochs) == len(worker_pids) and len(set(epochs)) == len(epochs), "worker epochs are not fresh") +require(report.get("persistentMarkerSurvived") is True, "persistent marker did not survive") +ambiguous_execution_count = report.get("ambiguousExecutionCount") +require(ambiguous_execution_count == 1, + "ambiguous counter did not record exactly one execution") +require(report.get("replayCount") == ambiguous_execution_count - 1, + "replayCount was not derived from the ambiguous execution counter") +require(report.get("replayCount") == 0, "a faulted request was replayed") +faults = report.get("faultEvidence", []) +require([item.get("label") for item in faults] == [ + "executorDeadlock", "nativePgSleep", "afterCommitBeforeCompleted" +], "fault evidence labels are incomplete or out of order") +for index, item in enumerate(faults): + require(item.get("initialWorkerPid") == worker_pids[index], "fault initial PID mismatch") + require(item.get("recoveredWorkerPid") == worker_pids[index + 1], "fault recovered PID mismatch") + require(item.get("initialEpoch") == epochs[index], "fault initial epoch mismatch") + require(item.get("recoveredEpoch") == epochs[index + 1], "fault recovered epoch mismatch") + require(item.get("terminal") == "outcomeUnknown", "fault terminal is not outcomeUnknown") +require([item.get("nativeDispatchObserved") for item in faults] == [False, True, False], + "native-dispatch evidence does not match executor/native/after-commit lanes") +require([item.get("nativePostgresOutputWitnessObserved") for item in faults] == + [False, True, False], + "native PostgreSQL-output witness must be present only for the native pg_sleep lane") +native_output_witness = faults[1] +require(native_output_witness.get("nativePostgresOutputWitnessRequestId") == + native_output_witness.get("requestId"), + "native PostgreSQL-output witness request does not match the faulted request") +require(native_output_witness.get("nativePostgresOutputWitnessBackendBytes", 0) > + 4 * 1024 * 1024, + "native PostgreSQL-output witness did not exceed 4 MiB of backend bytes") +require(native_output_witness.get("nativePostgresOutputWitnessElapsedRealtimeNanos", 0) > 0, + "native PostgreSQL-output witness monotonic timestamp is missing") +require(native_output_witness.get("nativePostgresOutputWatchdogDelayMilliseconds") == 2000, + "native PostgreSQL-output watchdog delay is not two seconds") +for item in (faults[0], faults[2]): + require("nativePostgresOutputWitnessRequestId" not in item and + "nativePostgresOutputWitnessBackendBytes" not in item and + "nativePostgresOutputWitnessElapsedRealtimeNanos" not in item and + "nativePostgresOutputWatchdogDelayMilliseconds" not in item, + "non-native fault unexpectedly published a native PostgreSQL-output marker") +require(all(item.get("binderDeathObserved") is True for item in faults), + "Binder death was not observed for every fail-stop") +slow8 = report.get("slowReader8MiB", {}) +slow32 = report.get("slowReader32MiB", {}) +require(slow8.get("responseBytes", 0) >= 8 * 1024 * 1024, "8 MiB stream is undersized") +require(slow32.get("responseBytes", 0) >= 32 * 1024 * 1024, "32 MiB stream is undersized") +require(slow8.get("responseChunks", 0) > 1 and slow32.get("responseChunks", 0) > 1, + "slow-reader streams were not chunked") +require(slow8.get("sampleCount", 0) > 0 and slow32.get("sampleCount", 0) > 0, + "slow-reader memory was not sampled") +maximum_encoded_frame_bytes = 40 + 256 * 1024 +for label, expected_bytes, result in ( + ("8 MiB", 8 * 1024 * 1024, slow8), + ("32 MiB", 32 * 1024 * 1024, slow32), +): + pss_span = result["maximumPssBytes"] - result["minimumPssBytes"] + rss_span = result["maximumRssBytes"] - result["minimumRssBytes"] + require(pss_span >= 0 and result.get("pssSpanBytes") == pss_span, + f"{label} PSS span is missing or inconsistent") + require(rss_span >= 0 and result.get("rssSpanBytes") == rss_span, + f"{label} RSS span is missing or inconsistent") + require(result.get("requestedSocketSendBufferBytes") == 512 * 1024, + f"{label} requested socket send buffer is not 512 KiB") + require(result.get("readReleaseMode") == "hostControlledGate", + f"{label} did not use the host-controlled read gate") + require(result.get("readGateReleasedAfterSecondSample") is True, + f"{label} read gate was not released after the second sample") + read_gate_created = result.get("readGateCreatedElapsedRealtimeNanos", 0) + read_gate_released = result.get("readGateReleasedElapsedRealtimeNanos", 0) + require(read_gate_created > 0 and + read_gate_released >= result.get("secondSampleElapsedRealtimeNanos", 0), + f"{label} read-gate timestamps do not cover the second sample") + require(result.get("readGateHeldMilliseconds") == + (read_gate_released - read_gate_created) // 1000 // 1000, + f"{label} read-gate duration is inconsistent") + require(result.get("readGateHeldMilliseconds", 0) >= 300, + f"{label} read gate was not held for the required stall") + require(result.get("stableStallSearchTimeoutMilliseconds") == 10000, + f"{label} stable-stall search timeout is not 10 seconds") + require(result.get("stableStallPollIntervalMilliseconds") == 10, + f"{label} stable-stall poll interval is not 10 ms") + require(result.get("transientStallCandidatesRejected", -1) >= 0, + f"{label} transient stall rejection count is missing") + require(result.get("slowReaderDrainTimeoutMilliseconds") == 30000, + f"{label} slow-reader drain timeout is not 30 seconds") + require(result.get("requiredStallMilliseconds") == 300, + f"{label} did not require a 300 ms sustained stall") + require(result.get("observedSameWriteStallMilliseconds", 0) >= 300, + f"{label} did not observe a 300 ms same-write stall") + require(result.get("activeWriteAgeAtSecondSampleMilliseconds", 0) >= 300, + f"{label} active write was not blocked for 300 ms") + require(result.get("socketNonBlockingProbeSucceeded") is True, + f"{label} socket blocking-mode probe failed") + require(result.get("socketNonBlocking") is False, + f"{label} broker socket was nonblocking") + require(result.get("firstSocketPollSucceeded") is True and + result.get("secondSocketPollSucceeded") is True, + f"{label} POLLOUT probe failed") + require(result.get("firstSocketWritableNow") is False and + result.get("secondSocketWritableNow") is False, + f"{label} socket was writable during the no-read stall") + require(result.get("firstSocketWriteInProgress") is True and + result.get("secondSocketWriteInProgress") is True, + f"{label} synchronous write was not active across both stall samples") + observed_stall_nanos = (result["secondSampleElapsedRealtimeNanos"] - + result["firstSampleElapsedRealtimeNanos"]) + active_write_age_nanos = (result["secondSampleElapsedRealtimeNanos"] - + result["activeWriteStartedElapsedRealtimeNanos"]) + require(observed_stall_nanos >= 300 * 1000 * 1000, + f"{label} raw sample timestamps do not span 300 ms") + require(active_write_age_nanos >= 300 * 1000 * 1000, + f"{label} raw write age is less than 300 ms") + require(result.get("observedSameWriteStallMilliseconds") == + observed_stall_nanos // 1000 // 1000, + f"{label} reported stall duration is inconsistent") + require(result.get("activeWriteAgeAtSecondSampleMilliseconds") == + active_write_age_nanos // 1000 // 1000, + f"{label} reported active-write age is inconsistent") + require(result.get("activeWriteFrameType") == "RESPONSE_BYTES", + f"{label} blocked write was not response data") + require(result.get("activeWriteRequestId", 0) > 0, + f"{label} blocked write request ID is missing") + require(result.get("firstActiveWriteSequence") == + result.get("secondActiveWriteSequence"), + f"{label} socket writer advanced during the stall") + require(result.get("firstWritesCompleted") == result.get("secondWritesCompleted"), + f"{label} completed-write count advanced during the stall") + require(result.get("firstCompletedEncodedBytes") == + result.get("secondCompletedEncodedBytes"), + f"{label} completed socket bytes advanced during the stall") + completed_delta = (result["firstCompletedEncodedBytes"] - + result["baselineCompletedEncodedBytes"]) + require(completed_delta >= 0 and + result.get("completedEncodedDeltaBeforeRead") == completed_delta, + f"{label} pre-read completed-byte delta is inconsistent") + accepted_bound = completed_delta + result["activeWriteEncodedBytes"] + require(result.get("acceptedWireBytesUpperBound") == accepted_bound, + f"{label} accepted-wire upper bound is inconsistent") + require(result.get("maximumEncodedFrameBytes") == maximum_encoded_frame_bytes, + f"{label} maximum encoded frame size is inconsistent") + require(accepted_bound + maximum_encoded_frame_bytes < expected_bytes, + f"{label} socket accepted nearly the full response before reads began") + require(result.get("afterDrainWritesCompleted", 0) >= + result.get("firstActiveWriteSequence", 1), + f"{label} blocked socket write did not complete after reads resumed") + require(result.get("afterDrainCompletedEncodedBytes", 0) - + result.get("baselineCompletedEncodedBytes", 0) >= result["responseBytes"], + f"{label} post-drain socket-byte count is smaller than the response") +accepted_bound_delta = abs( + slow32["acceptedWireBytesUpperBound"] - slow8["acceptedWireBytesUpperBound"] +) +require(report.get("acceptedWireBoundDeltaBytes") == accepted_bound_delta, + "large-vs-small accepted-wire bound delta is inconsistent") +require(report.get("maximumAcceptedWireBoundDeltaBytes") == maximum_encoded_frame_bytes, + "accepted-wire bound delta limit is not one maximum frame") +require(accepted_bound_delta <= maximum_encoded_frame_bytes, + "32 MiB and 8 MiB pre-read socket bounds differ by more than one frame") +with open(process_path, encoding="utf-8", errors="replace") as source: + processes = source.read() +require(any(line.split()[1:2] == [str(host_pid)] for line in processes.splitlines()), + "reported host PID is not live after report publication") +for stale_pid in worker_pids[:-1]: + require(not any(line.split()[1:2] == [str(stale_pid)] for line in processes.splitlines()), + f"stale worker PID {stale_pid} is still live") +with open(crash_path, encoding="utf-8", errors="replace") as source: + crashes = source.read() +for stale_pid in worker_pids[:-1]: + require(re.search(rf"Fatal signal 6 .* pid {stale_pid} ", crashes) is not None, + f"worker PID {stale_pid} has no retained SIGABRT record") +print(json.dumps({ + "status": "PASS", + "hostPid": host_pid, + "workerPids": worker_pids, + "workerEpochs": epochs, + "checks": sorted(checks), +}, sort_keys=True)) +PY + +echo "Android broker experiment PASS: $report" +echo "Evidence directory: $run_dir" diff --git a/spikes/ios-native-broker/.gitignore b/spikes/ios-native-broker/.gitignore new file mode 100644 index 00000000..673b74ec --- /dev/null +++ b/spikes/ios-native-broker/.gitignore @@ -0,0 +1 @@ +Generated/ diff --git a/spikes/ios-native-broker/BrokerAppExtension/BrokerAppExtension.swift b/spikes/ios-native-broker/BrokerAppExtension/BrokerAppExtension.swift new file mode 100644 index 00000000..b62d1707 --- /dev/null +++ b/spikes/ios-native-broker/BrokerAppExtension/BrokerAppExtension.swift @@ -0,0 +1,700 @@ +import Darwin +import Dispatch +import ExtensionFoundation +import Foundation +import OliphauntBrokerExtension +import OliphauntBrokerProtocol +import OliphauntBrokerXPC +import XPC +import os + +@main +struct BrokerAppExtension: AppExtension { + @AppExtensionPoint.Bind + var extensionPoint: AppExtensionPoint { + AppExtensionPoint.Identifier( + host: "dev.oliphaunt.brokerspike", + name: "OliphauntBroker" + ) + } + + init() {} + + var configuration: ConnectionHandler { + ConnectionHandler { request in + let sessionID = UUID() + return request.accept( + incomingMessageHandler: { message in + BrokerExtensionServer.shared.handle(message, sessionID: sessionID) + }, + cancellationHandler: { _ in + BrokerExtensionServer.shared.cancelSession(sessionID) + } + ) + } + } +} + +/// Production control-plane glue for the simulator fixture. PostgreSQL bytes +/// stay on BrokerSocketWorker's owned socket; XPC carries primitives only. +private final class BrokerExtensionServer: @unchecked Sendable { + private struct PendingHello { + var token: UUID + var sessionID: UUID + var cancelled: Bool + } + + private struct ActiveChannel { + var token: UUID + var sessionID: UUID + var worker: BrokerSocketWorker + var task: Task? + } + + private struct CheckpointMemoryEvidence: Sendable { + var epoch: BrokerEpoch + var sequence: UInt64 + var startedAtUptimeNanoseconds: UInt64 + var sampledAtUptimeNanoseconds: UInt64 + var completedAtUptimeNanoseconds: UInt64 + var memory: BrokerProcessMemorySnapshot + } + + static let shared = BrokerExtensionServer() + + private static let liboliphauntVersion = "0.1.1" + private static let startupConfigurationDigest = + "ios-native-broker-spike-v2-restricted-role" + private static let selectedPostgresExtensions = ["pg_trgm", "vector"] + + // These strings are also public constants in IOSBrokerXPC. Keep the + // extension independent of any host-only diagnostics model. + private enum DiagnosticsKey { + static let state = "state" + static let manifestDigest = "manifestDigest" + static let activeRequestID = "activeRequestID" + static let nativeDispatchStarted = "nativeDispatchStarted" + static let transactionStatus = "transactionStatus" + static let capabilities = "capabilities" + static let currentPhysFootprintBytes = "currentPhysFootprintBytes" + static let currentResidentBytes = "currentResidentBytes" + static let availableMemoryBytes = "availableMemoryBytes" + static let checkpointInProgress = "checkpointInProgress" + static let checkpointMemorySampleSequence = "checkpointMemorySampleSequence" + static let checkpointMemorySampleStartedAtUptimeNanoseconds = + "checkpointMemorySampleStartedAtUptimeNanoseconds" + static let checkpointMemorySampledAtUptimeNanoseconds = + "checkpointMemorySampledAtUptimeNanoseconds" + static let checkpointMemorySampleCompletedAtUptimeNanoseconds = + "checkpointMemorySampleCompletedAtUptimeNanoseconds" + static let checkpointMemorySamplePhysFootprintBytes = + "checkpointMemorySamplePhysFootprintBytes" + static let checkpointMemorySampleResidentBytes = + "checkpointMemorySampleResidentBytes" + static let checkpointMemorySampleAvailableMemoryBytes = + "checkpointMemorySampleAvailableMemoryBytes" + static let storageProtectionEvidenceJSON = "storageProtectionEvidenceJSON" + static let extensionEntryPreOpenPhysFootprintBytes = + "extensionEntryPreOpenPhysFootprintBytes" + static let extensionEntryPreOpenResidentBytes = + "extensionEntryPreOpenResidentBytes" + static let openedIdlePhysFootprintBytes = "openedIdlePhysFootprintBytes" + static let openedIdleResidentBytes = "openedIdleResidentBytes" + } + + private let lock = NSLock() + private var core: WorkerCore? + private var pendingHello: PendingHello? + private var activeChannel: ActiveChannel? + private var extensionEntryPreOpenMemory: BrokerProcessMemorySnapshot? + private var openedIdleMemory: BrokerProcessMemorySnapshot? + private var checkpointInProgress = false + private var checkpointMemorySampleSequence: UInt64 = 0 + private var checkpointMemoryEvidence: CheckpointMemoryEvidence? + private var storageProtectionEvidenceJSON: String? + + func handle(_ message: XPCDictionary, sessionID: UUID) -> XPCDictionary? { + do { + switch try IOSBrokerXPC.messageKind(in: message) { + case .hello: + return try acceptHello(message, sessionID: sessionID) + case .cancel: + return try cancel(message) + case .checkpoint: + return try checkpoint(message) + case .prepareForBackground: + return try prepareForBackground(message) + case .resumeFromBackground: + return try resumeFromBackground(message) + case .detach: + return try detach(message, sessionID: sessionID) + case .diagnostics: + return try diagnostics(message) + case .injectFault: + return try injectFault(message) + case .attachDataChannel: + throw BrokerError.protocolViolation( + "v1 attaches its data-channel descriptor in Hello" + ) + case .ready, .rejected, .cancelObserved: + throw BrokerError.protocolViolation( + "host sent an extension-only control message" + ) + } + } catch { + return rejection(error) + } + } + + /// The XPC session interruption path never waits for WorkerCore's actor. + /// It closes the socket immediately, sends native cancellation directly, + /// then marks the epoch interrupted in detached cleanup work. + func cancelSession(_ sessionID: UUID) { + let interruption: (WorkerCore, BrokerSocketWorker?)? = lock.withExtensionLock { + if pendingHello?.sessionID == sessionID { + pendingHello?.cancelled = true + } + guard let core else { return nil } + if let channel = activeChannel, channel.sessionID == sessionID { + return (core, channel.worker) + } + if pendingHello?.sessionID == sessionID { + return (core, nil) + } + return nil + } + guard let (core, worker) = interruption else { return } + worker?.stop() + let activeRequest = core.cancellationController.activeRequest + Task.detached(priority: .userInitiated) { + if let activeRequest { + _ = try? await core.cancellationController.requestCancellation( + epoch: activeRequest.epoch, + requestID: activeRequest.requestID + ) + } + await core.interruptCurrentEpoch() + } + } + + private func acceptHello( + _ message: XPCDictionary, + sessionID: UUID + ) throws -> XPCDictionary { + let decoded = try IOSBrokerXPC.decodeHello(message) + let token = UUID() + try reserveHello(token: token, sessionID: sessionID) + + var didStartCore = false + do { + let core = try workerCore() + // liboliphaunt is directly linked, so dyld has already mapped it at + // extension entry. This is a pre-open baseline, not a pre-load one. + let extensionEntryPreOpen = BrokerProcessMemorySnapshot.current() + lock.withExtensionLock { + extensionEntryPreOpenMemory = extensionEntryPreOpen + storageProtectionEvidenceJSON = nil + } + + let ready = try waitForAsync { + try await core.start(hello: decoded.hello) + } + didStartCore = true + let openedIdle = BrokerProcessMemorySnapshot.current() + lock.withExtensionLock { + openedIdleMemory = openedIdle + } + + let descriptor = try decoded.dataChannel.takeDescriptor() + let worker = try BrokerSocketWorker( + ownedFileDescriptor: descriptor, + core: core, + epoch: ready.epoch, + protocolVersion: ready.selectedProtocolVersion + ) + try installChannel( + token: token, + sessionID: sessionID, + worker: worker + ) + + let task = Task.detached(priority: .userInitiated) { [weak self] in + _ = try? await worker.run() + self?.channelFinished(token: token) + } + lock.withExtensionLock { + guard activeChannel?.token == token else { + task.cancel() + worker.stop() + return + } + activeChannel?.task = task + } + return try IOSBrokerXPC.makeReady(ready) + } catch { + decoded.dataChannel.close() + clearPendingHello(token: token) + if didStartCore, let core = lock.withExtensionLock({ self.core }) { + Task.detached { + await core.interruptCurrentEpoch() + } + } + throw error + } + } + + private func cancel(_ message: XPCDictionary) throws -> XPCDictionary { + let envelope = try IOSBrokerXPC.decodeControl(message) + guard envelope.kind == .cancel, + let epoch = envelope.epoch, + let requestID = envelope.requestID + else { + throw BrokerError.protocolViolation("Cancel requires epoch and request ID") + } + let core = try currentCore() + + // This is deliberately the first call: no WorkerCore hop precedes the + // native cancellation witness. + let direct = try waitForAsync { + try await core.cancellationController.requestCancellation( + epoch: epoch, + requestID: requestID + ) + } + switch direct { + case .signalSent, .alreadyRequested: + // The native signal is already in flight. Record the lifecycle + // transition without delaying this control-plane acknowledgement. + Task.detached(priority: .userInitiated) { + _ = try? await core.cancelRequest( + epoch: epoch, + requestID: requestID + ) + } + return IOSBrokerXPC.makeAcknowledgement(.cancel) + case .notRunning: + let disposition = try waitForAsync { + try await core.cancelRequest(epoch: epoch, requestID: requestID) + } + switch disposition { + case .canceledBeforeNativeDispatch: + return IOSBrokerXPC.makeAcknowledgement(.cancelObserved) + case .notCurrent, .nativeSignal: + // Cancellation is idempotent. Backend observation, when there + // is a running request, is also reported on the data plane. + return IOSBrokerXPC.makeAcknowledgement(.cancel) + } + } + } + + private func checkpoint(_ message: XPCDictionary) throws -> XPCDictionary { + let (core, _, expectedEpoch) = try controlCore(message, expected: .checkpoint) + let checkpointStart = lock.withExtensionLock { + () -> (sequence: UInt64, uptimeNanoseconds: UInt64) in + checkpointMemorySampleSequence += 1 + checkpointInProgress = true + return ( + checkpointMemorySampleSequence, + DispatchTime.now().uptimeNanoseconds + ) + } + let checkpointMemory = BrokerProcessMemorySnapshot.current() + let checkpointSampledAt = DispatchTime.now().uptimeNanoseconds + defer { lock.withExtensionLock { checkpointInProgress = false } } + try waitForAsync { + try await core.checkpoint(expectedEpoch: expectedEpoch) + } + let checkpointCompletedAt = DispatchTime.now().uptimeNanoseconds + lock.withExtensionLock { + checkpointMemoryEvidence = CheckpointMemoryEvidence( + epoch: expectedEpoch, + sequence: checkpointStart.sequence, + startedAtUptimeNanoseconds: checkpointStart.uptimeNanoseconds, + sampledAtUptimeNanoseconds: checkpointSampledAt, + completedAtUptimeNanoseconds: checkpointCompletedAt, + memory: checkpointMemory + ) + } + return IOSBrokerXPC.makeAcknowledgement(.checkpoint) + } + + private func prepareForBackground( + _ message: XPCDictionary + ) throws -> XPCDictionary { + let (core, envelope, expectedEpoch) = try controlCore( + message, + expected: .prepareForBackground + ) + guard let nanoseconds = envelope.deadlineUnixNanoseconds else { + throw BrokerError.protocolViolation( + "PrepareForBackground requires an absolute deadline" + ) + } + let deadline = Date( + timeIntervalSince1970: TimeInterval(nanoseconds) / 1_000_000_000 + ) + let result = try waitForAsync { + try await core.prepareForBackground( + expectedEpoch: expectedEpoch, + deadline: deadline + ) + } + if let diagnostics = try? waitForAsync({ + try await core.diagnostics(expectedEpoch: expectedEpoch) + }), diagnostics.state == .quiescing, + let encoded = try? encodeProtectionEvidence(rootURL: diagnostics.rootURL) + { + lock.withExtensionLock { + storageProtectionEvidenceJSON = encoded + } + } + var reply = IOSBrokerXPC.makeAcknowledgement(.prepareForBackground) + reply[IOSBrokerXPC.cancelledActiveWorkKey] = result.cancelledActiveWork + reply[IOSBrokerXPC.checkpointedKey] = result.checkpointed + return reply + } + + private func resumeFromBackground( + _ message: XPCDictionary + ) throws -> XPCDictionary { + let (core, _, expectedEpoch) = try controlCore( + message, + expected: .resumeFromBackground + ) + try waitForAsync { + try await core.resumeFromBackground(expectedEpoch: expectedEpoch) + } + return IOSBrokerXPC.makeAcknowledgement(.resumeFromBackground) + } + + private func detach( + _ message: XPCDictionary, + sessionID: UUID + ) throws -> XPCDictionary { + let (core, _, expectedEpoch) = try controlCore(message, expected: .detach) + let worker = lock.withExtensionLock { () -> BrokerSocketWorker? in + guard activeChannel?.sessionID == sessionID else { return nil } + return activeChannel?.worker + } + try waitForAsync { + try await core.detach(expectedEpoch: expectedEpoch) + } + worker?.stopGracefully() + return IOSBrokerXPC.makeAcknowledgement(.detach) + } + + private func diagnostics(_ message: XPCDictionary) throws -> XPCDictionary { + let currentMemory = BrokerProcessMemorySnapshot.current() + let (core, _, expectedEpoch) = try controlCore(message, expected: .diagnostics) + let value = try waitForAsync { + try await core.diagnostics(expectedEpoch: expectedEpoch) + } + let phaseState = lock.withExtensionLock { + ( + extensionEntryPreOpen: extensionEntryPreOpenMemory, + openedIdle: openedIdleMemory, + checkpointInProgress: checkpointInProgress, + checkpointMemoryEvidence: checkpointMemoryEvidence, + storageProtectionEvidenceJSON: storageProtectionEvidenceJSON + ) + } + + var reply = IOSBrokerXPC.makeAcknowledgement(.diagnostics) + reply[DiagnosticsKey.state] = stateName(value.state) + reply[BrokerControlKey.epoch] = value.epoch.description + reply[BrokerControlKey.extensionPID] = Int64(value.processID) + if let digest = value.manifestDigest { + reply[DiagnosticsKey.manifestDigest] = digest + } + if let requestID = value.activeRequestID { + reply[DiagnosticsKey.activeRequestID] = requestID.rawValue + } + reply[DiagnosticsKey.nativeDispatchStarted] = value.nativeDispatchStarted + reply[DiagnosticsKey.transactionStatus] = value.transactionStatus + reply[DiagnosticsKey.capabilities] = try encodedCapabilities(value.capabilities) + reply[DiagnosticsKey.currentPhysFootprintBytes] = currentMemory.physFootprintBytes + reply[DiagnosticsKey.currentResidentBytes] = currentMemory.residentBytes + reply[DiagnosticsKey.availableMemoryBytes] = currentMemory.availableMemoryBytes + reply[DiagnosticsKey.checkpointInProgress] = phaseState.checkpointInProgress + if let evidence = phaseState.checkpointMemoryEvidence, + evidence.epoch == value.epoch + { + reply[DiagnosticsKey.checkpointMemorySampleSequence] = evidence.sequence + reply[DiagnosticsKey.checkpointMemorySampleStartedAtUptimeNanoseconds] = + evidence.startedAtUptimeNanoseconds + reply[DiagnosticsKey.checkpointMemorySampledAtUptimeNanoseconds] = + evidence.sampledAtUptimeNanoseconds + reply[DiagnosticsKey.checkpointMemorySampleCompletedAtUptimeNanoseconds] = + evidence.completedAtUptimeNanoseconds + reply[DiagnosticsKey.checkpointMemorySamplePhysFootprintBytes] = + evidence.memory.physFootprintBytes + reply[DiagnosticsKey.checkpointMemorySampleResidentBytes] = + evidence.memory.residentBytes + reply[DiagnosticsKey.checkpointMemorySampleAvailableMemoryBytes] = + evidence.memory.availableMemoryBytes + } + if let protection = phaseState.storageProtectionEvidenceJSON { + reply[DiagnosticsKey.storageProtectionEvidenceJSON] = protection + } + if let before = phaseState.extensionEntryPreOpen { + reply[DiagnosticsKey.extensionEntryPreOpenPhysFootprintBytes] = + before.physFootprintBytes + reply[DiagnosticsKey.extensionEntryPreOpenResidentBytes] = before.residentBytes + } + if let opened = phaseState.openedIdle { + reply[DiagnosticsKey.openedIdlePhysFootprintBytes] = + opened.physFootprintBytes + reply[DiagnosticsKey.openedIdleResidentBytes] = opened.residentBytes + } + return reply + } + + private func injectFault(_ message: XPCDictionary) throws -> XPCDictionary { + let (core, envelope, expectedEpoch) = try controlCore( + message, + expected: .injectFault + ) + guard let fault = envelope.fault else { + throw BrokerError.protocolViolation("InjectFault requires a fault name") + } + #if DEBUG + try waitForAsync { + try await core.injectFault(fault, expectedEpoch: expectedEpoch) + } + return IOSBrokerXPC.makeAcknowledgement(.injectFault) + #else + _ = core + _ = fault + throw BrokerError.rejected( + .invalidRequest("fault injection is unavailable in release builds") + ) + #endif + } + + private func controlCore( + _ message: XPCDictionary, + expected: BrokerControlMessageKind + ) throws -> (WorkerCore, IOSBrokerControlEnvelope, BrokerEpoch) { + let envelope = try IOSBrokerXPC.decodeControl(message) + guard envelope.kind == expected, let requestedEpoch = envelope.epoch else { + throw BrokerError.protocolViolation( + "\(expected.rawValue) requires the current epoch" + ) + } + let core = try currentCore() + return (core, envelope, requestedEpoch) + } + + private func workerCore() throws -> WorkerCore { + if let existing = lock.withExtensionLock({ core }) { + return existing + } + let storage: BrokerExtensionStorage + do { + storage = try BrokerExtensionStorage.extensionPrivate() + } catch { + throw BrokerError.rejected(.rootOpen) + } + let configuration: BrokerWorkerConfiguration + do { + configuration = try BrokerWorkerConfiguration.nativeDirect( + storage: storage, + liboliphauntVersion: Self.liboliphauntVersion, + startupConfigurationDigest: Self.startupConfigurationDigest, + selectedPostgresExtensions: Self.selectedPostgresExtensions + ) + } catch { + throw BrokerError.brokerUnavailable + } + let created = WorkerCore(configuration: configuration) + return lock.withExtensionLock { + if let existing = core { return existing } + core = created + return created + } + } + + private func currentCore() throws -> WorkerCore { + guard let core = lock.withExtensionLock({ core }) else { + throw BrokerError.brokerUnavailable + } + return core + } + + private func reserveHello(token: UUID, sessionID: UUID) throws { + try lock.withExtensionLock { + guard pendingHello == nil, activeChannel == nil else { + throw BrokerError.rejected( + .invalidRequest("a broker data channel is already active") + ) + } + pendingHello = PendingHello( + token: token, + sessionID: sessionID, + cancelled: false + ) + } + } + + private func installChannel( + token: UUID, + sessionID: UUID, + worker: BrokerSocketWorker + ) throws { + try lock.withExtensionLock { + guard let pendingHello, + pendingHello.token == token, + pendingHello.sessionID == sessionID, + !pendingHello.cancelled, + activeChannel == nil + else { + throw BrokerError.workerInterrupted(epoch: nil) + } + self.pendingHello = nil + activeChannel = ActiveChannel( + token: token, + sessionID: sessionID, + worker: worker, + task: nil + ) + } + } + + private func clearPendingHello(token: UUID) { + lock.withExtensionLock { + guard pendingHello?.token == token else { return } + pendingHello = nil + } + } + + private func channelFinished(token: UUID) { + lock.withExtensionLock { + guard activeChannel?.token == token else { return } + activeChannel = nil + } + } + + private func rejection(_ error: Error) -> XPCDictionary { + let brokerError = IOSBrokerXPC.extensionBoundaryError(error) + if let encoded = try? IOSBrokerXPC.makeError(brokerError) { + return encoded + } + var fallback = XPCDictionary() + fallback[BrokerControlKey.message] = BrokerControlMessageKind.rejected.rawValue + fallback[BrokerControlKey.reason] = brokerError.description + return fallback + } +} + +private func encodeProtectionEvidence(rootURL: URL) throws -> String { + let storage = try BrokerExtensionStorage( + location: .extensionPrivate, + rootURL: rootURL + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(storage.recursiveProtectionEvidence()) + guard let value = String(data: data, encoding: .utf8) else { + throw BrokerError.protocolViolation("cannot encode storage-protection evidence") + } + return value +} + +private struct BrokerProcessMemorySnapshot: Sendable { + var physFootprintBytes: UInt64 + var residentBytes: UInt64 + var availableMemoryBytes: UInt64 + + /// Samples this extension process directly and does not enter WorkerCore. + static func current() -> BrokerProcessMemorySnapshot { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t( + MemoryLayout.size / MemoryLayout.size + ) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound( + to: integer_t.self, + capacity: Int(count) + ) { rebound in + task_info( + mach_task_self_, + task_flavor_t(TASK_VM_INFO), + rebound, + &count + ) + } + } + guard result == KERN_SUCCESS else { + return BrokerProcessMemorySnapshot( + physFootprintBytes: 0, + residentBytes: 0, + availableMemoryBytes: UInt64(os_proc_available_memory()) + ) + } + return BrokerProcessMemorySnapshot( + physFootprintBytes: UInt64(info.phys_footprint), + residentBytes: UInt64(info.resident_size), + availableMemoryBytes: UInt64(os_proc_available_memory()) + ) + } +} + +private final class BrokerBlockingResult: @unchecked Sendable { + private let lock = NSLock() + private var result: Result? + + func set(_ result: Result) { + lock.withExtensionLock { self.result = result } + } + + func take() -> Result { + lock.withExtensionLock { + precondition(result != nil) + return result! + } + } +} + +private func waitForAsync( + _ operation: @escaping @Sendable () async throws -> Value +) throws -> Value { + let semaphore = DispatchSemaphore(value: 0) + let result = BrokerBlockingResult() + Task.detached { + do { + result.set(.success(try await operation())) + } catch { + result.set(.failure(error)) + } + semaphore.signal() + } + semaphore.wait() + return try result.take().get() +} + +private func encodedCapabilities(_ value: BrokerCapabilities) throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + return String(decoding: try encoder.encode(value), as: UTF8.self) +} + +private func stateName(_ state: BrokerWorkerCoreState) -> String { + switch state { + case .created: "created" + case .starting: "starting" + case .ready: "ready" + case .quiescing: "quiescing" + case .interrupted: "interrupted" + case .detached: "detached" + case .failed: "failed" + } +} + +extension NSLock { + @discardableResult + fileprivate func withExtensionLock(_ body: () throws -> Result) rethrows -> Result { + lock() + defer { unlock() } + return try body() + } +} diff --git a/spikes/ios-native-broker/Host/BrokerPlatformProbe.swift b/spikes/ios-native-broker/Host/BrokerPlatformProbe.swift new file mode 100644 index 00000000..9230c0cb --- /dev/null +++ b/spikes/ios-native-broker/Host/BrokerPlatformProbe.swift @@ -0,0 +1,235 @@ +import Darwin +import ExtensionFoundation +import Foundation +import OliphauntBrokerProtocol +import XPC + +#if canImport(OliphauntIOSBroker) + import OliphauntIOSBroker +#endif + +enum BrokerPlatformProbe { + static func run( + retain: @MainActor @escaping (AppExtensionProcess, XPCSession) -> Void + ) async throws -> BrokerProbeResult { + let hostPID = getpid() + #if canImport(OliphauntIOSBroker) + let extensionPoint: AppExtensionPoint = .oliphauntBroker + #else + let extensionPoint: AppExtensionPoint = .oliphauntBrokerSpike + #endif + let monitor = try await AppExtensionPoint.Monitor(appExtensionPoint: extensionPoint) + guard + let identity = monitor.identities.first(where: { + $0.bundleIdentifier == BrokerFixtureBundleIdentifiers.extensionBundleIdentifier + }) + else { + throw BrokerProbeError.extensionMissing( + discovered: monitor.identities.map(\.bundleIdentifier) + ) + } + + let interruption = ProbeInterruptionFlag() + let configuration = AppExtensionProcess.Configuration( + appExtensionIdentity: identity, + onInterruption: { + interruption.markInterrupted() + } + ) + let process = try await AppExtensionProcess(configuration: configuration) + let session = try process.makeXPCSession() + session.setTargetQueue(DispatchQueue(label: "dev.oliphaunt.brokerspike.xpc")) + session.setCancellationHandler { error in + interruption.markInterrupted(reason: String(describing: error)) + } + session.setIncomingMessageHandler { (_: XPCDictionary) in nil } + try session.activate() + await retain(process, session) + + let pair = try ProbeSocketPair() + var hello = XPCDictionary() + hello[BrokerControlKey.message] = BrokerControlMessageKind.hello.rawValue + hello[BrokerControlKey.minimumProtocolVersion] = UInt64( + OliphauntBrokerProtocol.minimumVersion) + hello[BrokerControlKey.maximumProtocolVersion] = UInt64( + OliphauntBrokerProtocol.maximumVersion) + hello[BrokerControlKey.expectedABI] = UInt64(6) + hello[BrokerControlKey.rootID] = OliphauntBrokerProtocol.canonicalRootID + hello[BrokerControlKey.startupConfigurationDigest] = "simulator-probe-v1" + guard let boxedDescriptor = xpc_fd_create(pair.workerDescriptor) else { + throw BrokerProbeError.fileDescriptorBoxingFailed + } + hello[BrokerControlKey.dataChannel] = boxedDescriptor + + let reply = try await session.request(hello).dictionary + pair.closeWorkerOriginal() + guard + reply[BrokerControlKey.message, as: String.self] + == BrokerControlMessageKind.ready.rawValue, + let epochText = reply[BrokerControlKey.epoch, as: String.self], + let epochUUID = UUID(uuidString: epochText), + let workerPID = reply[BrokerControlKey.extensionPID, as: Int64.self] + else { + throw BrokerProbeError.invalidReady(String(describing: reply)) + } + let epoch = BrokerEpoch(epochUUID) + guard Int32(workerPID) != hostPID else { + throw BrokerProbeError.notProcessIsolated(pid: hostPID) + } + + let ping = try BrokerFrame( + frameType: .ping, + epoch: epoch, + requestID: 0 + ).encoded() + try await pair.host.writeFragmented(ping) + let pong = try await pair.host.readFrame(expectedEpoch: epoch) + guard pong.header.frameType == .pong else { + throw BrokerProbeError.unexpectedFrame(pong.header.frameType) + } + + let requestID = try BrokerRequestID(validating: 1) + let request = simpleQuery("SELECT 1") + try await pair.host.write( + try BrokerFrame( + frameType: .requestBegin, + epoch: epoch, + requestID: requestID.rawValue + ).encoded()) + for chunk in request.chunked(maximum: 3) { + try await pair.host.write( + try BrokerFrame( + frameType: .requestBytes, + epoch: epoch, + requestID: requestID.rawValue, + payload: chunk + ).encoded()) + } + try await pair.host.write( + try BrokerFrame( + frameType: .requestEnd, + epoch: epoch, + requestID: requestID.rawValue + ).encoded()) + + var echoed = Data() + while true { + let frame = try await pair.host.readFrame(expectedEpoch: epoch) + guard frame.header.requestID == requestID.rawValue else { + throw BrokerProbeError.unexpectedRequestID(frame.header.requestID) + } + switch frame.header.frameType { + case .responseBytes: + echoed.append(frame.payload) + case .completed: + guard echoed == request else { + throw BrokerProbeError.echoMismatch + } + try await pair.host.write( + try BrokerFrame( + frameType: .channelClose, + epoch: epoch, + requestID: 0 + ).encoded()) + return BrokerProbeResult( + hostPID: hostPID, + workerPID: Int32(workerPID), + epoch: epoch.description, + checks: [ + "extensionDiscovery", + "separatePID", + "xpcSession", + "fdTransfer", + "fragmentedFrame", + "boundedRequestAssembly", + ] + ) + default: + throw BrokerProbeError.unexpectedFrame(frame.header.frameType) + } + } + } + + private static func simpleQuery(_ sql: String) -> Data { + let sqlBytes = Data(sql.utf8) + let length = UInt32(sqlBytes.count + 5) + var result = Data([0x51]) + result.append(UInt8((length >> 24) & 0xff)) + result.append(UInt8((length >> 16) & 0xff)) + result.append(UInt8((length >> 8) & 0xff)) + result.append(UInt8(length & 0xff)) + result.append(sqlBytes) + result.append(0) + return result + } +} + +private final class ProbeInterruptionFlag: @unchecked Sendable { + private let lock = NSLock() + private(set) var reason: String? + + func markInterrupted(reason: String = "AppExtensionProcess interrupted") { + lock.withLock { + self.reason = reason + } + } +} + +extension XPCSession { + fileprivate func request(_ message: XPCDictionary) async throws -> ProbeXPCReply { + try await withCheckedThrowingContinuation { continuation in + send(message: message) { result in + switch result { + case .success(let dictionary): + continuation.resume(returning: ProbeXPCReply(dictionary)) + case .failure(let error): + continuation.resume( + throwing: BrokerProbeError.xpcRequestFailed(String(describing: error)) + ) + } + } + } + } +} + +private final class ProbeXPCReply: @unchecked Sendable { + let dictionary: XPCDictionary + + init(_ dictionary: XPCDictionary) { + self.dictionary = dictionary + } +} + +private enum BrokerProbeError: Error, CustomStringConvertible { + case extensionMissing(discovered: [String]) + case fileDescriptorBoxingFailed + case invalidReady(String) + case notProcessIsolated(pid: Int32) + case xpcRequestFailed(String) + case unexpectedFrame(BrokerFrameType) + case unexpectedRequestID(UInt64) + case echoMismatch + + var description: String { + switch self { + case .extensionMissing(let discovered): + "broker extension missing; discovered=\(discovered)" + case .fileDescriptorBoxingFailed: "xpc_fd_create failed" + case .invalidReady(let value): "invalid Ready reply: \(value)" + case .notProcessIsolated(let pid): "host and worker share PID \(pid)" + case .xpcRequestFailed(let reason): "XPC request failed: \(reason)" + case .unexpectedFrame(let type): "unexpected broker frame \(type)" + case .unexpectedRequestID(let id): "unexpected broker request ID \(id)" + case .echoMismatch: "extension data-channel echo mismatch" + } + } +} + +extension Data { + fileprivate func chunked(maximum: Int) -> [Data] { + guard !isEmpty else { return [] } + return stride(from: 0, to: count, by: maximum).map { offset in + subdata(in: offset.. Bool { + let notifications = NotificationCenter.default + notifications.addObserver( + self, + selector: #selector(didBecomeActive(_:)), + name: UIApplication.didBecomeActiveNotification, + object: nil + ) + notifications.addObserver( + self, + selector: #selector(willResignActive(_:)), + name: UIApplication.willResignActiveNotification, + object: nil + ) + notifications.addObserver( + self, + selector: #selector(didEnterBackground(_:)), + name: UIApplication.didEnterBackgroundNotification, + object: nil + ) + notifications.addObserver( + self, + selector: #selector(willEnterForeground(_:)), + name: UIApplication.willEnterForegroundNotification, + object: nil + ) + notifications.addObserver( + self, + selector: #selector(didReceiveMemoryWarning(_:)), + name: UIApplication.didReceiveMemoryWarningNotification, + object: nil + ) + publish(application.applicationState) + return true + } + + @objc private func didBecomeActive(_ notification: Notification) { + publish(.active) + } + + @objc private func willResignActive(_ notification: Notification) { + publish(.inactive) + } + + @objc private func didEnterBackground(_ notification: Notification) { + publish(.background) + } + + @objc private func willEnterForeground(_ notification: Notification) { + publish(.inactive) + } + + @objc private func didReceiveMemoryWarning(_ notification: Notification) { + events.send(.memoryWarning) + } + + private func publish(_ state: UIApplication.State) { + if managesDisplayIdleTimer { + UIApplication.shared.isIdleTimerDisabled = state == .active + } + switch state { + case .active: + events.send(.active) + case .inactive: + events.send(.inactive) + case .background: + events.send(.background) + @unknown default: + break + } + } +} + +@MainActor +@Observable +final class BrokerSpikeModel { + private(set) var status = "STARTING" + private var started = false + #if canImport(OliphauntIOSBroker) + private let lifecycleEvents = BrokerFixtureApplicationLifecycle.events + #endif + #if !canImport(OliphauntIOSBroker) + private var process: AppExtensionProcess? + private var session: XPCSession? + #endif + + func runIfNeeded() async { + guard !started else { return } + started = true + do { + #if canImport(OliphauntIOSBroker) + let mode = + ProcessInfo.processInfo.environment[ + "OLIPHAUNT_BROKER_FIXTURE_MODE" + ] ?? "default" + let result: BrokerProbeResult + switch mode { + case "default", "foreground", "semantic": + result = try await NativeBrokerFixture.run() + case "lifecycle": + result = try await DeviceLifecycleFixture.run(events: lifecycleEvents) + #if DEBUG + case "extendedFaults": + result = try await ExtendedFaultMatrix.run() + case "hang": + result = try await HangFaultMatrix.run() + #endif + case "handshakeNegatives": + result = try await HandshakeNegativeMatrix.run() + default: + throw BrokerFixtureModeFailure.unknown(mode) + } + #else + let result = try await BrokerPlatformProbe.run { process, session in + self.process = process + self.session = session + } + #endif + status = + "PASS\nhostPID=\(result.hostPID)\nworkerPID=\(result.workerPID)\nepoch=\(result.epoch)\nchecks=\(result.checks.joined(separator: ","))" + let reportData = try persist(result: result, error: nil) + if let reportJSON = String(data: reportData, encoding: .utf8) { + print("OLIPHAUNT_BROKER_SPIKE_JSON \(reportJSON)") + } + print("OLIPHAUNT_BROKER_SPIKE PASS \(result.logSummary)") + } catch { + status = "FAIL\n\(error)" + _ = try? persist(result: nil, error: String(describing: error)) + print("OLIPHAUNT_BROKER_SPIKE FAIL \(error)") + } + } + + @discardableResult + private func persist(result: BrokerProbeResult?, error: String?) throws -> Data { + let report = BrokerProbeReport(result: result, error: error) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(report) + let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + try data.write( + to: documents.appendingPathComponent("broker-spike-report.json"), + options: .atomic + ) + return data + } +} + +private enum BrokerFixtureModeFailure: Error, CustomStringConvertible { + case unknown(String) + + var description: String { + switch self { + case .unknown(let value): + "unknown OLIPHAUNT_BROKER_FIXTURE_MODE \(String(reflecting: value))" + } + } +} + +private struct BrokerProbeReport: Codable { + var result: BrokerProbeResult? + var error: String? +} + +struct BrokerProbeResult: Codable { + var hostPID: Int32 + var workerPID: Int32 + var epoch: String + var checks: [String] + var recoveredEpochs: [String] = [] + var diagnostics: [BrokerDiagnosticEvidence] = [] + var observations: [String: String] = [:] + + var logSummary: String { + "hostPID=\(hostPID) workerPID=\(workerPID) epoch=\(epoch) checks=\(checks.joined(separator: ","))" + } +} + +struct BrokerDiagnosticEvidence: Codable { + var phase: String + var managerState: String + var epoch: String? + var workerPID: Int32? + var logicalHandleCount: Int + var queuedOperationCount: Int + var activeRequestID: UInt64? + var launchCount: UInt64 + var interruptionCount: UInt64 + var admissionsPaused: Bool = false + var workerState: String? + var transactionStatus: String? + var manifestDigest: String? + var currentPhysFootprintBytes: UInt64? + var currentResidentBytes: UInt64? + var availableMemoryBytes: UInt64? + var nativeDispatchStarted: Bool = false + var checkpointInProgress: Bool = false + var storageProtectionEvidenceJSON: String? + var extensionEntryPreOpenPhysFootprintBytes: UInt64? + var extensionEntryPreOpenResidentBytes: UInt64? + var openedIdlePhysFootprintBytes: UInt64? + var openedIdleResidentBytes: UInt64? +} diff --git a/spikes/ios-native-broker/Host/DeviceLifecycleFixture.swift b/spikes/ios-native-broker/Host/DeviceLifecycleFixture.swift new file mode 100644 index 00000000..9fc031ab --- /dev/null +++ b/spikes/ios-native-broker/Host/DeviceLifecycleFixture.swift @@ -0,0 +1,1776 @@ +import Dispatch +import Foundation + +enum BrokerHostLifecycleEventKind: String, Codable, Hashable, Sendable { + case inactive + case background + case active + case memoryWarning +} + +struct BrokerHostLifecycleEvent: Codable, Equatable, Sendable { + var kind: BrokerHostLifecycleEventKind + var observedAtUnixNanoseconds: UInt64 + var observedAtUptimeNanoseconds: UInt64 +} + +private final class BrokerHostLifecycleEventStorage: @unchecked Sendable { + private let lock = NSLock() + private var values: [BrokerHostLifecycleEvent] = [] + private var lastApplicationStateKind: BrokerHostLifecycleEventKind? + + func append(_ event: BrokerHostLifecycleEvent) -> Bool { + lock.lock() + defer { lock.unlock() } + if event.kind != .memoryWarning { + guard event.kind != lastApplicationStateKind else { return false } + lastApplicationStateKind = event.kind + } + values.append(event) + return true + } + + func snapshot() -> [BrokerHostLifecycleEvent] { + lock.lock() + defer { lock.unlock() } + return values + } +} + +/// A process-lifecycle event channel owned by the host application. +/// +/// `BrokerSpikeAppDelegate` sends UIKit application-state and memory-warning +/// notifications into this value. The lifecycle fixture is its only stream consumer. Copies +/// share one continuation and one journal snapshot, so it is safe to retain one +/// copy in the model and pass another to `DeviceLifecycleFixture.run(events:)`. +struct BrokerHostLifecycleEventSource: Sendable { + fileprivate let stream: AsyncStream + private let continuation: AsyncStream.Continuation + private let storage: BrokerHostLifecycleEventStorage + + init() { + let pair = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(32) + ) + stream = pair.stream + continuation = pair.continuation + storage = BrokerHostLifecycleEventStorage() + } + + func send(_ kind: BrokerHostLifecycleEventKind) { + let event = BrokerHostLifecycleEvent( + kind: kind, + observedAtUnixNanoseconds: deviceLifecycleUnixNanoseconds(), + observedAtUptimeNanoseconds: DispatchTime.now().uptimeNanoseconds + ) + guard storage.append(event) else { return } + continuation.yield(event) + } + + func finish() { + continuation.finish() + } + + fileprivate func snapshot() -> [BrokerHostLifecycleEvent] { + storage.snapshot() + } +} + +#if canImport(OliphauntIOSBroker) + import Darwin + import Oliphaunt + import OliphauntBrokerProtocol + import OliphauntIOSBroker + + enum DeviceLifecycleFixture { + private static let startupConfigurationDigest = + "ios-native-broker-spike-v2-restricted-role" + private static let selectedExtensions = ["pg_trgm", "vector"] + private static let backgroundPreparationSeconds: TimeInterval = 8 + private static let filesystemTimestampToleranceNanoseconds: UInt64 = 2_000_000_000 + + static func run( + events: BrokerHostLifecycleEventSource + ) async throws -> BrokerProbeResult { + let rawEnvironment = DeviceLifecycleEnvironment.capture() + let journal = try DeviceLifecycleJournalWriter(environment: rawEnvironment) + var controlSession: IOSBrokerSession? + var database: OliphauntDatabase? + var checks = Set() + var observations: [String: String] = [:] + var diagnostics: [BrokerDiagnosticEvidence] = [] + var recoveredEpochs: [String] = [] + + do { + let environment = try rawEnvironment.validated() + #if targetEnvironment(simulator) + throw DeviceLifecycleFailure.assertion( + "device lifecycle qualification requires a physical iOS device" + ) + #elseif DEBUG + throw DeviceLifecycleFailure.assertion( + "device lifecycle qualification requires a Release build" + ) + #endif + + let hostPID = getpid() + let manager = IOSBrokerManager() + let brokerConfiguration = IOSBrokerConfiguration( + expectedABI: 6, + expectedRuntimeVersion: nil, + startupConfigurationDigest: startupConfigurationDigest, + maximumRequestBytes: OliphauntBrokerProtocol.defaultMaximumRequestBytes, + requestDeadline: .seconds(environment.orchestrationTimeoutSeconds * 3), + extensionBundleIdentifier: BrokerFixtureBundleIdentifiers + .extensionBundleIdentifier, + controlReplyTimeout: .seconds(20), + cancellationGracePeriod: .seconds(3) + ) + let databaseConfiguration = OliphauntConfiguration( + mode: .nativeBroker, + root: nil, + durability: .safe, + runtimeFootprint: .smallMobile, + extensions: selectedExtensions + ) + let engine = IOSBrokerEngine( + configuration: brokerConfiguration, + manager: manager + ) + + observations = [ + "buildConfiguration": "release", + "deviceLifecycleLaunchIndex": String(environment.launchIndex), + "deviceLifecycleRunToken": environment.runToken, + "expectWorkerKill": environment.expectWorkerKill ? "YES" : "NO", + "orchestrationTimeoutSeconds": String(environment.orchestrationTimeoutSeconds), + "backgroundActiveWorkSeconds": String( + environment.orchestrationTimeoutSeconds * 2 + ), + "requestDeadlineSeconds": String(environment.orchestrationTimeoutSeconds * 3), + ] + var iterator = events.stream.makeAsyncIterator() + var foregroundActiveEvent: BrokerHostLifecycleEvent? + while let event = await iterator.next() { + if event.kind == .active { + foregroundActiveEvent = event + break + } + } + let foregroundActive = try requireValue( + foregroundActiveEvent, + "host lifecycle event stream ended before initial foreground activation" + ) + observations["foregroundActiveUptimeNanoseconds"] = String( + foregroundActive.observedAtUptimeNanoseconds + ) + try journal.update( + phase: .foregroundActive, + events: events.snapshot() + ) { report in + report.hostPID = hostPID + report.observations = observations + } + + let openedControl = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: databaseConfiguration + ) + controlSession = openedControl + let openedDatabase = try await OliphauntDatabase.open( + configuration: databaseConfiguration, + engine: engine + ) + database = openedDatabase + + let initialManager = await manager.diagnostics() + let initialWorker = try await openedControl.workerDiagnostics() + let initialPID = initialWorker.extensionProcessIdentifier + let initialEpoch = initialWorker.epoch + let initialManifestDigest = try requireValue( + initialWorker.manifestDigest, + "initial worker diagnostics omitted the manifest digest" + ) + try require(initialPID != hostPID) { + "host and extension unexpectedly have the same PID" + } + try require(initialManager.state == .ready(initialEpoch)) { + "manager was not ready at the worker's initial epoch" + } + try require(!initialManager.admissionsPaused) { + "new manager started with admissions paused" + } + try require(!initialWorker.capabilities.backgroundContinuable) { + "worker overclaimed backgroundContinuable" + } + try requireAvailableMemory(initialWorker, phase: "openedIdle") + diagnostics.append( + evidence(phase: "openedIdle", manager: initialManager, worker: initialWorker) + ) + checks.formUnion([ + "extensionDiscovery", + "separatePID", + "workerDiagnostics", + "backgroundContinuableFalse", + "openedIdleMemory", + "availableMemory", + ]) + try journal.update( + phase: .foregroundQualification, + events: events.snapshot() + ) { report in + report.hostPID = hostPID + report.initialWorkerPID = initialPID + report.initialEpoch = initialEpoch.description + report.currentWorkerPID = initialPID + report.currentEpoch = initialEpoch.description + report.manifestDigest = initialManifestDigest + report.diagnostics = diagnostics + report.checks = checks.sorted() + report.observations = observations + } + + let capabilities = await openedControl.capabilities() + try require(capabilities.processIsolated) { + "host capability mapping did not preserve process isolation" + } + try require(capabilities.crashRestartable) { + "host capability mapping did not preserve crash recovery" + } + try require(!capabilities.rootSwitchable && !capabilities.multiRoot) { + "host capability mapping overclaimed root behavior" + } + checks.insert("capabilities") + + let launchObservations = try await verifyCrossLaunchPersistence( + database: openedDatabase, + environment: environment + ) + observations.merge(launchObservations) { _, current in current } + checks.insert("crossLaunchPersistence") + + let writeStartedAt = deviceLifecycleUnixNanoseconds() + try await recreateSizableRelation(database: openedDatabase) + let relationStats = try await openedDatabase.query( + """ + SELECT + count(*)::text AS row_count, + pg_total_relation_size('broker_lifecycle_pressure')::text AS relation_bytes + FROM broker_lifecycle_pressure + """ + ) + let relationRowCount = try integerValue( + relationStats, + column: "row_count", + label: "lifecycle relation row count" + ) + let relationBytes = try unsignedIntegerValue( + relationStats, + column: "relation_bytes", + label: "lifecycle relation size" + ) + try require(relationRowCount == 8_192) { + "sizable lifecycle relation contains \(relationRowCount) rows, expected 8192" + } + try require(relationBytes >= 24 * 1024 * 1024) { + "sizable lifecycle relation occupies only \(relationBytes) bytes" + } + observations["relationBytes"] = String(relationBytes) + observations["relationRows"] = String(relationRowCount) + checks.insert("sizableRelation") + + let foregroundIdleWorker = try await openedControl.workerDiagnostics() + try requireAvailableMemory(foregroundIdleWorker, phase: "foregroundIdle") + diagnostics.append( + evidence( + phase: "foregroundIdle", + manager: await manager.diagnostics(), + worker: foregroundIdleWorker + ) + ) + + let foregroundSleep = Task { + try await openedDatabase.query("SELECT pg_sleep(30)") + } + let foregroundExecuting = try await waitForActiveNativeRequest( + session: openedControl, + timeout: .seconds(5) + ) + try requireAvailableMemory(foregroundExecuting, phase: "executingBeforeCancel") + diagnostics.append( + evidence( + phase: "executingBeforeCancel", + manager: await manager.diagnostics(), + worker: foregroundExecuting + ) + ) + try await openedDatabase.cancel() + try await requirePostgresCancellation(foregroundSleep) + let afterCancel = try await openedDatabase.query( + "SELECT 'live-after-cancel'::text AS status" + ) + try require( + try afterCancel.getText(row: 0, column: "status") == "live-after-cancel" + ) { + "worker was not live after foreground cancellation" + } + diagnostics.append( + evidence( + phase: "afterCancel", + manager: await manager.diagnostics(), + worker: try await openedControl.workerDiagnostics() + ) + ) + checks.formUnion(["cancellation", "postCancelLiveness", "executingMemory"]) + + let declaredQueueCeiling = UInt64( + OliphauntBrokerProtocol.maximumQueuedBytesPerDirection + ) + let maximumSlowStreamFootprintDelta = declaredQueueCeiling * 2 + let requiredSlowStreamAvailableMemoryHeadroom = + declaredQueueCeiling + let smallSlowStreamSamplingDeadlineSeconds = 30 + let slowStreamSamplingDeadlineSeconds = 120 + observations["smallSlowStreamSamplingDeadlineSeconds"] = String( + smallSlowStreamSamplingDeadlineSeconds + ) + observations["slowStreamSamplingDeadlineSeconds"] = String( + slowStreamSamplingDeadlineSeconds + ) + + let smallStreamCounter = DeviceLifecycleStreamCounter() + let smallStreamFinished = DeviceLifecycleSignal() + let smallStreamStartedAt = DispatchTime.now().uptimeNanoseconds + let smallStreaming = Task { + defer { smallStreamFinished.signal() } + try await openedDatabase.execProtocolStream( + try OliphauntProtocol.simpleQuery( + "SELECT repeat('s', 8192) FROM generate_series(1, 1024)" + ) + ) { chunk in + smallStreamCounter.consume(chunk) + Thread.sleep(forTimeInterval: 0.005) + } + } + let smallStreamingSamples: DeviceLifecycleActiveStreamSamples + do { + smallStreamingSamples = try await sampleActiveStreaming( + session: openedControl, + counter: smallStreamCounter, + finished: smallStreamFinished, + timeout: .seconds(smallSlowStreamSamplingDeadlineSeconds) + ) + try await smallStreaming.value + } catch { + smallStreaming.cancel() + _ = try? await smallStreaming.value + throw error + } + let smallStreamingWorker = smallStreamingSamples.representative + try requireAvailableMemory(smallStreamingWorker, phase: "slowStreaming8MiB") + diagnostics.append( + evidence( + phase: "slowStreaming8MiB", + manager: await manager.diagnostics(), + worker: smallStreamingWorker + ) + ) + let smallStreamFinishedAt = DispatchTime.now().uptimeNanoseconds + let smallStreamBytes = smallStreamCounter.byteCount + try require( + smallStreamBytes > 8 * 1024 * 1024 && smallStreamCounter.chunkCount > 1 + ) { + "8 MiB slow-reader probe did not produce a multi-frame response" + } + observations["smallSlowStreamBytes"] = String(smallStreamBytes) + observations["smallSlowStreamChunks"] = String(smallStreamCounter.chunkCount) + observations["smallSlowStreamActiveSampleCount"] = String( + smallStreamingSamples.count + ) + observations["smallSlowStreamElapsedNanoseconds"] = String( + smallStreamFinishedAt &- smallStreamStartedAt + ) + + let streamCounter = DeviceLifecycleStreamCounter() + let streamFinished = DeviceLifecycleSignal() + let streamStartedAt = DispatchTime.now().uptimeNanoseconds + let streaming = Task { + defer { streamFinished.signal() } + try await openedDatabase.execProtocolStream( + try OliphauntProtocol.simpleQuery( + "SELECT repeat('s', 8192) FROM generate_series(1, 4096)" + ) + ) { chunk in + streamCounter.consume(chunk) + Thread.sleep(forTimeInterval: 0.005) + } + } + let streamingSamples: DeviceLifecycleActiveStreamSamples + do { + streamingSamples = try await sampleActiveStreaming( + session: openedControl, + counter: streamCounter, + finished: streamFinished, + timeout: .seconds(slowStreamSamplingDeadlineSeconds) + ) + try await streaming.value + } catch { + streaming.cancel() + _ = try? await streaming.value + throw error + } + let streamingWorker = streamingSamples.representative + try requireAvailableMemory(streamingWorker, phase: "slowStreaming32MiB") + diagnostics.append( + evidence( + phase: "slowStreaming32MiB", + manager: await manager.diagnostics(), + worker: streamingWorker + ) + ) + let streamFinishedAt = DispatchTime.now().uptimeNanoseconds + let streamElapsed = max(1, streamFinishedAt &- streamStartedAt) + let streamBytes = streamCounter.byteCount + let throughput = UInt64( + min( + Double(UInt64.max), + Double(streamBytes) * 1_000_000_000 / Double(streamElapsed) + ) + ) + observations["slowStreamBytes"] = String(streamBytes) + observations["slowStreamChunks"] = String(streamCounter.chunkCount) + observations["slowStreamActiveSampleCount"] = String(streamingSamples.count) + observations["slowStreamElapsedNanoseconds"] = String(streamElapsed) + observations["slowStreamBytesPerSecond"] = String(throughput) + let minimumStreamHeadroom = min( + smallStreamingSamples.minimumAvailableMemoryBytes, + streamingSamples.minimumAvailableMemoryBytes + ) + let responseSizeDelta = UInt64(streamBytes - smallStreamBytes) + let smallStreamingFootprint = smallStreamingSamples.peakPhysFootprintBytes + let largeStreamingFootprint = streamingSamples.peakPhysFootprintBytes + let streamingFootprintDelta = + largeStreamingFootprint > smallStreamingFootprint + ? largeStreamingFootprint - smallStreamingFootprint + : 0 + observations["declaredQueueCeilingBytes"] = String(declaredQueueCeiling) + observations["maximumSlowStreamFootprintDeltaBytes"] = String( + maximumSlowStreamFootprintDelta + ) + observations["slowStreamResponseSizeDeltaBytes"] = String(responseSizeDelta) + observations["requiredSlowStreamAvailableMemoryHeadroomBytes"] = String( + requiredSlowStreamAvailableMemoryHeadroom + ) + observations["minimumSlowStreamAvailableMemoryBytes"] = String( + minimumStreamHeadroom + ) + observations["smallSlowStreamPhysFootprintBytes"] = String( + smallStreamingFootprint + ) + observations["largeSlowStreamPhysFootprintBytes"] = String( + largeStreamingFootprint + ) + observations["slowStreamFootprintDeltaBytes"] = String( + streamingFootprintDelta + ) + observations["slowStreamPeakPhysFootprintBytes"] = String( + max(smallStreamingFootprint, largeStreamingFootprint) + ) + try require(streamBytes > 32 * 1024 * 1024) { + "slow streaming response delivered only \(streamBytes) bytes" + } + try require( + minimumStreamHeadroom > requiredSlowStreamAvailableMemoryHeadroom + ) { + "slow-reader minimum available memory \(minimumStreamHeadroom) did not exceed required bound \(requiredSlowStreamAvailableMemoryHeadroom)" + } + try require(responseSizeDelta > maximumSlowStreamFootprintDelta) { + "slow-reader response-size delta \(responseSizeDelta) did not exceed footprint bound \(maximumSlowStreamFootprintDelta)" + } + try require(streamingFootprintDelta <= maximumSlowStreamFootprintDelta) { + "slow-reader physical-footprint delta \(streamingFootprintDelta) exceeded bound \(maximumSlowStreamFootprintDelta)" + } + checks.formUnion([ + "slowStreamTwoSizes", + "slowStreamThroughput", + "slowStreamBoundedHeadroom", + ]) + + let protocolRTTSamples = try await measureProtocolRTT( + session: openedControl, + sampleCount: 20 + ) + let protocolRTTMedianMilliseconds = protocolRTTSamples[ + protocolRTTSamples.count / 2 + ] + observations["protocolRTTMedianMilliseconds"] = String( + format: "%.3f", + protocolRTTMedianMilliseconds + ) + observations["protocolRTTSampleCount"] = String(protocolRTTSamples.count) + checks.insert("protocolRTT") + + _ = try await openedDatabase.execute( + "UPDATE broker_lifecycle_pressure SET payload = reverse(payload)" + ) + let beforeCheckpointWorker = try await openedControl.workerDiagnostics() + let priorCheckpointSampleSequence = + beforeCheckpointWorker.checkpointMemorySample?.sequence ?? 0 + try await openedControl.checkpoint() + let afterCheckpointWorker = try await openedControl.workerDiagnostics() + let checkpointMemorySample = try requireValue( + afterCheckpointWorker.checkpointMemorySample, + "checkpoint diagnostics omitted the retained memory sample" + ) + try require( + checkpointMemorySample.sequence > priorCheckpointSampleSequence + ) { + "checkpoint diagnostics returned a stale retained memory sample" + } + try require( + checkpointMemorySample.startedAtUptimeNanoseconds + <= checkpointMemorySample.sampledAtUptimeNanoseconds + && checkpointMemorySample.sampledAtUptimeNanoseconds + <= checkpointMemorySample.completedAtUptimeNanoseconds + ) { + "checkpoint memory sample timestamps fall outside the checkpoint interval" + } + try require( + checkpointMemorySample.physFootprintBytes > 0 + && checkpointMemorySample.residentBytes > 0 + && checkpointMemorySample.availableMemoryBytes > 0 + ) { + "checkpoint memory sample omitted footprint, resident, or headroom evidence" + } + try require(!afterCheckpointWorker.checkpointInProgress) { + "checkpointInProgress remained set after checkpoint completion" + } + diagnostics.append( + evidence( + phase: "checkpointMemorySample", + manager: await manager.diagnostics(), + worker: afterCheckpointWorker, + checkpointMemorySample: checkpointMemorySample + ) + ) + try requireAvailableMemory(afterCheckpointWorker, phase: "afterCheckpoint") + diagnostics.append( + evidence( + phase: "afterCheckpoint", + manager: await manager.diagnostics(), + worker: afterCheckpointWorker + ) + ) + observations["priorCheckpointMemorySampleSequence"] = String( + priorCheckpointSampleSequence + ) + observations["checkpointMemorySampleSequence"] = String( + checkpointMemorySample.sequence + ) + observations["checkpointMemorySampleStartedAtUptimeNanoseconds"] = String( + checkpointMemorySample.startedAtUptimeNanoseconds + ) + observations["checkpointMemorySampledAtUptimeNanoseconds"] = String( + checkpointMemorySample.sampledAtUptimeNanoseconds + ) + observations["checkpointMemorySampleCompletedAtUptimeNanoseconds"] = String( + checkpointMemorySample.completedAtUptimeNanoseconds + ) + observations["checkpointInProgressAfterCompletion"] = + afterCheckpointWorker.checkpointInProgress ? "true" : "false" + checks.formUnion(["checkpointControl", "checkpointDiagnostics"]) + + do { + _ = try await openedControl.prepareForBackground( + deadline: Date(timeIntervalSinceNow: -1) + ) + throw DeviceLifecycleFailure.assertion( + "already-expired background preparation unexpectedly succeeded" + ) + } catch let error as BrokerError { + guard case .deadlineExceeded = error else { throw error } + } + let afterExpiredDeadline = await manager.diagnostics() + try require(!afterExpiredDeadline.admissionsPaused) { + "an already-expired background deadline paused admissions" + } + try require(afterExpiredDeadline.state == .ready(initialEpoch)) { + "an already-expired background deadline changed manager state" + } + let afterExpiredQuery = try await rawQuery( + openedControl, + "SELECT 'admitted'::text AS status" + ) + try require(try afterExpiredQuery.getText(row: 0, column: "status") == "admitted") { + "admissions did not remain usable after an already-expired deadline" + } + checks.insert("expiredDeadlineAdmission") + + let backgroundSleep = Task { + try await openedDatabase.query( + "SELECT pg_sleep(\(environment.orchestrationTimeoutSeconds * 2))" + ) + } + _ = try await waitForActiveNativeRequest( + session: openedControl, + timeout: .seconds(5) + ) + let queuedQuery = Task { + try await openedControl.execProtocolRaw( + try OliphauntProtocol.simpleQuery( + "SELECT 'must-not-run-before-resume'::text AS status" + ) + ) + } + try await waitForQueuedOperation(manager: manager, timeout: .seconds(5)) + + let readyEvents = events.snapshot() + let latestApplicationState = readyEvents.last { event in + event.kind != .memoryWarning + } + try require(latestApplicationState?.kind == .active) { + "host was not active immediately before the background handoff" + } + let backgroundTransitionNotBefore = DispatchTime.now().uptimeNanoseconds + observations["backgroundTransitionNotBeforeUptimeNanoseconds"] = String( + backgroundTransitionNotBefore + ) + try journal.update( + phase: .readyForBackground, + events: events.snapshot() + ) { report in + report.writeStartedAtUnixNanoseconds = writeStartedAt + report.checks = checks.sorted() + report.diagnostics = diagnostics + report.observations = observations + } + + var transitionEvent: BrokerHostLifecycleEvent? + while let event = await iterator.next() { + guard event.observedAtUptimeNanoseconds > backgroundTransitionNotBefore else { + continue + } + if event.kind == .inactive || event.kind == .background { + transitionEvent = event + break + } + } + let firstTransition = try requireValue( + transitionEvent, + "host lifecycle event stream ended before inactive/background" + ) + observations["backgroundTransitionUptimeNanoseconds"] = String( + firstTransition.observedAtUptimeNanoseconds + ) + if firstTransition.kind == .inactive { + try journal.update( + phase: .inactiveObserved, + events: events.snapshot() + ) { _ in } + } else { + try journal.update( + phase: .backgroundObserved, + events: events.snapshot() + ) { _ in } + } + + let backgroundDeadline = Date( + timeIntervalSinceNow: backgroundPreparationSeconds + ) + let preparationStartedAt = DispatchTime.now().uptimeNanoseconds + let preparation = Task { + try await openedControl.prepareForBackground(deadline: backgroundDeadline) + } + let paused = try await waitForAdmissionsPaused( + manager: manager, + epoch: initialEpoch, + timeout: .seconds(2) + ) + try require(paused.admissionsPaused) { + "manager did not pause admissions during background preparation" + } + + try await expectQueueClosed("query") { + _ = try await openedControl.execProtocolRaw( + try OliphauntProtocol.simpleQuery("SELECT 1") + ) + } + try await expectQueueClosed("checkpoint") { + try await openedControl.checkpoint() + } + try await expectQueueClosed("logical open") { + let unexpected = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: databaseConfiguration + ) + try await unexpected.close() + } + + let preparationResult = try await preparation.value + let preparationFinishedAt = DispatchTime.now().uptimeNanoseconds + try require(Date() < backgroundDeadline) { + "background preparation returned after its absolute deadline" + } + try require(preparationResult.cancelledActiveWork) { + "background preparation did not report active-work cancellation" + } + try await requirePostgresCancellation(backgroundSleep) + try await requireQueuedCancellation(queuedQuery) + + let quiescedManager = await manager.diagnostics() + let quiescedWorker = try await openedControl.workerDiagnostics() + try require(quiescedManager.state == .quiescing(initialEpoch)) { + "manager did not remain quiescing after background preparation" + } + try require(quiescedManager.admissionsPaused) { + "manager reopened admissions before resume" + } + try require( + quiescedManager.queuedOperationCount == 0 + && quiescedManager.activeRequestID == nil + ) { + "manager retained active or queued work after background preparation" + } + try require( + quiescedWorker.state == "quiescing" + && quiescedWorker.activeRequestID == nil + && quiescedWorker.transactionStatus == "idle" + ) { + "worker did not reach an idle quiescing state" + } + try requireAvailableMemory(quiescedWorker, phase: "quiesced") + diagnostics.append( + evidence( + phase: "quiesced", + manager: quiescedManager, + worker: quiescedWorker + ) + ) + + let protection = try decodeProtectionEvidence( + quiescedWorker.storageProtectionEvidenceJSON + ) + try validateProtectionEvidence( + protection, + writeStartedAtUnixNanoseconds: writeStartedAt + ) + observations["storageEntryCount"] = String(protection.entryCount) + observations["storageRegularFileBytes"] = String(protection.regularFileBytes) + observations["storageRelationFileCount"] = String(protection.relationFileCount) + observations["storageWALFileCount"] = String(protection.walFileCount) + observations["backgroundPreparationCheckpointed"] = + preparationResult.checkpointed ? "true" : "false" + observations["backgroundPreparationElapsedNanoseconds"] = String( + preparationFinishedAt &- preparationStartedAt + ) + checks.formUnion([ + "backgroundCancellation", + "backgroundAdmissionClosed", + "backgroundDeadline", + "recursiveStorageProtection", + "relationAndWALFreshness", + ]) + + var backgroundEvent: BrokerHostLifecycleEvent? + if firstTransition.kind == .background { + backgroundEvent = firstTransition + } else { + while let event = await iterator.next() { + if event.kind == .background { + backgroundEvent = event + break + } + } + } + let observedBackground = try requireValue( + backgroundEvent, + "host lifecycle event stream ended before actual background" + ) + observations["backgroundObservedUptimeNanoseconds"] = String( + observedBackground.observedAtUptimeNanoseconds + ) + checks.insert("actualBackground") + try journal.update( + phase: .backgroundObserved, + events: events.snapshot() + ) { report in + report.checks = checks.sorted() + report.diagnostics = diagnostics + report.observations = observations + report.storageProtection = protection + } + try journal.update( + phase: .quiesced, + events: events.snapshot() + ) { report in + report.checks = checks.sorted() + report.diagnostics = diagnostics + report.observations = observations + report.storageProtection = protection + } + + var activeEvent: BrokerHostLifecycleEvent? + while let event = await iterator.next() { + if event.kind == .active { + activeEvent = event + break + } + } + let observedActive = try requireValue( + activeEvent, + "host lifecycle event stream ended before foreground activation" + ) + observations["resumedActiveUptimeNanoseconds"] = String( + observedActive.observedAtUptimeNanoseconds + ) + try journal.update( + phase: .activeObserved, + events: events.snapshot() + ) { report in + report.observations = observations + } + + try await openedControl.resumeFromBackground() + let resumedManager = await manager.diagnostics() + let resumedWorker = try await openedControl.workerDiagnostics() + let resumedDigest = try requireValue( + resumedWorker.manifestDigest, + "resumed worker diagnostics omitted the manifest digest" + ) + try require(resumedDigest == initialManifestDigest) { + "resume changed the extension-private root manifest" + } + let resumedWithSamePID = + resumedWorker.extensionProcessIdentifier == initialPID + let resumedWithSameEpoch = resumedWorker.epoch == initialEpoch + if environment.expectWorkerKill { + try require( + !resumedWithSamePID && !resumedWithSameEpoch + ) { + "worker-kill qualification did not establish a fresh PID and epoch" + } + recoveredEpochs.append(resumedWorker.epoch.description) + checks.insert("backgroundWorkerKillRecovery") + } else { + try require(resumedWithSamePID == resumedWithSameEpoch) { + "healthy background resume produced a mixed PID/epoch identity" + } + if resumedWithSamePID { + checks.insert("backgroundSameWorkerResume") + } else { + recoveredEpochs.append(resumedWorker.epoch.description) + checks.insert("backgroundFreshWorkerResume") + } + } + try require( + resumedManager.state == .ready(resumedWorker.epoch) + && !resumedManager.admissionsPaused + ) { + "resume admitted work before the manager reached ready" + } + let resumedHealth = try await rawQuery( + openedControl, + "SELECT 'resumed'::text AS status" + ) + try require(try resumedHealth.getText(row: 0, column: "status") == "resumed") { + "post-resume health query returned the wrong value" + } + try requireAvailableMemory(resumedWorker, phase: "resumed") + diagnostics.append( + evidence( + phase: "resumed", + manager: resumedManager, + worker: resumedWorker + ) + ) + checks.formUnion(["backgroundResume", "postResumeHealth", "postResumeMemory"]) + try journal.update( + phase: .resumed, + events: events.snapshot() + ) { report in + report.currentWorkerPID = resumedWorker.extensionProcessIdentifier + report.currentEpoch = resumedWorker.epoch.description + report.checks = checks.sorted() + report.diagnostics = diagnostics + report.observations = observations + } + + let finalRelation = try await openedDatabase.query( + "SELECT count(*)::text AS row_count FROM broker_lifecycle_pressure" + ) + try require( + try integerValue( + finalRelation, + column: "row_count", + label: "post-resume lifecycle relation row count" + ) == 8_192 + ) { + "post-resume relation content changed" + } + let finalMarkers = try await openedDatabase.query( + """ + SELECT count(*)::text AS marker_count + FROM broker_lifecycle_launches + WHERE run_token = $1 + """, + parameters: [.text(environment.runToken)] + ) + try require( + try integerValue( + finalMarkers, + column: "marker_count", + label: "final lifecycle launch-marker count" + ) == environment.launchIndex + ) { + "post-resume launch-marker history changed" + } + checks.insert("postResumePersistence") + + observations["initialWorkerPID"] = String(initialPID) + observations["resumedWorkerPID"] = String( + resumedWorker.extensionProcessIdentifier + ) + observations["resumedEpoch"] = resumedWorker.epoch.description + let result = BrokerProbeResult( + hostPID: hostPID, + workerPID: initialPID, + epoch: initialEpoch.description, + checks: checks.sorted(), + recoveredEpochs: recoveredEpochs, + diagnostics: diagnostics, + observations: observations + ) + + try await openedDatabase.close() + database = nil + try await openedControl.close() + controlSession = nil + + try journal.complete( + result: result, + diagnostics: diagnostics, + checks: checks.sorted(), + observations: observations, + storageProtection: protection, + events: events.snapshot() + ) + return result + } catch { + if let database { + try? await database.cancel() + try? await database.close() + } + if let controlSession { + try? await controlSession.close() + } + try? journal.fail( + error: error, + diagnostics: diagnostics, + checks: checks.sorted(), + observations: observations, + events: events.snapshot() + ) + throw error + } + } + + private static func verifyCrossLaunchPersistence( + database: OliphauntDatabase, + environment: DeviceLifecycleEnvironment + ) async throws -> [String: String] { + _ = try await database.execute( + """ + CREATE TABLE IF NOT EXISTS broker_lifecycle_launches( + run_token text NOT NULL, + launch_index integer NOT NULL, + marker text NOT NULL, + created_at_unix_nanoseconds numeric NOT NULL, + PRIMARY KEY(run_token, launch_index) + ) + """ + ) + let before = try await database.query( + """ + SELECT + count(*)::text AS total, + count(*) FILTER (WHERE launch_index = 1)::text AS launch_one, + count(*) FILTER (WHERE launch_index = 2)::text AS launch_two + FROM broker_lifecycle_launches + WHERE run_token = $1 + """, + parameters: [.text(environment.runToken)] + ) + let total = try integerValue(before, column: "total", label: "prior marker count") + let launchOne = try integerValue( + before, + column: "launch_one", + label: "launch-one marker count" + ) + let launchTwo = try integerValue( + before, + column: "launch_two", + label: "launch-two marker count" + ) + if environment.launchIndex == 1 { + try require(total == 0 && launchOne == 0 && launchTwo == 0) { + "launch one found stale markers for its run token" + } + } else { + try require(total == 1 && launchOne == 1 && launchTwo == 0) { + "launch two did not find exactly the launch-one marker" + } + } + _ = try await database.query( + """ + INSERT INTO broker_lifecycle_launches( + run_token, + launch_index, + marker, + created_at_unix_nanoseconds + ) + VALUES ($1, $2::integer, $3, $4::numeric) + """, + parameters: [ + .text(environment.runToken), + .text(String(environment.launchIndex)), + .text("\(environment.runToken):\(environment.launchIndex)"), + .text(String(deviceLifecycleUnixNanoseconds())), + ] + ) + let after = try await database.query( + """ + SELECT count(*)::text AS total + FROM broker_lifecycle_launches + WHERE run_token = $1 + """, + parameters: [.text(environment.runToken)] + ) + try require( + try integerValue(after, column: "total", label: "current marker count") + == environment.launchIndex + ) { + "current launch marker was not persisted exactly once" + } + return [ + "priorLaunchMarkerCount": String(total), + "currentLaunchMarker": + "\(environment.runToken):\(environment.launchIndex)", + ] + } + + private static func recreateSizableRelation( + database: OliphauntDatabase + ) async throws { + _ = try await database.execute( + """ + CREATE EXTENSION IF NOT EXISTS vector; + CREATE EXTENSION IF NOT EXISTS pg_trgm; + DROP TABLE IF EXISTS broker_lifecycle_pressure; + CREATE TABLE broker_lifecycle_pressure( + id integer PRIMARY KEY, + payload text NOT NULL + ); + INSERT INTO broker_lifecycle_pressure(id, payload) + SELECT + value, + ( + SELECT string_agg( + md5(value::text || ':' || part::text), + '' + ORDER BY part + ) + FROM generate_series(1, 128) AS parts(part) + ) + FROM generate_series(1, 8192) AS value; + ANALYZE broker_lifecycle_pressure; + """ + ) + } + + private static func rawQuery( + _ session: IOSBrokerSession, + _ sql: String + ) async throws -> OliphauntQueryResult { + try await parseOliphauntQueryResponse( + session.execProtocolRaw(try OliphauntProtocol.simpleQuery(sql)) + ) + } + + private static func waitForActiveNativeRequest( + session: IOSBrokerSession, + timeout: Duration + ) async throws -> IOSBrokerWorkerDiagnostics { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + var lastError: (any Error)? + while clock.now < deadline { + do { + let diagnostics = try await session.workerDiagnostics() + if diagnostics.activeRequestID != nil && diagnostics.nativeDispatchStarted { + return diagnostics + } + } catch { + lastError = error + } + try await Task.sleep(for: .milliseconds(20)) + } + if let lastError { throw lastError } + throw DeviceLifecycleFailure.assertion( + "worker diagnostics did not observe active native dispatch" + ) + } + + private static func waitForQueuedOperation( + manager: IOSBrokerManager, + timeout: Duration + ) async throws { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + while clock.now < deadline { + let diagnostics = await manager.diagnostics() + if diagnostics.queuedOperationCount > 0 { + return + } + try await Task.sleep(for: .milliseconds(20)) + } + throw DeviceLifecycleFailure.assertion( + "manager diagnostics did not observe a queued operation" + ) + } + + private static func waitForAdmissionsPaused( + manager: IOSBrokerManager, + epoch: BrokerEpoch, + timeout: Duration + ) async throws -> IOSBrokerDiagnostics { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + while clock.now < deadline { + let diagnostics = await manager.diagnostics() + if diagnostics.admissionsPaused + && diagnostics.state == .quiescing(epoch) + { + return diagnostics + } + try await Task.sleep(for: .milliseconds(10)) + } + throw DeviceLifecycleFailure.assertion( + "manager did not pause admissions while quiescing" + ) + } + + private static func sampleActiveStreaming( + session: IOSBrokerSession, + counter: DeviceLifecycleStreamCounter, + finished: DeviceLifecycleSignal, + timeout: Duration + ) async throws -> DeviceLifecycleActiveStreamSamples { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + var lastError: (any Error)? + var lastSampledChunkCount = 0 + var samples: [IOSBrokerWorkerDiagnostics] = [] + while clock.now < deadline { + do { + let diagnostics = try await session.workerDiagnostics() + let chunkCount = counter.chunkCount + if chunkCount > lastSampledChunkCount, + diagnostics.activeRequestID != nil, + diagnostics.nativeDispatchStarted + { + try requireAvailableMemory(diagnostics, phase: "activeSlowStreaming") + samples.append(diagnostics) + lastSampledChunkCount = chunkCount + } + } catch { + lastError = error + } + if finished.isSignaled { break } + try await Task.sleep(for: .milliseconds(5)) + } + guard finished.isSignaled else { + if let lastError { throw lastError } + throw DeviceLifecycleFailure.assertion( + "slow-reader probe did not finish before its sampling deadline" + ) + } + try require(samples.count > 1) { + "slow-reader probe did not produce repeated active native-dispatch samples" + } + return try DeviceLifecycleActiveStreamSamples(samples: samples) + } + + private static func measureProtocolRTT( + session: IOSBrokerSession, + sampleCount: Int + ) async throws -> [Double] { + try require(sampleCount > 0) { + "protocol RTT sample count must be positive" + } + var milliseconds: [Double] = [] + milliseconds.reserveCapacity(sampleCount) + for _ in 0.. + ) async throws { + do { + _ = try await task.value + throw DeviceLifecycleFailure.assertion( + "active pg_sleep completed without cancellation" + ) + } catch OliphauntError.postgres(let postgresError) { + try require(postgresError.sqlstate == "57014") { + "active pg_sleep cancellation returned SQLSTATE \(postgresError.sqlstate ?? "nil")" + } + } + } + + private static func requireQueuedCancellation( + _ task: Task + ) async throws { + do { + _ = try await task.value + throw DeviceLifecycleFailure.assertion( + "queued query completed instead of being canceled before dispatch" + ) + } catch let error as BrokerError { + guard case .canceled = error else { throw error } + } + } + + private static func expectQueueClosed( + _ operationName: String, + operation: () async throws -> Void + ) async throws { + do { + try await operation() + throw DeviceLifecycleFailure.assertion( + "\(operationName) was admitted while the broker was quiescing" + ) + } catch let error as BrokerError { + guard case .rejected(.queueClosed) = error else { throw error } + } + } + + private static func requireAvailableMemory( + _ diagnostics: IOSBrokerWorkerDiagnostics, + phase: String + ) throws { + try require((diagnostics.availableMemoryBytes ?? 0) > 0) { + "\(phase) diagnostics omitted available process memory" + } + try require( + (diagnostics.currentPhysFootprintBytes ?? 0) > 0 + && (diagnostics.currentResidentBytes ?? 0) > 0 + ) { + "\(phase) diagnostics omitted worker footprint or resident memory" + } + } + + private static func decodeProtectionEvidence( + _ encoded: String? + ) throws -> DeviceLifecycleStorageProtectionEvidence { + let encoded = try requireValue( + encoded, + "quiesced diagnostics omitted recursive storage-protection evidence" + ) + let data = Data(encoded.utf8) + let object = try JSONSerialization.jsonObject(with: data) + guard let dictionary = object as? [String: Any] else { + throw DeviceLifecycleFailure.assertion( + "storage-protection evidence was not a JSON object" + ) + } + let allowedKeys: Set = [ + "expectedProtection", + "entryCount", + "regularFileCount", + "directoryCount", + "otherEntryCount", + "symbolicLinkCount", + "matchingProtectionCount", + "missingProtectionCount", + "mismatchedProtectionCount", + "protectionMetadataUnavailableCount", + "unreadableEntryCount", + "regularFileBytes", + "relationFileCount", + "walFileCount", + "newestRelationModificationUnixNanoseconds", + "newestWALModificationUnixNanoseconds", + "enumerationFailed", + ] + let unexpectedKeys = Set(dictionary.keys).subtracting(allowedKeys) + try require(unexpectedKeys.isEmpty) { + "storage-protection evidence exposed unexpected fields: \(unexpectedKeys.sorted())" + } + return try JSONDecoder().decode( + DeviceLifecycleStorageProtectionEvidence.self, + from: data + ) + } + + private static func validateProtectionEvidence( + _ evidence: DeviceLifecycleStorageProtectionEvidence, + writeStartedAtUnixNanoseconds: UInt64 + ) throws { + try require( + evidence.expectedProtection + == FileProtectionType.completeUntilFirstUserAuthentication.rawValue + ) { + "storage audit expected the wrong data-protection class" + } + try require(evidence.entryCount > 0 && evidence.regularFileBytes > 0) { + "storage audit did not observe nonempty recursive storage" + } + try require(evidence.allEntriesMatchExpectedProtection) { + "recursive storage entries did not all match the declared protection class" + } + try require(evidence.relationFileCount > 0 && evidence.walFileCount > 0) { + "storage audit did not observe both relation and WAL files" + } + let earliestAccepted = + writeStartedAtUnixNanoseconds > filesystemTimestampToleranceNanoseconds + ? writeStartedAtUnixNanoseconds - filesystemTimestampToleranceNanoseconds + : 0 + try require( + (evidence.newestRelationModificationUnixNanoseconds ?? 0) >= earliestAccepted + ) { + "newest relation-file modification predates the lifecycle write" + } + try require( + (evidence.newestWALModificationUnixNanoseconds ?? 0) >= earliestAccepted + ) { + "newest WAL-file modification predates the lifecycle write" + } + } + + private static func integerValue( + _ result: OliphauntQueryResult, + column: String, + label: String + ) throws -> Int { + let text = try requireValue( + result.getText(row: 0, column: column), + "\(label) was NULL" + ) + guard let value = Int(text) else { + throw DeviceLifecycleFailure.assertion("\(label) was not an integer: \(text)") + } + return value + } + + private static func unsignedIntegerValue( + _ result: OliphauntQueryResult, + column: String, + label: String + ) throws -> UInt64 { + let text = try requireValue( + result.getText(row: 0, column: column), + "\(label) was NULL" + ) + guard let value = UInt64(text) else { + throw DeviceLifecycleFailure.assertion( + "\(label) was not an unsigned integer: \(text)" + ) + } + return value + } + + private static func evidence( + phase: String, + manager: IOSBrokerDiagnostics, + worker: IOSBrokerWorkerDiagnostics?, + checkpointMemorySample: IOSBrokerCheckpointMemorySample? = nil + ) -> BrokerDiagnosticEvidence { + BrokerDiagnosticEvidence( + phase: phase, + managerState: managerState(manager.state), + epoch: manager.epoch?.description, + workerPID: manager.extensionProcessIdentifier, + logicalHandleCount: manager.logicalHandleCount, + queuedOperationCount: manager.queuedOperationCount, + activeRequestID: worker?.activeRequestID?.rawValue + ?? manager.activeRequestID?.rawValue, + launchCount: manager.launchCount, + interruptionCount: manager.interruptionCount, + admissionsPaused: manager.admissionsPaused, + workerState: worker?.state, + transactionStatus: worker?.transactionStatus, + manifestDigest: worker?.manifestDigest, + currentPhysFootprintBytes: + checkpointMemorySample?.physFootprintBytes + ?? worker?.currentPhysFootprintBytes, + currentResidentBytes: + checkpointMemorySample?.residentBytes ?? worker?.currentResidentBytes, + availableMemoryBytes: + checkpointMemorySample?.availableMemoryBytes + ?? worker?.availableMemoryBytes, + nativeDispatchStarted: worker?.nativeDispatchStarted ?? false, + checkpointInProgress: worker?.checkpointInProgress ?? false, + storageProtectionEvidenceJSON: worker?.storageProtectionEvidenceJSON, + extensionEntryPreOpenPhysFootprintBytes: + worker?.extensionEntryPreOpenPhysFootprintBytes, + extensionEntryPreOpenResidentBytes: + worker?.extensionEntryPreOpenResidentBytes, + openedIdlePhysFootprintBytes: worker?.openedIdlePhysFootprintBytes, + openedIdleResidentBytes: worker?.openedIdleResidentBytes + ) + } + + private static func managerState(_ state: IOSBrokerManagerState) -> String { + switch state { + case .unavailable: "unavailable" + case .idle: "idle" + case .launching: "launching" + case .binding: "binding" + case .recovering: "recovering" + case .ready: "ready" + case .quiescing: "quiescing" + case .interrupted: "interrupted" + case .closing: "closing" + } + } + + private static func require( + _ condition: @autoclosure () throws -> Bool, + _ message: () -> String + ) throws { + guard try condition() else { + throw DeviceLifecycleFailure.assertion(message()) + } + } + + private static func requireValue( + _ value: @autoclosure () throws -> T?, + _ message: @autoclosure () -> String + ) throws -> T { + guard let value = try value() else { + throw DeviceLifecycleFailure.assertion(message()) + } + return value + } + } + + private struct DeviceLifecycleEnvironment: Sendable { + var runToken: String + var launchIndexText: String + var expectWorkerKillText: String + var orchestrationTimeoutSecondsText: String + var launchIndex: Int = 0 + var expectWorkerKill = false + var orchestrationTimeoutSeconds = 0 + + static func capture() -> DeviceLifecycleEnvironment { + let environment = ProcessInfo.processInfo.environment + return DeviceLifecycleEnvironment( + runToken: environment["OLIPHAUNT_BROKER_LIFECYCLE_RUN_TOKEN"] ?? "", + launchIndexText: + environment["OLIPHAUNT_BROKER_LIFECYCLE_LAUNCH_INDEX"] ?? "", + expectWorkerKillText: + environment["OLIPHAUNT_BROKER_LIFECYCLE_EXPECT_WORKER_KILL"] ?? "", + orchestrationTimeoutSecondsText: + environment["OLIPHAUNT_BROKER_LIFECYCLE_ORCHESTRATION_TIMEOUT_SECONDS"] ?? "" + ) + } + + func validated() throws -> DeviceLifecycleEnvironment { + let token = runToken.trimmingCharacters(in: .whitespacesAndNewlines) + guard !token.isEmpty, token.utf8.count <= 256, !token.utf8.contains(0) else { + throw DeviceLifecycleFailure.invalidEnvironment( + "OLIPHAUNT_BROKER_LIFECYCLE_RUN_TOKEN must be 1...256 UTF-8 bytes" + ) + } + guard let launchIndex = Int(launchIndexText), launchIndex == 1 || launchIndex == 2 + else { + throw DeviceLifecycleFailure.invalidEnvironment( + "OLIPHAUNT_BROKER_LIFECYCLE_LAUNCH_INDEX must be 1 or 2" + ) + } + let normalizedKill = expectWorkerKillText.uppercased() + guard normalizedKill == "YES" || normalizedKill == "NO" else { + throw DeviceLifecycleFailure.invalidEnvironment( + "OLIPHAUNT_BROKER_LIFECYCLE_EXPECT_WORKER_KILL must be YES or NO" + ) + } + guard + let orchestrationTimeoutSeconds = Int(orchestrationTimeoutSecondsText), + (30...600).contains(orchestrationTimeoutSeconds) + else { + throw DeviceLifecycleFailure.invalidEnvironment( + "OLIPHAUNT_BROKER_LIFECYCLE_ORCHESTRATION_TIMEOUT_SECONDS must be 30...600" + ) + } + var value = self + value.runToken = token + value.launchIndex = launchIndex + value.expectWorkerKill = normalizedKill == "YES" + value.orchestrationTimeoutSeconds = orchestrationTimeoutSeconds + return value + } + } + + private enum DeviceLifecycleJournalPhase: String, Codable, Sendable { + case starting + case foregroundActive + case foregroundQualification + case readyForBackground + case inactiveObserved + case backgroundObserved + case quiesced + case activeObserved + case resumed + case completed + case failed + } + + private struct DeviceLifecycleStorageProtectionEvidence: Codable, Equatable, Sendable { + var expectedProtection = "" + var entryCount = 0 + var regularFileCount = 0 + var directoryCount = 0 + var otherEntryCount = 0 + var symbolicLinkCount = 0 + var matchingProtectionCount = 0 + var missingProtectionCount = 0 + var mismatchedProtectionCount = 0 + var protectionMetadataUnavailableCount = 0 + var unreadableEntryCount = 0 + var regularFileBytes: UInt64 = 0 + var relationFileCount = 0 + var walFileCount = 0 + var newestRelationModificationUnixNanoseconds: UInt64? + var newestWALModificationUnixNanoseconds: UInt64? + var enumerationFailed = false + + var allEntriesMatchExpectedProtection: Bool { + !enumerationFailed + && unreadableEntryCount == 0 + && missingProtectionCount == 0 + && mismatchedProtectionCount == 0 + && protectionMetadataUnavailableCount == 0 + && symbolicLinkCount == 0 + && entryCount == matchingProtectionCount + } + } + + private struct DeviceLifecycleJournalReport: Codable { + var schemaVersion = 1 + var status = "running" + var phase = DeviceLifecycleJournalPhase.starting + var runToken = "" + var launchIndex = 0 + var expectWorkerKill = false + var hostPID: Int32? + var initialWorkerPID: Int32? + var initialEpoch: String? + var currentWorkerPID: Int32? + var currentEpoch: String? + var manifestDigest: String? + var updatedAtUnixNanoseconds: UInt64 = deviceLifecycleUnixNanoseconds() + var writeStartedAtUnixNanoseconds: UInt64? + var checks: [String] = [] + var events: [BrokerHostLifecycleEvent] = [] + var diagnostics: [BrokerDiagnosticEvidence] = [] + var observations: [String: String] = [:] + var storageProtection: DeviceLifecycleStorageProtectionEvidence? + var result: BrokerProbeResult? + var error: String? + } + + private final class DeviceLifecycleJournalWriter { + private let url: URL + private var report: DeviceLifecycleJournalReport + + init(environment: DeviceLifecycleEnvironment) throws { + guard + let documents = FileManager.default.urls( + for: .documentDirectory, + in: .userDomainMask + ).first + else { + throw DeviceLifecycleFailure.assertion("host Documents directory is unavailable") + } + url = documents.appendingPathComponent( + "broker-lifecycle-report.json", + isDirectory: false + ) + report = DeviceLifecycleJournalReport( + runToken: environment.runToken, + launchIndex: Int(environment.launchIndexText) ?? 0, + expectWorkerKill: environment.expectWorkerKillText.uppercased() == "YES" + ) + try persist() + publishPhase() + } + + func update( + phase: DeviceLifecycleJournalPhase, + events: [BrokerHostLifecycleEvent], + _ update: (inout DeviceLifecycleJournalReport) -> Void + ) throws { + report.phase = phase + report.events = events + report.updatedAtUnixNanoseconds = deviceLifecycleUnixNanoseconds() + update(&report) + try persist() + publishPhase() + } + + func complete( + result: BrokerProbeResult, + diagnostics: [BrokerDiagnosticEvidence], + checks: [String], + observations: [String: String], + storageProtection: DeviceLifecycleStorageProtectionEvidence, + events: [BrokerHostLifecycleEvent] + ) throws { + report.status = "pass" + report.phase = .completed + report.result = result + report.diagnostics = diagnostics + report.checks = checks + report.observations = observations + report.storageProtection = storageProtection + report.events = events + report.updatedAtUnixNanoseconds = deviceLifecycleUnixNanoseconds() + report.error = nil + try persist() + publishPhase() + } + + func fail( + error: any Error, + diagnostics: [BrokerDiagnosticEvidence], + checks: [String], + observations: [String: String], + events: [BrokerHostLifecycleEvent] + ) throws { + report.status = "fail" + report.phase = .failed + report.error = String(describing: error) + report.diagnostics = diagnostics + report.checks = checks + report.observations = observations + report.events = events + report.updatedAtUnixNanoseconds = deviceLifecycleUnixNanoseconds() + try persist() + publishPhase() + } + + private func persist() throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + try encoder.encode(report).write(to: url, options: .atomic) + } + + private func publishPhase() { + print( + "OLIPHAUNT_BROKER_LIFECYCLE phase=\(report.phase.rawValue) status=\(report.status)" + ) + } + } + + private final class DeviceLifecycleSignal: @unchecked Sendable { + private let lock = NSLock() + private var signaled = false + + var isSignaled: Bool { + lock.lock() + defer { lock.unlock() } + return signaled + } + + func signal() { + lock.lock() + signaled = true + lock.unlock() + } + } + + private struct DeviceLifecycleActiveStreamSamples: Sendable { + var representative: IOSBrokerWorkerDiagnostics + var count: Int + var peakPhysFootprintBytes: UInt64 + var minimumAvailableMemoryBytes: UInt64 + + init(samples: [IOSBrokerWorkerDiagnostics]) throws { + guard + let representative = samples.max(by: { + ($0.currentPhysFootprintBytes ?? 0) < ($1.currentPhysFootprintBytes ?? 0) + }), + let minimumAvailableMemoryBytes = samples.compactMap(\.availableMemoryBytes).min() + else { + throw DeviceLifecycleFailure.assertion( + "active stream samples omitted physical-footprint or available-memory evidence" + ) + } + self.representative = representative + count = samples.count + peakPhysFootprintBytes = representative.currentPhysFootprintBytes ?? 0 + self.minimumAvailableMemoryBytes = minimumAvailableMemoryBytes + } + } + + private final class DeviceLifecycleStreamCounter: @unchecked Sendable { + private let lock = NSLock() + private var bytes = 0 + private var chunks = 0 + + func consume(_ data: Data) { + lock.lock() + bytes += data.count + chunks += 1 + lock.unlock() + } + + var byteCount: Int { + lock.lock() + defer { lock.unlock() } + return bytes + } + + var chunkCount: Int { + lock.lock() + defer { lock.unlock() } + return chunks + } + } + + private enum DeviceLifecycleFailure: Error, CustomStringConvertible { + case invalidEnvironment(String) + case assertion(String) + + var description: String { + switch self { + case .invalidEnvironment(let message): + "device lifecycle environment is invalid: \(message)" + case .assertion(let message): + "device lifecycle assertion failed: \(message)" + } + } + } +#endif + +private func deviceLifecycleUnixNanoseconds(_ date: Date = Date()) -> UInt64 { + let interval = date.timeIntervalSince1970 + guard interval.isFinite, interval >= 0, + interval < Double(UInt64.max) / 1_000_000_000 + else { + return 0 + } + return UInt64((interval * 1_000_000_000).rounded()) +} diff --git a/spikes/ios-native-broker/Host/ExtendedFaultMatrix.swift b/spikes/ios-native-broker/Host/ExtendedFaultMatrix.swift new file mode 100644 index 00000000..e70c7a79 --- /dev/null +++ b/spikes/ios-native-broker/Host/ExtendedFaultMatrix.swift @@ -0,0 +1,365 @@ +#if DEBUG && canImport(OliphauntIOSBroker) + import Foundation + import Oliphaunt + import OliphauntBrokerProtocol + import OliphauntIOSBroker + + /// Destructive DEBUG-only fault matrix. The ordinary semantic fixture remains + /// stable; the simulator runner selects this mode explicitly in a separate + /// installed-app launch. + enum ExtendedFaultMatrix { + static func run() async throws -> BrokerProbeResult { + let manager = IOSBrokerManager() + let brokerConfiguration = IOSBrokerConfiguration( + expectedABI: 6, + startupConfigurationDigest: "ios-native-broker-spike-v2-restricted-role", + maximumRequestBytes: OliphauntBrokerProtocol.defaultMaximumRequestBytes, + requestDeadline: .seconds(6), + extensionBundleIdentifier: + BrokerFixtureBundleIdentifiers.extensionBundleIdentifier, + controlReplyTimeout: .seconds(6), + cancellationGracePeriod: .seconds(1) + ) + let databaseConfiguration = OliphauntConfiguration( + mode: .nativeBroker, + durability: .safe, + runtimeFootprint: .smallMobile, + extensions: ["pg_trgm", "vector"] + ) + let engine = IOSBrokerEngine( + configuration: brokerConfiguration, + manager: manager + ) + let control = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: databaseConfiguration + ) + let database = try await OliphauntDatabase.open( + configuration: databaseConfiguration, + engine: engine + ) + let initial = try await control.workerDiagnostics() + var current = initial + var checks = Set() + var recoveredEpochs: [String] = [] + var observations: [String: String] = [ + "initialWorkerPID": String(initial.extensionProcessIdentifier), + "initialManifestDigest": initial.manifestDigest ?? "", + ] + + let capabilities = try await database.capabilities() + try require(!capabilities.backupRestore) { + "broker advertised whole-archive backup/restore" + } + checks.insert("archiveBoundaryRejected") + + var differentRoot = databaseConfiguration + differentRoot.root = URL(fileURLWithPath: "/tmp/not-broker-default") + do { + _ = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: differentRoot + ) + throw ExtendedFaultFailure.assertion("a different broker root was accepted") + } catch let error as BrokerError { + guard case .rootMismatch = error else { throw error } + } + checks.insert("differentRootRejected") + + _ = try await database.execute( + """ + CREATE TABLE IF NOT EXISTS broker_fault_matrix( + marker text PRIMARY KEY + ); + DELETE FROM broker_fault_matrix; + """ + ) + + let racingSession = try await engine.open( + configuration: databaseConfiguration + ) + guard let racingControl = racingSession as? IOSBrokerSession else { + throw ExtendedFaultFailure.assertion( + "broker engine did not return IOSBrokerSession for race fixture" + ) + } + let racingExecution = Task { + try await racingControl.execProtocolRaw( + try OliphauntProtocol.simpleQuery("SELECT pg_sleep(3)") + ) + } + try await waitForNativeExecution(manager, session: racingControl) + let racingCancel = Task { try await cancelDuringCloseRace(racingControl) } + let racingClose = Task { try await racingControl.close() } + let cancelControlOutcome = try await racingCancel.value + try await racingClose.value + let response = try await racingExecution.value + let raceTerminal = try cancellationTerminal(from: response) + observations["closeCancelCompletionTerminal"] = raceTerminal + observations["closeCancelControlOutcome"] = cancelControlOutcome + let postRaceHealth = try await database.query( + "SELECT 'live-after-race'::text AS status" + ) + try require( + try postRaceHealth.getText(row: 0, column: "status") == "live-after-race" + ) { + "physical session was not live after close/cancel/completion race" + } + let postRaceManager = await manager.diagnostics() + try require( + postRaceManager.activeRequestID == nil + && postRaceManager.queuedOperationCount == 0 + ) { + "close/cancel/completion race left a request pending" + } + checks.insert("closeCancelCompletionRace") + + try await control.injectFault(.beforeNativeDispatch) + try await expectOutcomeUnknown { + _ = try await database.execute( + "INSERT INTO broker_fault_matrix(marker) VALUES ('before-dispatch')" + ) + } + current = try await recover( + from: current, + control: control, + database: database, + recoveredEpochs: &recoveredEpochs + ) + let beforeDispatch = try await database.query( + "SELECT count(*)::text AS count FROM broker_fault_matrix WHERE marker = 'before-dispatch'" + ) + try require(try beforeDispatch.getText(row: 0, column: "count") == "0") { + "the before-dispatch crash reached PostgreSQL" + } + checks.formUnion(["beforeDispatchCrash", "beforeDispatchNotReplayed"]) + + let partial = LockedFaultStreamCounter() + try await control.injectFault(.afterResponseChunks) + try await expectOutcomeUnknown { + try await control.execProtocolStream( + try OliphauntProtocol.simpleQuery( + "SELECT repeat('f', 4096) FROM generate_series(1, 512)" + ) + ) { chunk in + partial.consume(chunk) + } + } + try require(partial.byteCount > 0) { + "after-N-chunks crash delivered no partial streaming evidence" + } + observations["partialResponseBytesBeforeCrash"] = String(partial.byteCount) + observations["partialResponseChunksBeforeCrash"] = String(partial.chunkCount) + current = try await recover( + from: current, + control: control, + database: database, + recoveredEpochs: &recoveredEpochs + ) + checks.formUnion(["afterResponseChunksCrash", "partialStreamOutcomeUnknown"]) + + try await control.injectFault(.duringCheckpoint) + try await expectControlInterruption { + try await control.checkpoint() + } + current = try await recover( + from: current, + control: control, + database: database, + recoveredEpochs: &recoveredEpochs + ) + checks.insert("checkpointCrashRecovery") + + try await expectControlInterruption { + try await control.injectFault(.abort) + } + current = try await recover( + from: current, + control: control, + database: database, + recoveredEpochs: &recoveredEpochs + ) + checks.insert("idleAbortRecovery") + + try await expectControlInterruption { + try await control.injectFault(.invalidMemoryAccess) + } + current = try await recover( + from: current, + control: control, + database: database, + recoveredEpochs: &recoveredEpochs + ) + checks.insert("idleSIGSEGVRecovery") + + observations["finalWorkerPID"] = String(current.extensionProcessIdentifier) + observations["finalManifestDigest"] = current.manifestDigest ?? "" + observations["recoveryCount"] = String(recoveredEpochs.count) + try require(current.manifestDigest == initial.manifestDigest) { + "fault recovery changed the resident root manifest" + } + + try await database.close() + try await control.close() + return BrokerProbeResult( + hostPID: Int32(ProcessInfo.processInfo.processIdentifier), + workerPID: initial.extensionProcessIdentifier, + epoch: initial.epoch.description, + checks: checks.sorted(), + recoveredEpochs: recoveredEpochs, + observations: observations + ) + } + + private static func recover( + from stale: IOSBrokerWorkerDiagnostics, + control: IOSBrokerSession, + database: OliphauntDatabase, + recoveredEpochs: inout [String] + ) async throws -> IOSBrokerWorkerDiagnostics { + let probe = try await database.query("SELECT 'healthy'::text AS status") + try require(try probe.getText(row: 0, column: "status") == "healthy") { + "post-fault health check failed" + } + let recovered = try await control.workerDiagnostics() + try require(recovered.epoch != stale.epoch) { + "fault recovery reused the stale epoch" + } + try require(recovered.extensionProcessIdentifier != stale.extensionProcessIdentifier) { + "crash recovery reused the dead worker PID" + } + try require(recovered.manifestDigest == stale.manifestDigest) { + "crash recovery changed the root manifest digest" + } + recoveredEpochs.append(recovered.epoch.description) + return recovered + } + + private static func expectOutcomeUnknown( + _ operation: () async throws -> Void + ) async throws { + do { + try await operation() + } catch let error as BrokerError { + guard case .outcomeUnknown = error else { throw error } + return + } + throw ExtendedFaultFailure.assertion( + "faulted data operation completed without OutcomeUnknown" + ) + } + + private static func expectControlInterruption( + _ operation: () async throws -> Void + ) async throws { + do { + try await operation() + } catch let error as BrokerError { + switch error { + case .workerInterrupted, .deadlineExceeded: + return + default: + throw error + } + } + throw ExtendedFaultFailure.assertion( + "fatal control fault unexpectedly returned success" + ) + } + + private static func waitForNativeExecution( + _ manager: IOSBrokerManager, + session: IOSBrokerSession + ) async throws { + for _ in 0..<100 { + let host = await manager.diagnostics() + if let activeRequestID = host.activeRequestID, + let worker = try? await session.workerDiagnostics(), + worker.activeRequestID == activeRequestID, + worker.nativeDispatchStarted + { + return + } + try await Task.sleep(for: .milliseconds(20)) + } + throw ExtendedFaultFailure.assertion( + "close/cancel/completion race never reached native execution" + ) + } + + /// Running cancellation deliberately returns PostgreSQL's normal backend + /// ErrorResponse + ReadyForQuery bytes before the broker sends Completed. + /// `execProtocolRaw` therefore succeeds at the transport layer; parse those + /// bytes so a real SQL completion cannot impersonate a canceled terminal. + private static func cancellationTerminal(from response: Data) throws -> String { + do { + _ = try parseOliphauntQueryResponse(response) + } catch OliphauntError.postgres(let postgresError) { + guard postgresError.sqlstate == "57014" else { + throw OliphauntError.postgres(postgresError) + } + return "postgresCanceledCompleted" + } + throw ExtendedFaultFailure.assertion( + "close/cancel/completion race returned a successful PostgreSQL result" + ) + } + + private static func cancelDuringCloseRace( + _ session: IOSBrokerSession + ) async throws -> String { + do { + try await session.cancel() + return "acknowledged" + } catch OliphauntError.databaseClosed { + return "databaseClosed" + } catch BrokerError.databaseClosed { + return "databaseClosed" + } + } + + private static func require( + _ condition: @autoclosure () throws -> Bool, + _ message: () -> String + ) throws { + guard try condition() else { + throw ExtendedFaultFailure.assertion(message()) + } + } + } + + private enum ExtendedFaultFailure: Error, CustomStringConvertible { + case assertion(String) + + var description: String { + switch self { + case .assertion(let message): "extended fault matrix failed: \(message)" + } + } + } + + private final class LockedFaultStreamCounter: @unchecked Sendable { + private let lock = NSLock() + private var bytes = 0 + private var chunks = 0 + + func consume(_ chunk: Data) { + lock.lock() + bytes += chunk.count + chunks += 1 + lock.unlock() + } + + var byteCount: Int { + lock.lock() + defer { lock.unlock() } + return bytes + } + + var chunkCount: Int { + lock.lock() + defer { lock.unlock() } + return chunks + } + } +#endif diff --git a/spikes/ios-native-broker/Host/HandshakeNegativeMatrix.swift b/spikes/ios-native-broker/Host/HandshakeNegativeMatrix.swift new file mode 100644 index 00000000..ad77f938 --- /dev/null +++ b/spikes/ios-native-broker/Host/HandshakeNegativeMatrix.swift @@ -0,0 +1,236 @@ +#if canImport(OliphauntIOSBroker) + import ExtensionFoundation + import Foundation + import OliphauntBrokerProtocol + import OliphauntBrokerXPC + import OliphauntIOSBroker + import XPC + + enum HandshakeNegativeMatrix { + static func run() async throws -> BrokerProbeResult { + let monitor = try await AppExtensionPoint.Monitor(appExtensionPoint: .oliphauntBroker) + guard + let identity = monitor.identities.first(where: { + $0.bundleIdentifier == BrokerFixtureBundleIdentifiers.extensionBundleIdentifier + }) + else { + throw HandshakeMatrixFailure.assertion("broker extension was not discovered") + } + let process = try await AppExtensionProcess( + configuration: .init(appExtensionIdentity: identity, onInterruption: {}) + ) + let session = try process.makeXPCSession() + session.setTargetQueue( + DispatchQueue(label: "dev.oliphaunt.brokerspike.negative-handshake") + ) + session.setCancellationHandler { _ in } + session.setIncomingMessageHandler { (_: XPCDictionary) in nil } + try session.activate() + defer { + session.cancel(reason: "negative handshake matrix complete") + process.invalidate() + } + + let capabilities: Set = [ + .processIsolated, + .crashRestartable, + .sameRootLogicalReopen, + .protocolRaw, + .protocolStream, + .queryCancel, + ] + let valid = BrokerHello( + expectedABI: 6, + startupConfigurationDigest: "ios-native-broker-spike-v2-restricted-role", + requestedCapabilities: capabilities + ) + var checks = Set() + + var incompatibleProtocol = valid + incompatibleProtocol.minimumProtocolVersion = 99 + incompatibleProtocol.maximumProtocolVersion = 99 + try await expectRejection( + incompatibleProtocol, + through: session, + expected: .incompatibleProtocol + ) + checks.insert("incompatibleProtocolRejected") + + var incompatibleABI = valid + incompatibleABI.expectedABI = 7 + try await expectRejection( + incompatibleABI, + through: session, + expected: .incompatibleABI + ) + checks.insert("incompatibleABIRejected") + + var runtimeMismatch = valid + runtimeMismatch.expectedRuntimeVersion = "not-the-linked-runtime" + try await expectRejection( + runtimeMismatch, + through: session, + expected: .runtimeMismatch + ) + checks.insert("runtimeMismatchRejected") + + var rootMismatch = valid + rootMismatch.rootID = "different-root" + try await expectRejection( + rootMismatch, + through: session, + expected: .rootMismatch + ) + checks.insert("rootMismatchRejected") + + var configurationMismatch = valid + configurationMismatch.startupConfigurationDigest = "different-configuration" + try await expectRejection( + configurationMismatch, + through: session, + expected: .invalidConfiguration + ) + checks.insert("startupConfigurationRejected") + + let pair = try ProbeSocketPair() + let owned = try IOSBrokerOwnedFileDescriptor( + takingOwnershipOf: pair.takeWorkerDescriptor() + ) + let message = try IOSBrokerXPC.makeHello(valid, dataChannel: owned) + owned.close() + let ready = try IOSBrokerXPC.decodeReady( + try await session.handshakeMatrixRequest(message) + ) + guard ready.extensionPID != Int32(ProcessInfo.processInfo.processIdentifier) else { + throw HandshakeMatrixFailure.assertion("valid retry was not process isolated") + } + try await pair.host.write( + try BrokerFrame( + protocolVersion: ready.selectedProtocolVersion, + frameType: .ping, + epoch: ready.epoch, + requestID: 0 + ).encoded() + ) + let pong = try await pair.host.readFrame(expectedEpoch: ready.epoch) + guard pong.header.frameType == .pong else { + throw HandshakeMatrixFailure.assertion("valid retry did not pass health check") + } + + let secondPair = try ProbeSocketPair() + let secondOwned = try IOSBrokerOwnedFileDescriptor( + takingOwnershipOf: secondPair.takeWorkerDescriptor() + ) + let secondReply = try await session.handshakeMatrixRequest( + try IOSBrokerXPC.makeHello(valid, dataChannel: secondOwned) + ) + secondOwned.close() + let secondError = try IOSBrokerXPC.decodeError(secondReply) + guard case .rejected(.invalidRequest(let reason)) = secondError, + reason.contains("data channel is already active") + else { + throw HandshakeMatrixFailure.assertion( + "second active data channel returned \(secondError)" + ) + } + checks.insert("secondActiveDataChannelRejected") + + try await pair.host.write( + try BrokerFrame( + protocolVersion: ready.selectedProtocolVersion, + frameType: .channelClose, + epoch: ready.epoch, + requestID: 0 + ).encoded() + ) + checks.insert("validHandshakeAfterRejections") + + return BrokerProbeResult( + hostPID: Int32(ProcessInfo.processInfo.processIdentifier), + workerPID: ready.extensionPID, + epoch: ready.epoch.description, + checks: checks.sorted(), + observations: [ + "rootManifestDigest": ready.rootManifestDigest, + "selectedProtocolVersion": String(ready.selectedProtocolVersion), + ] + ) + } + + private static func expectRejection( + _ hello: BrokerHello, + through session: XPCSession, + expected: ExpectedHandshakeRejection + ) async throws { + let pair = try ProbeSocketPair() + let owned = try IOSBrokerOwnedFileDescriptor( + takingOwnershipOf: pair.takeWorkerDescriptor() + ) + let message = try IOSBrokerXPC.makeHello(hello, dataChannel: owned) + owned.close() + let reply = try await session.handshakeMatrixRequest(message) + let error = try IOSBrokerXPC.decodeError(reply) + guard expected.matches(error) else { + throw HandshakeMatrixFailure.assertion( + "expected \(expected.rawValue), received \(error)" + ) + } + } + } + + private enum ExpectedHandshakeRejection: String { + case incompatibleProtocol + case incompatibleABI + case runtimeMismatch + case rootMismatch + case invalidConfiguration + + func matches(_ error: BrokerError) -> Bool { + switch (self, error) { + case (.incompatibleProtocol, .incompatibleProtocol), + (.incompatibleABI, .incompatibleABI), + (.runtimeMismatch, .runtimeMismatch), + (.rootMismatch, .rootMismatch), + (.invalidConfiguration, .invalidConfiguration): + true + default: + false + } + } + } + + private enum HandshakeMatrixFailure: Error, CustomStringConvertible { + case assertion(String) + + var description: String { + switch self { + case .assertion(let message): "negative handshake matrix failed: \(message)" + } + } + } + + private final class HandshakeMatrixReply: @unchecked Sendable { + let dictionary: XPCDictionary + + init(_ dictionary: XPCDictionary) { + self.dictionary = dictionary + } + } + + extension XPCSession { + fileprivate func handshakeMatrixRequest( + _ message: XPCDictionary + ) async throws -> XPCDictionary { + try await withCheckedThrowingContinuation { continuation in + send(message: message) { result in + switch result { + case .success(let reply): + continuation.resume(returning: HandshakeMatrixReply(reply)) + case .failure(let error): + continuation.resume(throwing: error) + } + } + }.dictionary + } + } +#endif diff --git a/spikes/ios-native-broker/Host/HangFaultMatrix.swift b/spikes/ios-native-broker/Host/HangFaultMatrix.swift new file mode 100644 index 00000000..5b961240 --- /dev/null +++ b/spikes/ios-native-broker/Host/HangFaultMatrix.swift @@ -0,0 +1,284 @@ +#if DEBUG && canImport(OliphauntIOSBroker) + import Foundation + import Oliphaunt + import OliphauntBrokerProtocol + import OliphauntIOSBroker + + /// Runs last, in its own launch. It measures whether the selected teardown + /// strategy obtains a fresh, Ready worker after deliberately wedging WorkerCore. + enum HangFaultMatrix { + private static let maximumRecoveryDelayMilliseconds = 60_000 + + static func run() async throws -> BrokerProbeResult { + let configuredRecoveryDelayMilliseconds = try recoveryDelayMilliseconds() + let recoveryStrategy = try hangRecoveryStrategy() + let manager = IOSBrokerManager() + let brokerConfiguration = IOSBrokerConfiguration( + expectedABI: 6, + startupConfigurationDigest: "ios-native-broker-spike-v2-restricted-role", + requestDeadline: .seconds(5), + extensionBundleIdentifier: + BrokerFixtureBundleIdentifiers.extensionBundleIdentifier, + controlReplyTimeout: .seconds(5), + cancellationGracePeriod: .seconds(1) + ) + let databaseConfiguration = OliphauntConfiguration( + mode: .nativeBroker, + durability: .safe, + runtimeFootprint: .smallMobile, + extensions: ["pg_trgm", "vector"] + ) + let engine = IOSBrokerEngine( + configuration: brokerConfiguration, + manager: manager + ) + let control = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: databaseConfiguration + ) + let database = try await OliphauntDatabase.open( + configuration: databaseConfiguration, + engine: engine + ) + let initialManager = await manager.diagnostics() + let initial = try await control.workerDiagnostics() + guard !initial.capabilities.hangRestartable else { + throw HangFaultFailure.assertion("worker overclaimed hang restartability") + } + + let faultAcknowledgementStartedAt = ContinuousClock.now + let deadlockFault: BrokerWorkerFault = + switch recoveryStrategy { + case "selfExitWatchdog": .deadlockWithFailStop + case "nativeFailStopWatchdog": .duringNativeExecution + default: .deadlock + } + try await control.injectFault(deadlockFault) + let faultAcknowledgementElapsedMilliseconds = elapsedMilliseconds( + faultAcknowledgementStartedAt.duration(to: .now) + ) + let postAcknowledgement = try await control.workerDiagnostics() + guard + postAcknowledgement.extensionProcessIdentifier + == initial.extensionProcessIdentifier, + postAcknowledgement.epoch == initial.epoch + else { + throw HangFaultFailure.assertion( + "worker identity changed before the armed deadlock was triggered" + ) + } + let postAcknowledgementManager = await manager.diagnostics() + guard + postAcknowledgementManager.interruptionCount + == initialManager.interruptionCount + else { + throw HangFaultFailure.assertion( + "fault acknowledgement interrupted the worker before the trigger query" + ) + } + + let heartbeat = await MainActor.run { MainActorHeartbeat() } + let hangTriggerStartedAt = ContinuousClock.now + let hangingSQL = + recoveryStrategy == "nativeFailStopWatchdog" + ? "SELECT pg_sleep(60), 'must-not-complete'::text AS status" + : "SELECT 'must-not-complete'::text AS status" + let hangingQuery = Task.detached { + try await database.query(hangingSQL) + } + try await Task.sleep(for: .milliseconds(250)) + await MainActor.run { + heartbeat.beat() + } + guard await MainActor.run(body: { heartbeat.wasObserved }) else { + throw HangFaultFailure.assertion("main actor did not remain responsive") + } + + var timeoutDescription = "" + do { + _ = try await hangingQuery.value + throw HangFaultFailure.assertion("armed WorkerCore deadlock did not trigger") + } catch let error as BrokerError { + switch error { + case .deadlineExceeded, .workerInterrupted, .outcomeUnknown: + timeoutDescription = error.description + default: + throw error + } + } + let hangTriggerElapsedMilliseconds = elapsedMilliseconds( + hangTriggerStartedAt.duration(to: .now) + ) + let interrupted = await manager.diagnostics() + guard + interrupted.interruptionCount + > postAcknowledgementManager.interruptionCount + else { + throw HangFaultFailure.assertion("hang timeout did not invalidate the epoch") + } + + let recoveryDelayStartedAt = ContinuousClock.now + if configuredRecoveryDelayMilliseconds > 0 { + try await Task.sleep( + for: .milliseconds(Int64(configuredRecoveryDelayMilliseconds)) + ) + } + let actualRecoveryDelayMilliseconds = elapsedMilliseconds( + recoveryDelayStartedAt.duration(to: .now) + ) + + var observations: [String: String] = [ + "actualHangRecoveryDelayMilliseconds": String(actualRecoveryDelayMilliseconds), + "configuredHangRecoveryDelayMilliseconds": String( + configuredRecoveryDelayMilliseconds), + "faultAcknowledged": "true", + "faultAcknowledgementElapsedMilliseconds": String( + faultAcknowledgementElapsedMilliseconds), + "hangTriggerElapsedMilliseconds": String(hangTriggerElapsedMilliseconds), + "hangTriggerOutcome": timeoutDescription, + "initialWorkerPID": String(initial.extensionProcessIdentifier), + "initialEpoch": initial.epoch.description, + "initialLaunchAttemptCount": String(initialManager.launchAttemptCount), + "interruptedLaunchAttemptCount": String(interrupted.launchAttemptCount), + "initialLaunchCount": String(initialManager.launchCount), + "interruptedLaunchCount": String(interrupted.launchCount), + "postAckEpoch": postAcknowledgement.epoch.description, + "postAckInterruptionCount": String( + postAcknowledgementManager.interruptionCount), + "postAckWorkerPID": String(postAcknowledgement.extensionProcessIdentifier), + "postAckWorkerResponsive": "true", + "timeout": timeoutDescription, + "hangRestartableCapability": "false", + "hangRecoveryStrategy": recoveryStrategy, + ] + var recoveredEpochs: [String] = [] + var freshProcessObtained = false + do { + let health = try await database.query("SELECT 'healthy'::text AS status") + guard try health.getText(row: 0, column: "status") == "healthy" else { + throw HangFaultFailure.assertion("fresh worker health check returned bad data") + } + let recovered = try await control.workerDiagnostics() + observations["recoveredWorkerPID"] = String( + recovered.extensionProcessIdentifier) + observations["recoveredEpoch"] = recovered.epoch.description + let freshEpoch = recovered.epoch != initial.epoch + let freshProcessIdentifier = + recovered.extensionProcessIdentifier != initial.extensionProcessIdentifier + if freshEpoch && freshProcessIdentifier { + freshProcessObtained = true + observations["freshProcessObtained"] = "true" + recoveredEpochs.append(recovered.epoch.description) + } else { + observations["freshProcessObtained"] = "false" + observations["recoveryFailure"] = + "replacement did not establish both a fresh epoch and a different PID" + } + } catch { + observations["freshProcessObtained"] = "false" + observations["recoveryFailure"] = String(describing: error) + } + let afterRecoveryAttempt = await manager.diagnostics() + guard afterRecoveryAttempt.launchAttemptCount > interrupted.launchAttemptCount else { + throw HangFaultFailure.assertion( + "post-hang operation did not attempt replacement process initialization" + ) + } + guard afterRecoveryAttempt.launchCount >= interrupted.launchCount else { + throw HangFaultFailure.assertion("post-hang successful launch count regressed") + } + if freshProcessObtained, + afterRecoveryAttempt.launchCount <= interrupted.launchCount + { + throw HangFaultFailure.assertion( + "fresh-process classification had no validated Ready launch" + ) + } + observations["postRecoveryLaunchAttemptCount"] = String( + afterRecoveryAttempt.launchAttemptCount + ) + observations["replacementLaunchAttemptDelta"] = String( + afterRecoveryAttempt.launchAttemptCount - interrupted.launchAttemptCount + ) + observations["postRecoveryLaunchCount"] = String( + afterRecoveryAttempt.launchCount + ) + observations["successfulLaunchCountDelta"] = String( + afterRecoveryAttempt.launchCount - interrupted.launchCount + ) + + try? await database.close() + try? await control.close() + return BrokerProbeResult( + hostPID: Int32(ProcessInfo.processInfo.processIdentifier), + workerPID: initial.extensionProcessIdentifier, + epoch: initial.epoch.description, + checks: [ + "hangCapabilityConservative", + "hangTimeout", + "mainActorResponsiveDuringHang", + "oldHangEpochInvalidated", + "replacementLaunchAttempted", + ], + recoveredEpochs: recoveredEpochs, + observations: observations + ) + } + + private static func recoveryDelayMilliseconds() throws -> Int { + let variable = "OLIPHAUNT_BROKER_HANG_RECOVERY_DELAY_MILLISECONDS" + guard let rawValue = ProcessInfo.processInfo.environment[variable] else { + return 0 + } + guard let value = Int(rawValue), + (0...maximumRecoveryDelayMilliseconds).contains(value) + else { + throw HangFaultFailure.assertion( + "\(variable) must be an integer from 0 through \(maximumRecoveryDelayMilliseconds)" + ) + } + return value + } + + private static func hangRecoveryStrategy() throws -> String { + let variable = "OLIPHAUNT_BROKER_HANG_RECOVERY_STRATEGY" + let value = ProcessInfo.processInfo.environment[variable] ?? "public" + let supported = [ + "public", + "selfExitWatchdog", + "nativeFailStopWatchdog", + ] + guard supported.contains(value) else { + throw HangFaultFailure.assertion( + "\(variable) must be one of \(supported.joined(separator: ", "))" + ) + } + return value + } + + private static func elapsedMilliseconds(_ duration: Duration) -> Int64 { + let components = duration.components + return components.seconds * 1_000 + + components.attoseconds / 1_000_000_000_000_000 + } + } + + @MainActor + private final class MainActorHeartbeat { + private(set) var wasObserved = false + + func beat() { + wasObserved = true + } + } + + private enum HangFaultFailure: Error, CustomStringConvertible { + case assertion(String) + + var description: String { + switch self { + case .assertion(let message): "hang fault matrix failed: \(message)" + } + } + } +#endif diff --git a/spikes/ios-native-broker/Host/NativeBrokerFixture.swift b/spikes/ios-native-broker/Host/NativeBrokerFixture.swift new file mode 100644 index 00000000..4bbf4c9c --- /dev/null +++ b/spikes/ios-native-broker/Host/NativeBrokerFixture.swift @@ -0,0 +1,1336 @@ +#if canImport(OliphauntIOSBroker) + import Darwin + import Foundation + import Oliphaunt + import OliphauntBrokerProtocol + import OliphauntIOSBroker + + enum NativeBrokerFixture { + private static let startupConfigurationDigest = + "ios-native-broker-spike-v2-restricted-role" + private static let selectedExtensions = ["pg_trgm", "vector"] + private static let restrictedDatabaseRole = "oliphaunt_broker" + + static func run() async throws -> BrokerProbeResult { + let hostPID = getpid() + let manager = IOSBrokerManager() + let brokerConfiguration = IOSBrokerConfiguration( + expectedABI: 6, + expectedRuntimeVersion: nil, + startupConfigurationDigest: startupConfigurationDigest, + maximumRequestBytes: OliphauntBrokerProtocol.defaultMaximumRequestBytes, + requestDeadline: .seconds(15), + extensionBundleIdentifier: BrokerFixtureBundleIdentifiers.extensionBundleIdentifier, + controlReplyTimeout: .seconds(20), + cancellationGracePeriod: .seconds(2) + ) + let databaseConfiguration = OliphauntConfiguration( + mode: .nativeBroker, + root: nil, + durability: .safe, + runtimeFootprint: .smallMobile, + extensions: selectedExtensions + ) + let engine = IOSBrokerEngine( + configuration: brokerConfiguration, + manager: manager + ) + + var checks = Set() + var observations: [String: String] = [:] + var diagnostics: [BrokerDiagnosticEvidence] = [] + var recoveredEpochs: [String] = [] + + let controlSession = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: databaseConfiguration + ) + let initialManagerDiagnostics = await manager.diagnostics() + let initialWorkerDiagnostics = try await controlSession.workerDiagnostics() + try require(initialManagerDiagnostics.state.epoch == initialWorkerDiagnostics.epoch) { + "manager and worker disagree about the initial epoch" + } + try require(initialWorkerDiagnostics.extensionProcessIdentifier != hostPID) { + "host and extension unexpectedly have the same PID" + } + try require( + initialWorkerDiagnostics.currentPhysFootprintBytes ?? 0 > 0 + && initialWorkerDiagnostics.currentResidentBytes ?? 0 > 0 + && initialWorkerDiagnostics.extensionEntryPreOpenPhysFootprintBytes ?? 0 > 0 + && initialWorkerDiagnostics.extensionEntryPreOpenResidentBytes ?? 0 > 0 + && initialWorkerDiagnostics.openedIdlePhysFootprintBytes ?? 0 > 0 + && initialWorkerDiagnostics.openedIdleResidentBytes ?? 0 > 0 + ) { + "worker diagnostics did not provide extensionEntryPreOpen/openedIdle memory samples" + } + let initialEpoch = initialWorkerDiagnostics.epoch.description + let initialWorkerPID = initialWorkerDiagnostics.extensionProcessIdentifier + diagnostics.append( + evidence( + phase: "openedIdle", + manager: initialManagerDiagnostics, + worker: initialWorkerDiagnostics + ) + ) + checks.formUnion([ + "extensionDiscovery", + "separatePID", + "xpcSession", + "fdTransfer", + "workerDiagnostics", + "openedIdleMemory", + ]) + + let database = try await OliphauntDatabase.open( + configuration: databaseConfiguration, + engine: engine + ) + let capabilities = try await database.capabilities() + try require(capabilities.processIsolated) { "broker did not report process isolation" } + try require(capabilities.crashRestartable) { "broker did not report crash recovery" } + try require(capabilities.sameRootLogicalReopen) { + "broker did not report same-root logical reopen" + } + try require(!capabilities.rootSwitchable && !capabilities.multiRoot) { + "broker overclaimed root capabilities" + } + try require(!capabilities.backupRestore && capabilities.connectionString == nil) { + "broker overclaimed backup or server capabilities" + } + checks.insert("capabilities") + + let roleState = try await database.query( + """ + SELECT + current_user AS current_role, + session_user AS session_role, + rolsuper::text AS is_superuser, + rolcreatedb::text AS can_create_database, + rolcreaterole::text AS can_create_role, + rolinherit::text AS inherits_roles, + rolcanlogin::text AS can_login, + rolreplication::text AS can_replicate, + rolbypassrls::text AS can_bypass_rls, + current_setting('is_superuser') AS is_superuser_setting, + pg_has_role(current_user, 'pg_checkpoint', 'MEMBER')::text + AS can_checkpoint, + pg_has_role(current_user, 'pg_database_owner', 'USAGE')::text + AS can_assume_database_owner, + ( + SELECT string_agg(candidate.rolname, ',' ORDER BY candidate.rolname) + FROM pg_roles candidate + WHERE pg_has_role(current_user, candidate.oid, 'USAGE') + ) AS effective_roles + FROM pg_roles + WHERE rolname = current_user + """ + ) + try require( + try roleState.getText(row: 0, column: "current_role") == restrictedDatabaseRole + && roleState.getText(row: 0, column: "session_role") == restrictedDatabaseRole + && roleState.getText(row: 0, column: "is_superuser") == "false" + && roleState.getText(row: 0, column: "can_create_database") == "false" + && roleState.getText(row: 0, column: "can_create_role") == "false" + && roleState.getText(row: 0, column: "inherits_roles") == "true" + && roleState.getText(row: 0, column: "can_login") == "true" + && roleState.getText(row: 0, column: "can_replicate") == "false" + && roleState.getText(row: 0, column: "can_bypass_rls") == "false" + && roleState.getText(row: 0, column: "is_superuser_setting") == "off" + && roleState.getText(row: 0, column: "can_checkpoint") == "true" + && roleState.getText(row: 0, column: "can_assume_database_owner") == "false" + && roleState.getText(row: 0, column: "effective_roles") + == "oliphaunt_broker,pg_checkpoint" + ) { + "host SQL was not confined to the restricted broker database role" + } + + let databaseOwner = try await database.query( + """ + SELECT pg_get_userbyid(datdba) AS owner + FROM pg_database + WHERE datname = current_database() + """ + ) + let extensionOwners = try await database.query( + """ + SELECT string_agg( + extname || ':' || pg_get_userbyid(extowner), + ',' ORDER BY extname + ) AS owners + FROM pg_extension + WHERE extname = ANY (ARRAY['pg_trgm', 'vector']::name[]) + """ + ) + let brokerSchemaOwner = try await database.query( + """ + SELECT pg_get_userbyid(nspowner) AS owner + FROM pg_namespace + WHERE nspname = 'oliphaunt_broker' + """ + ) + try require( + try databaseOwner.getText(row: 0, column: "owner") == "postgres" + && extensionOwners.getText(row: 0, column: "owners") + == "pg_trgm:postgres,vector:postgres" + && brokerSchemaOwner.getText(row: 0, column: "owner") == restrictedDatabaseRole + ) { + "broker role unexpectedly owned the database, selected extensions, or wrong schema" + } + + let dataDirectorySQLState = try await expectPathAccessDenied("dataDirectory") { + _ = try await database.query("SHOW data_directory") + } + let parameterizedDataDirectorySQLState = try await expectPathAccessDenied( + "parameterizedDataDirectory" + ) { + _ = try await database.query( + "SELECT current_setting($1)", + parameters: [.text("data_directory")] + ) + } + let serverFileSQLState = try await expectPathAccessDenied("serverFile") { + _ = try await database.query("SELECT pg_read_file('PG_VERSION')") + } + let bootstrapEscalationSQLState = try await expectPathAccessDenied( + "bootstrapEscalation" + ) { + _ = try await database.query("SET ROLE postgres") + } + let sessionAuthorizationEscalationSQLState = try await expectPathAccessDenied( + "sessionAuthorizationEscalation" + ) { + _ = try await database.query("SET SESSION AUTHORIZATION postgres") + } + let databaseOwnerEscalationSQLState = try await expectPathAccessDenied( + "databaseOwnerEscalation" + ) { + _ = try await database.query("SET ROLE pg_database_owner") + } + let relationPathSQLState = try await expectPathAccessDenied("relationPath") { + _ = try await database.query( + "SELECT pg_relation_filepath('pg_catalog.pg_class'::regclass)" + ) + } + let tablespacePathSQLState = try await expectPathAccessDenied("tablespacePath") { + _ = try await database.query( + "SELECT pg_tablespace_location(oid) FROM pg_tablespace LIMIT 1" + ) + } + let listDirectorySQLState = try await expectPathAccessDenied("listDirectory") { + _ = try await database.query("SELECT pg_ls_dir('.')") + } + let statFileSQLState = try await expectPathAccessDenied("statFile") { + _ = try await database.query("SELECT pg_stat_file('PG_VERSION')") + } + let callerPathProbe = "/private/oliphaunt-denied" + let largeObjectImportSQLState = try await expectPathAccessDenied( + "largeObjectImport", + allowingEchoOf: callerPathProbe + ) { + _ = try await database.query("SELECT lo_import('/private/oliphaunt-denied')") + } + let externalCopySQLState = try await expectPathAccessDenied( + "externalCopy", + allowingEchoOf: callerPathProbe + ) { + _ = try await database.query( + "COPY (SELECT 1) TO '/private/oliphaunt-denied'" + ) + } + _ = try await database.query( + "CREATE TEMP TABLE oliphaunt_private_copy_probe(value text)" + ) + let externalCopyFromSQLState = try await expectPathAccessDenied( + "externalCopyFrom", + allowingEchoOf: callerPathProbe + ) { + _ = try await database.query( + "COPY oliphaunt_private_copy_probe FROM '/private/oliphaunt-denied'" + ) + } + let alterSystemSQLState = try await expectPathAccessDenied("alterSystem") { + _ = try await database.query("ALTER SYSTEM SET application_name = 'denied'") + } + let createRoleSQLState = try await expectPathAccessDenied("createRole") { + _ = try await database.query("CREATE ROLE oliphaunt_broker_escalation_probe") + } + let selfSuperuserEscalationSQLState = try await expectPathAccessDenied( + "selfSuperuserEscalation" + ) { + _ = try await database.query("ALTER ROLE oliphaunt_broker SUPERUSER") + } + let grantFileRoleSQLState = try await expectPathAccessDenied("grantFileRole") { + _ = try await database.query( + "GRANT pg_read_server_files TO oliphaunt_broker" + ) + } + let dropSelectedExtensionSQLState = try await expectPathAccessDenied( + "dropSelectedExtension" + ) { + _ = try await database.query("DROP EXTENSION pg_trgm") + } + let createTablespaceSQLState = try await expectPathAccessDenied( + "createTablespace", + allowingEchoOf: callerPathProbe + ) { + _ = try await database.query( + "CREATE TABLESPACE oliphaunt_denied LOCATION '/private/oliphaunt-denied'" + ) + } + let createNativeFunctionSQLState = try await expectPathAccessDenied( + "createNativeFunction", + allowingEchoOf: callerPathProbe + ) { + _ = try await database.query( + """ + CREATE FUNCTION oliphaunt_broker.oliphaunt_denied_native() + RETURNS integer + AS '/private/oliphaunt-denied', 'oliphaunt_denied' + LANGUAGE C + """ + ) + } + let loadLibrarySQLState = try await expectPathAccessDenied( + "loadLibrary", + allowingEchoOf: callerPathProbe + ) { + _ = try await database.query("LOAD '/private/oliphaunt-denied'") + } + let nonDefaultTablespaces = try await database.query( + """ + SELECT count(*)::text AS count + FROM pg_tablespace + WHERE spcname NOT IN ('pg_default', 'pg_global') + """ + ) + try require(try nonDefaultTablespaces.getText(row: 0, column: "count") == "0") { + "broker root contained an unsupported non-default tablespace" + } + let visibleSettings = try await database.query( + "SELECT setting FROM pg_settings WHERE name = 'data_directory'" + ) + try require(visibleSettings.rowCount == 0) { + "pg_settings exposed data_directory to the host-visible broker role" + } + let restrictedFunctionPrivileges = try await database.query( + """ + SELECT count(*)::text AS count + FROM unnest(ARRAY[ + 'pg_catalog.pg_backup_start(text,boolean)', + 'pg_catalog.pg_backup_stop(boolean)', + 'pg_catalog.pg_current_logfile()', + 'pg_catalog.pg_current_logfile(text)', + 'pg_catalog.lo_import(text)', + 'pg_catalog.lo_import(text,oid)', + 'pg_catalog.lo_export(oid,text)', + 'pg_catalog.pg_ls_logdir()', + 'pg_catalog.pg_ls_waldir()', + 'pg_catalog.pg_ls_archive_statusdir()', + 'pg_catalog.pg_ls_summariesdir()', + 'pg_catalog.pg_ls_tmpdir()', + 'pg_catalog.pg_ls_tmpdir(oid)', + 'pg_catalog.pg_read_file(text)', + 'pg_catalog.pg_read_file(text,boolean)', + 'pg_catalog.pg_read_file(text,bigint,bigint)', + 'pg_catalog.pg_read_file(text,bigint,bigint,boolean)', + 'pg_catalog.pg_read_binary_file(text)', + 'pg_catalog.pg_read_binary_file(text,boolean)', + 'pg_catalog.pg_read_binary_file(text,bigint,bigint)', + 'pg_catalog.pg_read_binary_file(text,bigint,bigint,boolean)', + 'pg_catalog.pg_stat_file(text)', + 'pg_catalog.pg_stat_file(text,boolean)', + 'pg_catalog.pg_ls_dir(text)', + 'pg_catalog.pg_ls_dir(text,boolean,boolean)', + 'pg_catalog.pg_show_all_file_settings()', + 'pg_catalog.pg_hba_file_rules()', + 'pg_catalog.pg_ident_file_mappings()', + 'pg_catalog.pg_config()' + ]::text[]) restricted(signature) + WHERE has_function_privilege(current_user, signature, 'EXECUTE') + """ + ) + let restrictedViewPrivileges = try await database.query( + """ + SELECT count(*)::text AS count + FROM unnest(ARRAY[ + 'pg_catalog.pg_file_settings', + 'pg_catalog.pg_hba_file_rules', + 'pg_catalog.pg_ident_file_mappings', + 'pg_catalog.pg_config' + ]::text[]) restricted(name) + WHERE has_table_privilege(current_user, name, 'SELECT') + """ + ) + let visibleSettingPathEvidence = try await database.query( + """ + SELECT + count(*) FILTER ( + WHERE sourcefile IS NOT NULL OR sourceline IS NOT NULL + )::text AS source_path_rows, + count(*) FILTER ( + WHERE setting LIKE '%/private/var/mobile/%' + OR setting LIKE '%/var/mobile/%' + OR setting LIKE '%/Users/%' + OR lower(setting) LIKE '%application support%' + OR lower(setting) LIKE '%runtime-cache%' + OR lower(setting) LIKE '%pgdata%' + )::text AS private_path_rows + FROM pg_settings + """ + ) + try require( + try restrictedFunctionPrivileges.getText(row: 0, column: "count") == "0" + && restrictedViewPrivileges.getText(row: 0, column: "count") == "0" + && visibleSettingPathEvidence.getText(row: 0, column: "source_path_rows") == "0" + && visibleSettingPathEvidence.getText(row: 0, column: "private_path_rows") + == "0" + ) { + "catalog privilege or GUC visibility exposed a private filesystem path" + } + _ = try await database.query("RESET ROLE") + _ = try await database.query("RESET SESSION AUTHORIZATION") + let afterReset = try await database.query( + """ + SELECT + current_user AS current_role, + session_user AS session_role, + current_setting('is_superuser') AS is_superuser + """ + ) + try require( + try afterReset.getText(row: 0, column: "current_role") == restrictedDatabaseRole + && afterReset.getText(row: 0, column: "session_role") == restrictedDatabaseRole + && afterReset.getText(row: 0, column: "is_superuser") == "off" + ) { + "RESET ROLE or RESET SESSION AUTHORIZATION escaped the restricted broker role" + } + let afterResetDataDirectorySQLState = try await expectPathAccessDenied( + "afterResetDataDirectory" + ) { + _ = try await database.query("SHOW data_directory") + } + _ = try await database.query("DISCARD ALL") + let afterDiscard = try await database.query( + """ + SELECT + current_user AS current_role, + session_user AS session_role, + current_setting('is_superuser') AS is_superuser, + current_schemas(false)::text AS schemas + """ + ) + try require( + try afterDiscard.getText(row: 0, column: "current_role") == restrictedDatabaseRole + && afterDiscard.getText(row: 0, column: "session_role") + == restrictedDatabaseRole + && afterDiscard.getText(row: 0, column: "is_superuser") == "off" + && afterDiscard.getText(row: 0, column: "schemas") + == "{oliphaunt_broker,public}" + ) { + "DISCARD ALL escaped the restricted role or its broker-owned schema" + } + let afterDiscardDataDirectorySQLState = try await expectPathAccessDenied( + "afterDiscardDataDirectory" + ) { + _ = try await database.query("SHOW data_directory") + } + + _ = try await database.query( + """ + DROP TEXT SEARCH DICTIONARY IF EXISTS + oliphaunt_broker.private_path_error_probe; + CREATE TEXT SEARCH DICTIONARY + oliphaunt_broker.private_path_error_probe ( + TEMPLATE = pg_catalog.simple, + STOPWORDS = 'oliphaunt_missing_private_path_probe' + ) + """ + ) + let sanitizedBackendErrorSQLState = try await expectSanitizedPostgresError { + _ = try await database.query( + """ + SELECT ts_lexize( + 'oliphaunt_broker.private_path_error_probe', + 'probe' + ) + """ + ) + } + _ = try? await database.query( + "DROP TEXT SEARCH DICTIONARY oliphaunt_broker.private_path_error_probe" + ) + let afterSanitizedError = try await database.query( + "SELECT 'alive'::text AS status" + ) + try require(try afterSanitizedError.getText(row: 0, column: "status") == "alive") { + "backend ErrorResponse sanitization did not preserve session liveness" + } + observations["restrictedDatabaseRole"] = restrictedDatabaseRole + observations["databaseOwner"] = + try databaseOwner.getText(row: 0, column: "owner") ?? "" + observations["selectedExtensionOwners"] = + try extensionOwners.getText(row: 0, column: "owners") ?? "" + observations["brokerSchemaOwner"] = + try brokerSchemaOwner.getText(row: 0, column: "owner") ?? "" + observations["dataDirectorySQLState"] = dataDirectorySQLState + observations["parameterizedDataDirectorySQLState"] = + parameterizedDataDirectorySQLState + observations["serverFileSQLState"] = serverFileSQLState + observations["bootstrapEscalationSQLState"] = bootstrapEscalationSQLState + observations["sessionAuthorizationEscalationSQLState"] = + sessionAuthorizationEscalationSQLState + observations["databaseOwnerEscalationSQLState"] = databaseOwnerEscalationSQLState + observations["relationPathSQLState"] = relationPathSQLState + observations["tablespacePathSQLState"] = tablespacePathSQLState + observations["listDirectorySQLState"] = listDirectorySQLState + observations["statFileSQLState"] = statFileSQLState + observations["largeObjectImportSQLState"] = largeObjectImportSQLState + observations["externalCopySQLState"] = externalCopySQLState + observations["externalCopyFromSQLState"] = externalCopyFromSQLState + observations["alterSystemSQLState"] = alterSystemSQLState + observations["createRoleSQLState"] = createRoleSQLState + observations["selfSuperuserEscalationSQLState"] = selfSuperuserEscalationSQLState + observations["grantFileRoleSQLState"] = grantFileRoleSQLState + observations["dropSelectedExtensionSQLState"] = dropSelectedExtensionSQLState + observations["createTablespaceSQLState"] = createTablespaceSQLState + observations["createNativeFunctionSQLState"] = createNativeFunctionSQLState + observations["loadLibrarySQLState"] = loadLibrarySQLState + observations["nonDefaultTablespaceCount"] = + try nonDefaultTablespaces.getText(row: 0, column: "count") ?? "" + observations["afterResetDataDirectorySQLState"] = afterResetDataDirectorySQLState + observations["afterDiscardDataDirectorySQLState"] = + afterDiscardDataDirectorySQLState + observations["afterDiscardSearchPath"] = + try afterDiscard.getText(row: 0, column: "schemas") ?? "" + observations["sanitizedBackendErrorSQLState"] = sanitizedBackendErrorSQLState + observations["pgSettingsDataDirectoryRows"] = String(visibleSettings.rowCount) + observations["restrictedFunctionExecuteCount"] = + try restrictedFunctionPrivileges.getText(row: 0, column: "count") ?? "" + observations["restrictedViewSelectCount"] = + try restrictedViewPrivileges.getText(row: 0, column: "count") ?? "" + observations["pgSettingsSourcePathRows"] = + try visibleSettingPathEvidence.getText(row: 0, column: "source_path_rows") ?? "" + observations["visiblePrivatePathSettingRows"] = + try visibleSettingPathEvidence.getText(row: 0, column: "private_path_rows") ?? "" + checks.insert("pgdataPathConfidentiality") + + let currentLaunchMarker = UUID().uuidString.lowercased() + _ = try await database.query( + """ + CREATE TABLE IF NOT EXISTS broker_spike_launch_history( + marker text PRIMARY KEY + ) + """ + ) + let priorLaunchHistory = try await database.query( + """ + SELECT coalesce(string_agg(marker, ',' ORDER BY marker), '') AS markers + FROM broker_spike_launch_history + """ + ) + observations["priorLaunchMarkers"] = + try priorLaunchHistory.getText(row: 0, column: "markers") ?? "" + observations["currentLaunchMarker"] = currentLaunchMarker + _ = try await database.query( + "INSERT INTO broker_spike_launch_history(marker) VALUES ($1)", + parameters: [.text(currentLaunchMarker)] + ) + + let select = try await database.query("SELECT 42::text AS value") + try require(try select.getText(row: 0, column: "value") == "42") { + "real SELECT returned the wrong value" + } + checks.insert("realSelect") + + _ = try await database.query( + """ + CREATE TABLE IF NOT EXISTS broker_spike_events( + operation_id text PRIMARY KEY, + payload text NOT NULL + ); + TRUNCATE broker_spike_events; + """ + ) + let inserted = try await database.query( + """ + INSERT INTO broker_spike_events(operation_id, payload) + VALUES ($1, $2) + RETURNING payload + """, + parameters: [.text("parameterized"), .text("bound-value")] + ) + try require(try inserted.getText(row: 0, column: "payload") == "bound-value") { + "parameterized write returned the wrong payload" + } + checks.formUnion(["ddl", "write", "parameterizedQuery"]) + + var expectedSQLState: String? + do { + _ = try await database.query("SELECT * FROM broker_spike_missing_relation") + } catch OliphauntError.postgres(let postgresError) { + expectedSQLState = postgresError.sqlstate + } + try require(expectedSQLState == "42P01") { + "missing-relation error did not preserve SQLSTATE 42P01" + } + let afterError = try await database.query("SELECT 'alive'::text AS status") + try require(try afterError.getText(row: 0, column: "status") == "alive") { + "session did not recover after PostgreSQL ErrorResponse" + } + checks.insert("postgresErrorRecovery") + + _ = try await database.query("CREATE EXTENSION IF NOT EXISTS vector") + _ = try await database.query("CREATE EXTENSION IF NOT EXISTS pg_trgm") + let vector = try await database.query( + """ + SELECT round( + ('[1,2,3]'::vector <-> '[1,2,4]'::vector)::numeric, + 2 + )::text AS distance + """ + ) + try require(try vector.getText(row: 0, column: "distance") == "1.00") { + "vector extension returned an unexpected distance" + } + let trigram = try await database.query( + "SELECT (similarity('postgres', 'postgress') > 0.5)::text AS similar" + ) + try require(try trigram.getText(row: 0, column: "similar") == "true") { + "pg_trgm similarity function was not usable" + } + checks.formUnion(["vectorExtension", "pgTrgmExtension"]) + + let largeParameter = String(repeating: "x", count: 300 * 1024) + let largeRequest = try await database.query( + "SELECT length($1::text)::text AS length", + parameters: [.text(largeParameter)] + ) + try require( + try largeRequest.getText(row: 0, column: "length") == String(largeParameter.count) + ) { + "multi-frame request assembly changed the bound parameter" + } + checks.formUnion(["fragmentedFrame", "boundedRequestAssembly", "multiFrameRequest"]) + + let streamCounter = BrokerStreamCounter() + let streamingStarted = BrokerFixtureSignal() + let streaming = Task { + try await database.execProtocolStream( + try OliphauntProtocol.simpleQuery( + "SELECT repeat('s', 1024) FROM generate_series(1, 2048)" + ) + ) { chunk in + streamCounter.consume(chunk) + if streamingStarted.signal() { + Thread.sleep(forTimeInterval: 0.5) + } + } + } + await streamingStarted.wait() + let streamingWorker = try await controlSession.workerDiagnostics() + try require( + streamingWorker.activeRequestID != nil + && streamingWorker.nativeDispatchStarted + ) { + "streaming diagnostics did not observe active native dispatch" + } + diagnostics.append( + evidence( + phase: "streaming", + manager: await manager.diagnostics(), + worker: streamingWorker + ) + ) + try await streaming.value + try require(streamCounter.byteCount > 2 * 1024 * 1024) { + "streaming response did not deliver the expected byte volume" + } + observations["streamedBytes"] = String(streamCounter.byteCount) + observations["streamedChunks"] = String(streamCounter.chunkCount) + checks.insert("streamingResponse") + + let secondDatabase = try await OliphauntDatabase.open( + configuration: databaseConfiguration, + engine: engine + ) + let simultaneous = await manager.diagnostics() + try require(simultaneous.logicalHandleCount == 3) { + "simultaneous logical opens were not reference counted" + } + diagnostics.append( + evidence( + phase: "simultaneousHandles", + manager: simultaneous, + worker: try await controlSession.workerDiagnostics() + ) + ) + checks.formUnion(["simultaneousHandles", "referenceCounting"]) + + _ = try await database.query( + """ + CREATE TABLE IF NOT EXISTS broker_spike_fifo_events( + position bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + caller text NOT NULL + ); + TRUNCATE broker_spike_fifo_events RESTART IDENTITY; + """ + ) + let activeCaller = Task { + try await database.query( + """ + INSERT INTO broker_spike_fifo_events(caller) + SELECT 'active' + FROM (SELECT pg_sleep(2)) AS delay + RETURNING position::text AS position, caller + """ + ) + } + let fifoActiveWorker = try await waitForActiveNativeRequest( + session: controlSession, + timeout: .seconds(5) + ) + let fifoActiveManager = await manager.diagnostics() + let fifoActiveRequestID = try requireValue( + fifoActiveWorker.activeRequestID, + "FIFO active worker diagnostics omitted the request ID" + ) + try require( + fifoActiveManager.activeRequestID == fifoActiveRequestID + && fifoActiveManager.queuedOperationCount == 0 + ) { + "FIFO active manager diagnostics did not show exactly one unqueued request" + } + diagnostics.append( + evidence( + phase: "fifoActive", + manager: fifoActiveManager, + worker: fifoActiveWorker + ) + ) + + let queuedCaller = Task { + try await secondDatabase.query( + """ + INSERT INTO broker_spike_fifo_events(caller) + VALUES ('queued') + RETURNING position::text AS position, caller + """ + ) + } + let fifoQueuedManager = try await waitForQueuedOperation( + manager: manager, + timeout: .seconds(5) + ) + let fifoQueuedWorker = try await controlSession.workerDiagnostics() + try require( + fifoQueuedManager.activeRequestID == fifoActiveRequestID + && fifoQueuedManager.queuedOperationCount == 1 + && fifoQueuedWorker.activeRequestID == fifoActiveRequestID + && fifoQueuedWorker.nativeDispatchStarted + ) { + "FIFO diagnostics did not preserve one active native request and one queued operation" + } + diagnostics.append( + evidence( + phase: "fifoQueued", + manager: fifoQueuedManager, + worker: fifoQueuedWorker + ) + ) + + let (activeResult, queuedResult) = try await ( + activeCaller.value, + queuedCaller.value + ) + try require( + try activeResult.getText(row: 0, column: "caller") == "active" + && activeResult.getText(row: 0, column: "position") == "1" + ) { + "FIFO active caller did not complete first" + } + try require( + try queuedResult.getText(row: 0, column: "caller") == "queued" + && queuedResult.getText(row: 0, column: "position") == "2" + ) { + "FIFO queued caller did not complete second" + } + let fifoOrderResult = try await database.query( + """ + SELECT string_agg(caller, ',' ORDER BY position) AS observed_order + FROM broker_spike_fifo_events + """ + ) + let fifoObservedOrder = try requireValue( + fifoOrderResult.getText(row: 0, column: "observed_order"), + "FIFO ordering query returned NULL" + ) + try require(fifoObservedOrder == "active,queued") { + "logical operations were not serialized in FIFO order: \(fifoObservedOrder)" + } + let fifoDrainedManager = await manager.diagnostics() + try require( + fifoDrainedManager.activeRequestID == nil + && fifoDrainedManager.queuedOperationCount == 0 + ) { + "FIFO manager did not drain active and queued operations" + } + observations["fifoActiveRequestID"] = String(fifoActiveRequestID.rawValue) + observations["fifoQueuedOperationCount"] = String( + fifoQueuedManager.queuedOperationCount + ) + observations["fifoObservedOrder"] = fifoObservedOrder + diagnostics.append( + evidence( + phase: "fifoDrained", + manager: fifoDrainedManager, + worker: try await controlSession.workerDiagnostics() + ) + ) + checks.insert("fifoSerialization") + + let transactionStarted = BrokerFixtureLatch() + let rollingBack = Task { + do { + try await database.transaction { transaction -> Void in + _ = try await transaction.query( + """ + INSERT INTO broker_spike_events(operation_id, payload) + VALUES ('rolled-back-owner', 'must-not-persist') + """ + ) + await transactionStarted.signal() + try await Task.sleep(for: .milliseconds(250)) + throw PlannedFixtureRollback() + } + throw BrokerFixtureFailure.assertion( + "planned transaction unexpectedly committed") + } catch is PlannedFixtureRollback { + return + } + } + await transactionStarted.wait() + let independentWrite = Task { + _ = try await secondDatabase.query( + """ + INSERT INTO broker_spike_events(operation_id, payload) + VALUES ('independent-handle', 'must-persist') + """ + ) + } + try await rollingBack.value + try await independentWrite.value + let transactionCounts = try await secondDatabase.query( + """ + SELECT + count(*) FILTER (WHERE operation_id = 'rolled-back-owner')::text AS rolled_back, + count(*) FILTER (WHERE operation_id = 'independent-handle')::text AS independent + FROM broker_spike_events + """ + ) + try require( + try transactionCounts.getText(row: 0, column: "rolled_back") == "0" + && transactionCounts.getText(row: 0, column: "independent") == "1" + ) { + "logical handles interleaved across a physical transaction" + } + checks.insert("transactionHandlePinning") + + let sleepingQuery = Task { + try await secondDatabase.query("SELECT pg_sleep(10)") + } + try await Task.sleep(for: .milliseconds(150)) + let executingWorker = try await controlSession.workerDiagnostics() + try require( + executingWorker.activeRequestID != nil + && executingWorker.nativeDispatchStarted + ) { + "executing diagnostics did not observe active native dispatch" + } + diagnostics.append( + evidence( + phase: "executing", + manager: await manager.diagnostics(), + worker: executingWorker + ) + ) + try await secondDatabase.cancel() + var cancellationSQLState: String? + do { + _ = try await sleepingQuery.value + } catch OliphauntError.postgres(let postgresError) { + cancellationSQLState = postgresError.sqlstate + } + try require(cancellationSQLState == "57014") { + "cancellation did not produce PostgreSQL SQLSTATE 57014" + } + let afterCancel = try await secondDatabase.query("SELECT 'live'::text AS status") + try require(try afterCancel.getText(row: 0, column: "status") == "live") { + "worker was not live after cancellation" + } + checks.formUnion(["cancellation", "postCancelLiveness"]) + + try await controlSession.checkpoint() + diagnostics.append( + evidence( + phase: "afterCheckpoint", + manager: await manager.diagnostics(), + worker: try await controlSession.workerDiagnostics() + ) + ) + let prepared = try await controlSession.prepareForBackground(timeout: .seconds(5)) + try require(prepared.checkpointed) { + "idle background preparation did not checkpoint" + } + try await controlSession.resumeFromBackground() + let resumed = try await rawQuery(controlSession, "SELECT 'resumed'::text AS status") + try require(try resumed.getText(row: 0, column: "status") == "resumed") { + "background resume health check did not preserve liveness" + } + checks.formUnion(["checkpointControl", "backgroundLifecycle"]) + + try await database.close() + let afterFirstClose = try await secondDatabase.query( + "SELECT count(*)::text AS count FROM broker_spike_events" + ) + try require(try afterFirstClose.getText(row: 0, column: "count") != nil) { + "closing one logical handle detached the shared physical session" + } + try await secondDatabase.close() + try await controlSession.close() + let detached = await manager.diagnostics() + try require(detached.logicalHandleCount == 0 && detached.state == .idle) { + "last logical close did not detach the manager" + } + diagnostics.append(evidence(phase: "logicalDetach", manager: detached, worker: nil)) + + let reopenedControl = try await manager.open( + configuration: brokerConfiguration, + databaseConfiguration: databaseConfiguration + ) + let reopenedWorker = try await reopenedControl.workerDiagnostics() + try require(reopenedWorker.epoch.description != initialEpoch) { + "same-root reopen reused a stale worker epoch" + } + recoveredEpochs.append(reopenedWorker.epoch.description) + let reopenedDatabase = try await OliphauntDatabase.open( + configuration: databaseConfiguration, + engine: engine + ) + let persisted = try await reopenedDatabase.query( + """ + SELECT count(*)::text AS count + FROM broker_spike_events + WHERE operation_id = 'parameterized' + """ + ) + try require(try persisted.getText(row: 0, column: "count") == "1") { + "same-root reopen lost committed data" + } + diagnostics.append( + evidence( + phase: "sameRootReopen", + manager: await manager.diagnostics(), + worker: reopenedWorker + ) + ) + checks.insert("sameRootReopen") + + #if DEBUG + let postCommitEpoch = reopenedWorker.epoch + try await reopenedControl.injectFault(.afterNativeSuccessBeforeCompleted) + try await expectOutcomeUnknown { + _ = try await reopenedControl.execProtocolRaw( + try OliphauntProtocol.simpleQuery( + """ + BEGIN; + INSERT INTO broker_spike_events(operation_id, payload) + VALUES ('post-commit-ambiguity', 'committed-once'); + COMMIT; + """ + ) + ) + } + let postCommitCount = try await rawQuery( + reopenedControl, + """ + SELECT count(*)::text AS count + FROM broker_spike_events + WHERE operation_id = 'post-commit-ambiguity' + """ + ) + try require(try postCommitCount.getText(row: 0, column: "count") == "1") { + "commit-ambiguity marker was absent or replayed" + } + let afterPostCommitRecovery = try await reopenedControl.workerDiagnostics() + try require(afterPostCommitRecovery.epoch != postCommitEpoch) { + "post-commit crash did not establish a new epoch" + } + recoveredEpochs.append(afterPostCommitRecovery.epoch.description) + diagnostics.append( + evidence( + phase: "postCommitRecovery", + manager: await manager.diagnostics(), + worker: afterPostCommitRecovery + ) + ) + checks.formUnion(["outcomeUnknown", "postCommitAmbiguity", "crashRecovery"]) + + let preCommitEpoch = afterPostCommitRecovery.epoch + try await reopenedControl.injectFault(.duringNativeExecution) + try await expectOutcomeUnknown { + _ = try await reopenedControl.execProtocolRaw( + try OliphauntProtocol.simpleQuery( + """ + BEGIN; + INSERT INTO broker_spike_events(operation_id, payload) + VALUES ('pre-commit-crash', 'must-roll-back'); + SELECT pg_sleep(5); + COMMIT; + """ + ) + ) + } + let preCommitCount = try await rawQuery( + reopenedControl, + """ + SELECT count(*)::text AS count + FROM broker_spike_events + WHERE operation_id = 'pre-commit-crash' + """ + ) + try require(try preCommitCount.getText(row: 0, column: "count") == "0") { + "uncommitted marker survived worker crash and WAL recovery" + } + let afterPreCommitRecovery = try await reopenedControl.workerDiagnostics() + try require(afterPreCommitRecovery.epoch != preCommitEpoch) { + "pre-commit crash did not establish a new epoch" + } + recoveredEpochs.append(afterPreCommitRecovery.epoch.description) + diagnostics.append( + evidence( + phase: "preCommitRecovery", + manager: await manager.diagnostics(), + worker: afterPreCommitRecovery + ) + ) + checks.formUnion(["preCommitRollbackRecovery", "noAutomaticReplay"]) + #else + observations["faultInjection"] = "not compiled in this configuration" + #endif + + let finalDiagnostics = await manager.diagnostics() + observations["launchCount"] = String(finalDiagnostics.launchCount) + observations["interruptionCount"] = String(finalDiagnostics.interruptionCount) + observations["initialWorkerPID"] = String(initialWorkerPID) + + if ProcessInfo.processInfo.environment["OLIPHAUNT_BROKER_DEVICE_LAUNCH_INDEX"] == "2" { + _ = try await reopenedDatabase.query("DELETE FROM broker_spike_launch_history") + } else { + _ = try await reopenedDatabase.query( + "DELETE FROM broker_spike_launch_history WHERE marker <> $1", + parameters: [.text(currentLaunchMarker)] + ) + } + + try await reopenedDatabase.close() + try await reopenedControl.close() + + return BrokerProbeResult( + hostPID: hostPID, + workerPID: initialWorkerPID, + epoch: initialEpoch, + checks: checks.sorted(), + recoveredEpochs: recoveredEpochs, + diagnostics: diagnostics, + observations: observations + ) + } + + private static func rawQuery( + _ session: IOSBrokerSession, + _ sql: String + ) async throws -> OliphauntQueryResult { + try await parseOliphauntQueryResponse( + session.execProtocolRaw(try OliphauntProtocol.simpleQuery(sql)) + ) + } + + private static func expectPathAccessDenied( + _ label: String, + allowingEchoOf callerSuppliedPath: String? = nil, + _ operation: () async throws -> Void + ) async throws -> String { + do { + try await operation() + } catch OliphauntError.postgres(let postgresError) { + try require(postgresError.sqlstate == "42501") { + "path-sensitive SQL returned SQLSTATE \(postgresError.sqlstate ?? "none")" + } + var visibleError = ([postgresError.description] + postgresError.fields.map(\.value)) + .joined(separator: " ") + if let callerSuppliedPath { + visibleError = visibleError.replacingOccurrences( + of: callerSuppliedPath, + with: "" + ) + } + try require( + !visibleError.contains("/") + && !visibleError.lowercased().contains("pgdata") + && !visibleError.lowercased().contains("application support") + ) { + "path-sensitive PostgreSQL denial exposed a private filesystem path" + } + return postgresError.sqlstate ?? "" + } + throw BrokerFixtureFailure.assertion( + "path-sensitive SQL unexpectedly reached the host-visible broker session: \(label)" + ) + } + + private static func expectSanitizedPostgresError( + _ operation: () async throws -> Void + ) async throws -> String { + do { + try await operation() + } catch OliphauntError.postgres(let postgresError) { + try require(postgresError.sqlstate?.isEmpty == false) { + "sanitized backend ErrorResponse lost its SQLSTATE" + } + let visibleError = ([postgresError.description] + postgresError.fields.map(\.value)) + .joined(separator: " ") + try require( + !visibleError.contains("/") + && !visibleError.lowercased().contains("pgdata") + && !visibleError.lowercased().contains("application support") + && !visibleError.lowercased().contains( + "oliphaunt_missing_private_path_probe.stop") + ) { + "sanitized PostgreSQL ErrorResponse exposed a private filesystem path" + } + return postgresError.sqlstate ?? "" + } + throw BrokerFixtureFailure.assertion( + "path-producing PostgreSQL query unexpectedly completed without an error" + ) + } + + private static func waitForActiveNativeRequest( + session: IOSBrokerSession, + timeout: Duration + ) async throws -> IOSBrokerWorkerDiagnostics { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + var lastError: (any Error)? + while clock.now < deadline { + do { + let diagnostics = try await session.workerDiagnostics() + if diagnostics.activeRequestID != nil && diagnostics.nativeDispatchStarted { + return diagnostics + } + } catch { + lastError = error + } + try await Task.sleep(for: .milliseconds(20)) + } + if let lastError { throw lastError } + throw BrokerFixtureFailure.assertion( + "worker diagnostics did not observe active native dispatch" + ) + } + + private static func waitForQueuedOperation( + manager: IOSBrokerManager, + timeout: Duration + ) async throws -> IOSBrokerDiagnostics { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: timeout) + while clock.now < deadline { + let diagnostics = await manager.diagnostics() + if diagnostics.queuedOperationCount > 0 { + return diagnostics + } + try await Task.sleep(for: .milliseconds(20)) + } + throw BrokerFixtureFailure.assertion( + "manager diagnostics did not observe a queued operation" + ) + } + + private static func requireValue( + _ value: Value?, + _ message: String + ) throws -> Value { + guard let value else { + throw BrokerFixtureFailure.assertion(message) + } + return value + } + + #if DEBUG + private static func expectOutcomeUnknown( + _ operation: () async throws -> Void + ) async throws { + do { + try await operation() + } catch let error as BrokerError { + guard case .outcomeUnknown = error else { throw error } + return + } + throw BrokerFixtureFailure.assertion( + "faulted operation completed without OutcomeUnknown" + ) + } + #endif + + private static func evidence( + phase: String, + manager: IOSBrokerDiagnostics, + worker: IOSBrokerWorkerDiagnostics? + ) -> BrokerDiagnosticEvidence { + BrokerDiagnosticEvidence( + phase: phase, + managerState: managerState(manager.state), + epoch: manager.epoch?.description, + workerPID: manager.extensionProcessIdentifier, + logicalHandleCount: manager.logicalHandleCount, + queuedOperationCount: manager.queuedOperationCount, + activeRequestID: manager.activeRequestID?.rawValue, + launchCount: manager.launchCount, + interruptionCount: manager.interruptionCount, + admissionsPaused: manager.admissionsPaused, + workerState: worker?.state, + transactionStatus: worker?.transactionStatus, + manifestDigest: worker?.manifestDigest, + currentPhysFootprintBytes: worker?.currentPhysFootprintBytes, + currentResidentBytes: worker?.currentResidentBytes, + availableMemoryBytes: worker?.availableMemoryBytes, + nativeDispatchStarted: worker?.nativeDispatchStarted ?? false, + checkpointInProgress: worker?.checkpointInProgress ?? false, + storageProtectionEvidenceJSON: worker?.storageProtectionEvidenceJSON, + extensionEntryPreOpenPhysFootprintBytes: + worker?.extensionEntryPreOpenPhysFootprintBytes, + extensionEntryPreOpenResidentBytes: worker?.extensionEntryPreOpenResidentBytes, + openedIdlePhysFootprintBytes: worker?.openedIdlePhysFootprintBytes, + openedIdleResidentBytes: worker?.openedIdleResidentBytes + ) + } + + private static func managerState(_ state: IOSBrokerManagerState) -> String { + switch state { + case .unavailable: "unavailable" + case .idle: "idle" + case .launching: "launching" + case .binding: "binding" + case .recovering: "recovering" + case .ready: "ready" + case .quiescing: "quiescing" + case .interrupted: "interrupted" + case .closing: "closing" + } + } + + private static func require( + _ condition: @autoclosure () throws -> Bool, + _ message: () -> String + ) throws { + guard try condition() else { + throw BrokerFixtureFailure.assertion(message()) + } + } + } + + private struct PlannedFixtureRollback: Error {} + + private enum BrokerFixtureFailure: Error, CustomStringConvertible { + case assertion(String) + + var description: String { + switch self { + case .assertion(let message): "broker fixture assertion failed: \(message)" + } + } + } + + private actor BrokerFixtureLatch { + private var signaled = false + private var waiters: [CheckedContinuation] = [] + + func wait() async { + if signaled { return } + await withCheckedContinuation { continuation in + waiters.append(continuation) + } + } + + func signal() { + signaled = true + let pending = waiters + waiters.removeAll() + for waiter in pending { + waiter.resume() + } + } + } + + private final class BrokerFixtureSignal: @unchecked Sendable { + private let lock = NSLock() + private var signaled = false + private var waiters: [CheckedContinuation] = [] + + func wait() async { + await withCheckedContinuation { continuation in + lock.lock() + if signaled { + lock.unlock() + continuation.resume() + } else { + waiters.append(continuation) + lock.unlock() + } + } + } + + @discardableResult + func signal() -> Bool { + lock.lock() + guard !signaled else { + lock.unlock() + return false + } + signaled = true + let pending = waiters + waiters.removeAll() + lock.unlock() + for waiter in pending { + waiter.resume() + } + return true + } + } + + private final class BrokerStreamCounter: @unchecked Sendable { + private let lock = NSLock() + private var bytes = 0 + private var chunks = 0 + + func consume(_ chunk: Data) { + lock.lock() + bytes += chunk.count + chunks += 1 + lock.unlock() + } + + var byteCount: Int { + lock.lock() + defer { lock.unlock() } + return bytes + } + + var chunkCount: Int { + lock.lock() + defer { lock.unlock() } + return chunks + } + } +#endif diff --git a/spikes/ios-native-broker/Host/ProbeSocket.swift b/spikes/ios-native-broker/Host/ProbeSocket.swift new file mode 100644 index 00000000..7d325b91 --- /dev/null +++ b/spikes/ios-native-broker/Host/ProbeSocket.swift @@ -0,0 +1,216 @@ +import Darwin +import Foundation +import OliphauntBrokerProtocol + +final class ProbeSocketPair: @unchecked Sendable { + let host: ProbeSocket + private let lock = NSLock() + private var workerOriginal: Int32 + + var workerDescriptor: Int32 { + lock.withLock { workerOriginal } + } + + func takeWorkerDescriptor() throws -> Int32 { + try lock.withLock { + guard workerOriginal >= 0 else { + throw ProbeSocketError.systemCall("take worker descriptor", EBADF) + } + let descriptor = workerOriginal + workerOriginal = -1 + return descriptor + } + } + + init() throws { + var descriptors = [Int32](repeating: -1, count: 2) + guard socketpair(AF_UNIX, SOCK_STREAM, 0, &descriptors) == 0 else { + throw ProbeSocketError.systemCall("socketpair", errno) + } + do { + try ProbeSocket.configure(descriptors[0]) + try ProbeSocket.configure(descriptors[1]) + } catch { + Darwin.close(descriptors[0]) + Darwin.close(descriptors[1]) + throw error + } + host = ProbeSocket(adopting: descriptors[0]) + workerOriginal = descriptors[1] + } + + deinit { + closeWorkerOriginal() + } + + func closeWorkerOriginal() { + lock.withLock { + if workerOriginal >= 0 { + Darwin.close(workerOriginal) + workerOriginal = -1 + } + } + } +} + +final class ProbeSocket: @unchecked Sendable { + private let descriptor: Int32 + private let ioQueue = DispatchQueue(label: "dev.oliphaunt.brokerspike.socket") + private let closeLock = NSLock() + private var closed = false + private var decoder = BrokerFrameDecoder() + private var pendingFrames: [BrokerFrame] = [] + + init(adopting descriptor: Int32) { + self.descriptor = descriptor + } + + deinit { + close() + } + + static func configure(_ descriptor: Int32) throws { + let descriptorFlags = fcntl(descriptor, F_GETFD) + guard descriptorFlags >= 0, + fcntl(descriptor, F_SETFD, descriptorFlags | FD_CLOEXEC) == 0 + else { + throw ProbeSocketError.systemCall("fcntl(FD_CLOEXEC)", errno) + } + let statusFlags = fcntl(descriptor, F_GETFL) + guard statusFlags >= 0, + fcntl(descriptor, F_SETFL, statusFlags | O_NONBLOCK) == 0 + else { + throw ProbeSocketError.systemCall("fcntl(O_NONBLOCK)", errno) + } + var enabled: Int32 = 1 + guard + setsockopt( + descriptor, + SOL_SOCKET, + SO_NOSIGPIPE, + &enabled, + socklen_t(MemoryLayout.size(ofValue: enabled)) + ) == 0 + else { + throw ProbeSocketError.systemCall("setsockopt(SO_NOSIGPIPE)", errno) + } + } + + func write(_ data: Data) async throws { + try await onIOQueue { + try Self.writeAll(data, to: self.descriptor) + } + } + + func writeFragmented(_ data: Data) async throws { + var offset = 0 + var fragment = 1 + while offset < data.count { + let end = min(data.count, offset + fragment) + try await write(data.subdata(in: offset.. BrokerFrame { + try await onIOQueue { + self.decoder.expectedEpoch = expectedEpoch + if !self.pendingFrames.isEmpty { + return self.pendingFrames.removeFirst() + } + while true { + let bytes = try Self.readSome(from: self.descriptor) + self.pendingFrames.append(contentsOf: try self.decoder.append(bytes)) + if !self.pendingFrames.isEmpty { + return self.pendingFrames.removeFirst() + } + } + } + } + + func close() { + closeLock.withLock { + guard !closed else { return } + closed = true + Darwin.close(descriptor) + } + } + + private func onIOQueue( + _ work: @escaping @Sendable () throws -> T + ) async throws -> T { + try await withCheckedThrowingContinuation { continuation in + ioQueue.async { + continuation.resume(with: Result(catching: work)) + } + } + } + + private static func writeAll(_ data: Data, to descriptor: Int32) throws { + try data.withUnsafeBytes { rawBuffer in + guard let base = rawBuffer.baseAddress else { return } + var offset = 0 + while offset < rawBuffer.count { + let written = Darwin.write( + descriptor, + base.advanced(by: offset), + rawBuffer.count - offset + ) + if written > 0 { + offset += written + } else if written < 0 && (errno == EAGAIN || errno == EWOULDBLOCK) { + try wait(descriptor: descriptor, events: Int16(POLLOUT)) + } else if written < 0 && errno == EINTR { + continue + } else { + throw ProbeSocketError.systemCall("write", errno) + } + } + } + } + + private static func readSome(from descriptor: Int32) throws -> Data { + var storage = [UInt8](repeating: 0, count: 64 * 1024) + while true { + let count = Darwin.read(descriptor, &storage, storage.count) + if count > 0 { + return Data(storage.prefix(count)) + } + if count == 0 { + throw ProbeSocketError.endOfFile + } + if errno == EAGAIN || errno == EWOULDBLOCK { + try wait(descriptor: descriptor, events: Int16(POLLIN)) + } else if errno != EINTR { + throw ProbeSocketError.systemCall("read", errno) + } + } + } + + private static func wait(descriptor: Int32, events: Int16) throws { + var item = pollfd(fd: descriptor, events: events, revents: 0) + while true { + let result = Darwin.poll(&item, 1, 5_000) + if result > 0 { return } + if result == 0 { throw ProbeSocketError.timeout } + if errno != EINTR { + throw ProbeSocketError.systemCall("poll", errno) + } + } + } +} + +enum ProbeSocketError: Error, CustomStringConvertible { + case systemCall(String, Int32) + case endOfFile + case timeout + + var description: String { + switch self { + case .systemCall(let name, let code): "\(name) failed: \(String(cString: strerror(code)))" + case .endOfFile: "broker socket closed" + case .timeout: "broker socket timed out" + } + } +} diff --git a/spikes/ios-native-broker/README.md b/spikes/ios-native-broker/README.md new file mode 100644 index 00000000..90520d13 --- /dev/null +++ b/spikes/ios-native-broker/README.md @@ -0,0 +1,26 @@ +# iOS NativeBroker feasibility harness + +This fixture is an iOS 26-only host application with a bundle-only, non-UI +ExtensionFoundation app extension. It exists to exercise the public +`AppExtensionProcess` model on an iOS simulator without adding an example app +to the released Swift package. Device packaging remains a separate qualification +gate. + +The checked-in Ruby generator creates an Xcode project under `Generated/`. +The generated project: + +- enables `EX_ENABLE_EXTENSION_POINT_GENERATION` in the host and extension; +- embeds an ExtensionKit extension in the host app; +- links broker protocol/host code into the app; +- links native Oliphaunt code and `liboliphaunt.xcframework` only into the + extension when the XCFramework path is supplied; +- copies runtime/template-PGDATA resources only into the extension. + +Run `src/sdks/swift/tools/run-ios-broker-simulator.sh` from the repository root. The runner +builds the required local artifacts, boots an iOS 26 simulator, installs the +host app, launches its self-test, and writes logs and a JSON result beneath +`target/ios-native-broker-spike/`. + +The simulator proves packaging, discovery, process separation, XPC file- +descriptor transfer, framing and host survival. Real-device memory ceilings, +background suspension behavior, and App Store review remain separate gates. diff --git a/spikes/ios-native-broker/generate_project.rb b/spikes/ios-native-broker/generate_project.rb new file mode 100644 index 00000000..1cd9573c --- /dev/null +++ b/spikes/ios-native-broker/generate_project.rb @@ -0,0 +1,182 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "fileutils" +require "pathname" +require "xcodeproj" + +fixture_root = Pathname.new(__dir__).realpath +repo_root = fixture_root.join("../..").realpath +generated_root = fixture_root.join("Generated") +project_path = generated_root.join("OliphauntBrokerSpike.xcodeproj") +host_bundle_identifier = ENV.fetch( + "OLIPHAUNT_IOS_BROKER_BUNDLE_ID", + "dev.oliphaunt.brokerspike" +) +abort("the ExtensionFoundation binding requires host bundle ID dev.oliphaunt.brokerspike") unless host_bundle_identifier == "dev.oliphaunt.brokerspike" +extension_bundle_identifier = ENV.fetch( + "OLIPHAUNT_IOS_BROKER_EXTENSION_BUNDLE_ID", + "dev.oliphaunt.brokerspike.extension" +) +abort("the fixture requires extension bundle ID dev.oliphaunt.brokerspike.extension") unless extension_bundle_identifier == "dev.oliphaunt.brokerspike.extension" +development_team = ENV.fetch("OLIPHAUNT_IOS_BROKER_DEVELOPMENT_TEAM", "") +artifact_platform = ENV.fetch( + "OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM", + "simulator" +) +abort("unsupported broker artifact platform: #{artifact_platform}") unless %w[simulator device].include?(artifact_platform) +FileUtils.rm_rf(generated_root) +FileUtils.mkdir_p(generated_root) + +project = Xcodeproj::Project.new(project_path.to_s) +project.root_object.attributes["LastSwiftUpdateCheck"] = "2640" +project.root_object.attributes["LastUpgradeCheck"] = "2640" + +sources_group = project.main_group.new_group("Sources") +host_group = sources_group.new_group("Host") +extension_group = sources_group.new_group("BrokerAppExtension") + +host_target = project.new_target( + :application, + "OliphauntBrokerSpike", + :ios, + "26.0" +) +extension_target = project.new_target( + :app_extension, + "BrokerAppExtension", + :ios, + "26.0" +) +extension_target.product_type = "com.apple.product-type.extensionkit-extension" +extension_target.product_reference.explicit_file_type = "wrapper.extensionkit-extension" + +def apply_common_settings(target, development_team) + target.build_configurations.each do |configuration| + settings = configuration.build_settings + settings["SWIFT_VERSION"] = "6.0" + settings["MARKETING_VERSION"] = "1.0" + settings["CURRENT_PROJECT_VERSION"] = "1" + settings["IPHONEOS_DEPLOYMENT_TARGET"] = "26.0" + settings["TARGETED_DEVICE_FAMILY"] = "1" + settings["GENERATE_INFOPLIST_FILE"] = "YES" + settings["EX_ENABLE_EXTENSION_POINT_GENERATION"] = "YES" + settings["CODE_SIGN_STYLE"] = "Automatic" + settings["DEVELOPMENT_TEAM"] = development_team + settings["ENABLE_USER_SCRIPT_SANDBOXING"] = "YES" + settings["SWIFT_STRICT_CONCURRENCY"] = "complete" + end +end + +apply_common_settings(host_target, development_team) +apply_common_settings(extension_target, development_team) + +host_target.build_configurations.each do |configuration| + settings = configuration.build_settings + settings["PRODUCT_BUNDLE_IDENTIFIER"] = host_bundle_identifier + settings["PRODUCT_NAME"] = "OliphauntBrokerSpike" + settings["INFOPLIST_KEY_CFBundleDisplayName"] = "Oliphaunt Broker Spike" + settings["INFOPLIST_KEY_UILaunchScreen_Generation"] = "YES" + settings["INFOPLIST_KEY_UIApplicationSceneManifest_Generation"] = "YES" + settings["LD_RUNPATH_SEARCH_PATHS"] = ["$(inherited)", "@executable_path/Frameworks"] +end + +extension_target.build_configurations.each do |configuration| + settings = configuration.build_settings + settings["PRODUCT_BUNDLE_IDENTIFIER"] = extension_bundle_identifier + settings["PRODUCT_NAME"] = "BrokerAppExtension" + settings["INFOPLIST_KEY_CFBundleDisplayName"] = "Oliphaunt Broker Worker" + settings["APPLICATION_EXTENSION_API_ONLY"] = "YES" + settings["SKIP_INSTALL"] = "YES" + settings["LD_RUNPATH_SEARCH_PATHS"] = [ + "$(inherited)", + "@executable_path/Frameworks", + "@executable_path/../../Frameworks" + ] +end + +Dir[fixture_root.join("Host/**/*.swift")].sort.each do |source| + reference = host_group.new_file(source) + host_target.source_build_phase.add_file_reference(reference) +end +Dir[fixture_root.join("BrokerAppExtension/**/*.swift")].sort.each do |source| + reference = extension_group.new_file(source) + extension_target.source_build_phase.add_file_reference(reference) +end + +package_reference = project.new( + Xcodeproj::Project::Object::XCLocalSwiftPackageReference +) +package_reference.relative_path = repo_root.to_s +project.root_object.package_references << package_reference + +def add_package_product(project, target, package_reference, product_name) + dependency = project.new( + Xcodeproj::Project::Object::XCSwiftPackageProductDependency + ) + dependency.package = package_reference + dependency.product_name = product_name + target.package_product_dependencies << dependency + build_file = project.new(Xcodeproj::Project::Object::PBXBuildFile) + build_file.product_ref = dependency + target.frameworks_build_phase.files << build_file +end + +add_package_product(project, host_target, package_reference, "OliphauntBrokerProtocol") +if ENV["OLIPHAUNT_BROKER_INCLUDE_SDK"] == "1" + add_package_product(project, host_target, package_reference, "Oliphaunt") + add_package_product(project, host_target, package_reference, "OliphauntBrokerXPC") + add_package_product(project, host_target, package_reference, "OliphauntIOSBroker") +end +add_package_product(project, extension_target, package_reference, "OliphauntBrokerProtocol") +add_package_product(project, extension_target, package_reference, "OliphauntBrokerXPC") +add_package_product(project, extension_target, package_reference, "OliphauntBrokerExtension") + +native_xcframework = ENV["OLIPHAUNT_IOS_BROKER_XCFRAMEWORK"] +if native_xcframework && !native_xcframework.empty? + native_path = Pathname.new(native_xcframework).realpath + abort("not an XCFramework: #{native_path}") unless native_path.directory? && native_path.extname == ".xcframework" + native_reference = project.frameworks_group.new_file(native_path.to_s) + extension_target.frameworks_build_phase.add_file_reference(native_reference, true) + + # The simulator fixture can keep its dylib/framework private to the worker + # bundle. A signed device framework must be embedded once in the containing + # app; the worker already searches @executable_path/../../Frameworks. + embed_target = artifact_platform == "device" ? host_target : extension_target + embed_frameworks = embed_target.new_copy_files_build_phase("Embed Broker Framework") + embed_frameworks.dst_subfolder_spec = "10" + embedded = embed_frameworks.add_file_reference(native_reference, true) + embedded.settings = { "ATTRIBUTES" => ["CodeSignOnCopy", "RemoveHeadersOnCopy"] } +end + +runtime_resources = ENV["OLIPHAUNT_IOS_BROKER_RESOURCES"] +if runtime_resources && !runtime_resources.empty? + resources_path = Pathname.new(runtime_resources).realpath + abort("runtime resources must contain oliphaunt/: #{resources_path}") unless resources_path.join("oliphaunt").directory? + resource_reference = extension_group.new_file( + resources_path.join("oliphaunt").to_s, + :group + ) + extension_target.resources_build_phase.add_file_reference(resource_reference, true) +end + +host_target.add_dependency(extension_target) +embed_extensions = host_target.new_copy_files_build_phase("Embed ExtensionKit Extensions") +embed_extensions.dst_subfolder_spec = "16" +embed_extensions.dst_path = "$(EXTENSIONS_FOLDER_PATH)" +embedded_extension = embed_extensions.add_file_reference( + extension_target.product_reference, + true +) +embedded_extension.settings = { + "ATTRIBUTES" => ["CodeSignOnCopy", "RemoveHeadersOnCopy"] +} + +project.save + +scheme = Xcodeproj::XCScheme.new +scheme.add_build_target(host_target) +scheme.set_launch_target(host_target) +scheme.save_as(project_path, "OliphauntBrokerSpike", true) + +puts project_path diff --git a/src/extensions/external/postgis/.release-semantic-inputs.json b/src/extensions/external/postgis/.release-semantic-inputs.json index fd32cd81..42ceb105 100644 --- a/src/extensions/external/postgis/.release-semantic-inputs.json +++ b/src/extensions/external/postgis/.release-semantic-inputs.json @@ -79,27 +79,27 @@ }, { "path": "src/runtimes/liboliphaunt/native/bin/build-postgres18-android-arm64.sh", - "sha256": "203fea01484138fcb07481b979b707f0aab19ed2134dea5781d0ccd094b908dc" + "sha256": "f3c31b4f7c59e9237f1814774ea7f68f5b555efceafd62db0631dd5c9d87c1c9" }, { "path": "src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-device.sh", - "sha256": "bfbd6056f74c4069d6aa0819a775410e084b60bbb22c5c923c3a6ac9da4cf307" + "sha256": "3923c71fd81ae42ebe5d652af0dade932c10fe2a02b8c95d5fdbb1d6e6a7f132" }, { "path": "src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-simulator.sh", - "sha256": "71acd0774c4ac9d0d32e7f0c229bd9777e57083ca270fab495f3f126a033b1d9" + "sha256": "b697eaae1c5ad18f69a2db3319303efa6d939fee9fd03fb49a7898e43fdb786a" }, { "path": "src/runtimes/liboliphaunt/native/bin/build-postgres18-linux.sh", - "sha256": "d9ded61bf55c42197f84115940c0546f2801876b1562084b17cae80b41aeae68" + "sha256": "eadba63f4fa3ef71fb11d54cf7aa1627d21c5f841178ee73188c8e88e9f4be80" }, { "path": "src/runtimes/liboliphaunt/native/bin/build-postgres18-macos.sh", - "sha256": "b53b165f9942a080ecfe84af9f2bffd41c194af79657ab09fa1018c156af20ca" + "sha256": "1f7745eb78ba7c7e55a9e367861cb75d0f12641c436fc5c2589d5b17367f21b3" }, { "path": "src/runtimes/liboliphaunt/native/bin/build-postgres18-windows.ps1", - "sha256": "3ecdda83cf6ed8df45852ead088dc5baa415b07e302824b0531a4ff839cbb8c6" + "sha256": "6f8c48a9bba63943251e613976c901724099138ea51804957fb0128ef7f3865d" }, { "path": "src/runtimes/liboliphaunt/native/bin/mobile-postgis-extensions.sh", @@ -390,5 +390,5 @@ ] } ], - "sha256": "a8d8102915899d7308025b7c9fd5401f64e18c347d2d53aed9c083c7b001c2a4" + "sha256": "6df1c50270daa602c6991a67b5d30877d6e09d2bf3390b5ba1601841622c7273" } diff --git a/src/runtimes/liboliphaunt/native/bin/build-postgres18-android-arm64.sh b/src/runtimes/liboliphaunt/native/bin/build-postgres18-android-arm64.sh index 94209117..c6291a74 100755 --- a/src/runtimes/liboliphaunt/native/bin/build-postgres18-android-arm64.sh +++ b/src/runtimes/liboliphaunt/native/bin/build-postgres18-android-arm64.sh @@ -289,6 +289,10 @@ patched_source_ready() { grep -Fq 'oliphaunt_embedded_main' "$build_dir/src/backend/tcop/postgres.c" && grep -Fq 'oliphaunt_embedded_kill' "$build_dir/src/port/pqsignal.c" && grep -Fq 'oliphaunt_embedded_raise' "$build_dir/src/port/pqsignal.c" && + grep -Fq 'MyProcPort->oliphaunt_io != NULL' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq '!role_form->rolcanlogin' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'ResetOliphauntAuthenticatedRoleLatch' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'oliphaunt_authenticated_role_is_superuser = false' "$build_dir/src/backend/commands/variable.c" && grep -Fq 'getenv("ICU_DATA")' "$build_dir/src/bin/initdb/initdb.c" && grep -Fq 'oliphaunt_embedded' "$build_dir/meson_options.txt" && grep -Fq 'OLIPHAUNT_EMBEDDED' "$build_dir/meson.build" diff --git a/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-device.sh b/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-device.sh index 0b72876e..7214695e 100755 --- a/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-device.sh +++ b/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-device.sh @@ -271,6 +271,10 @@ patched_source_ready() { grep -Fq 'oliphaunt_embedded_main' "$build_dir/src/backend/tcop/postgres.c" && grep -Fq 'oliphaunt_embedded_kill' "$build_dir/src/port/pqsignal.c" && grep -Fq 'oliphaunt_embedded_raise' "$build_dir/src/port/pqsignal.c" && + grep -Fq 'MyProcPort->oliphaunt_io != NULL' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq '!role_form->rolcanlogin' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'ResetOliphauntAuthenticatedRoleLatch' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'oliphaunt_authenticated_role_is_superuser = false' "$build_dir/src/backend/commands/variable.c" && grep -Fq 'getenv("ICU_DATA")' "$build_dir/src/bin/initdb/initdb.c" && grep -Fq 'oliphaunt_embedded' "$build_dir/meson_options.txt" && grep -Fq 'OLIPHAUNT_EMBEDDED' "$build_dir/meson.build" diff --git a/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-simulator.sh b/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-simulator.sh index a5a49e7f..1fbeb422 100755 --- a/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-simulator.sh +++ b/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-simulator.sh @@ -271,6 +271,10 @@ patched_source_ready() { grep -Fq 'oliphaunt_embedded_main' "$build_dir/src/backend/tcop/postgres.c" && grep -Fq 'oliphaunt_embedded_kill' "$build_dir/src/port/pqsignal.c" && grep -Fq 'oliphaunt_embedded_raise' "$build_dir/src/port/pqsignal.c" && + grep -Fq 'MyProcPort->oliphaunt_io != NULL' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq '!role_form->rolcanlogin' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'ResetOliphauntAuthenticatedRoleLatch' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'oliphaunt_authenticated_role_is_superuser = false' "$build_dir/src/backend/commands/variable.c" && grep -Fq 'getenv("ICU_DATA")' "$build_dir/src/bin/initdb/initdb.c" && grep -Fq 'oliphaunt_embedded' "$build_dir/meson_options.txt" && grep -Fq 'OLIPHAUNT_EMBEDDED' "$build_dir/meson.build" diff --git a/src/runtimes/liboliphaunt/native/bin/build-postgres18-linux.sh b/src/runtimes/liboliphaunt/native/bin/build-postgres18-linux.sh index b962d45c..37d602be 100755 --- a/src/runtimes/liboliphaunt/native/bin/build-postgres18-linux.sh +++ b/src/runtimes/liboliphaunt/native/bin/build-postgres18-linux.sh @@ -545,6 +545,10 @@ patched_source_ready() { grep -Fq 'oliphaunt_embedded_main' "$build_dir/src/backend/tcop/postgres.c" && grep -Fq 'oliphaunt_embedded_kill' "$build_dir/src/port/pqsignal.c" && grep -Fq 'oliphaunt_embedded_raise' "$build_dir/src/port/pqsignal.c" && + grep -Fq 'MyProcPort->oliphaunt_io != NULL' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq '!role_form->rolcanlogin' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'ResetOliphauntAuthenticatedRoleLatch' "$build_dir/src/backend/utils/init/postinit.c" && + grep -Fq 'oliphaunt_authenticated_role_is_superuser = false' "$build_dir/src/backend/commands/variable.c" && grep -Fq 'getenv("ICU_DATA")' "$build_dir/src/bin/initdb/initdb.c" && grep -Fq 'oliphaunt_embedded' "$build_dir/meson_options.txt" && grep -Fq 'OLIPHAUNT_EMBEDDED' "$build_dir/meson.build" diff --git a/src/runtimes/liboliphaunt/native/bin/build-postgres18-macos.sh b/src/runtimes/liboliphaunt/native/bin/build-postgres18-macos.sh index 2220d860..d5b8168c 100755 --- a/src/runtimes/liboliphaunt/native/bin/build-postgres18-macos.sh +++ b/src/runtimes/liboliphaunt/native/bin/build-postgres18-macos.sh @@ -1005,6 +1005,22 @@ embedded_core_module_avoids_provider_collisions() { rm -f "$engine_symbols" } +embedded_core_module_ready() { + local stem="$1" + local module="$embedded_modules_dir/$stem.dylib" + module_depends_on_liboliphaunt "$module" && + module_has_postgres_symbols_bound_to_liboliphaunt "$module" && + embedded_core_module_avoids_provider_collisions "$stem" +} + +embedded_dict_snowball_module_ready() { + embedded_core_module_ready dict_snowball +} + +embedded_plpgsql_module_ready() { + embedded_core_module_ready plpgsql +} + embedded_dict_snowball_avoids_provider_collisions() { embedded_core_module_avoids_provider_collisions dict_snowball } @@ -1346,7 +1362,11 @@ patches_applied() { grep -q 'OLIPHAUNT_EMBEDDED_NO_SHELL_COMMANDS' src/backend/access/transam/xlogarchive.c && grep -q 'oliphaunt_pg_hash_create' src/include/utils/hsearch.h && grep -q 'oliphaunt_embedded_kill' src/port/pqsignal.c && - grep -q 'oliphaunt_embedded_raise' src/port/pqsignal.c + grep -q 'oliphaunt_embedded_raise' src/port/pqsignal.c && + grep -q 'MyProcPort->oliphaunt_io != NULL' src/backend/utils/init/postinit.c && + grep -q '!role_form->rolcanlogin' src/backend/utils/init/postinit.c && + grep -q 'ResetOliphauntAuthenticatedRoleLatch' src/backend/utils/init/postinit.c && + grep -q 'oliphaunt_authenticated_role_is_superuser = false' src/backend/commands/variable.c } if ! patches_applied; then @@ -2225,22 +2245,6 @@ build_postgis_extension() { stage_postgis_data_files "$postgis_build_dir" } -embedded_core_module_ready() { - local stem="$1" - local module="$embedded_modules_dir/$stem.dylib" - module_depends_on_liboliphaunt "$module" && - module_has_postgres_symbols_bound_to_liboliphaunt "$module" && - embedded_core_module_avoids_provider_collisions "$stem" -} - -embedded_dict_snowball_module_ready() { - embedded_core_module_ready dict_snowball -} - -embedded_plpgsql_module_ready() { - embedded_core_module_ready plpgsql -} - build_embedded_dict_snowball_module() { local module="$embedded_modules_dir/dict_snowball.dylib" local desired_module_hash diff --git a/src/runtimes/liboliphaunt/native/bin/build-postgres18-windows.ps1 b/src/runtimes/liboliphaunt/native/bin/build-postgres18-windows.ps1 index d3aca38c..8f895422 100644 --- a/src/runtimes/liboliphaunt/native/bin/build-postgres18-windows.ps1 +++ b/src/runtimes/liboliphaunt/native/bin/build-postgres18-windows.ps1 @@ -622,6 +622,10 @@ function Assert-PatchedSource { Assert-FileContains (Join-Path $BuildDir "src/backend/tcop/postgres.c") "oliphaunt_embedded_main" Assert-FileContains (Join-Path $BuildDir "src/port/pqsignal.c") "oliphaunt_embedded_kill" Assert-FileContains (Join-Path $BuildDir "src/port/pqsignal.c") "oliphaunt_embedded_raise" + Assert-FileContains (Join-Path $BuildDir "src/backend/utils/init/postinit.c") "MyProcPort->oliphaunt_io != NULL" + Assert-FileContains (Join-Path $BuildDir "src/backend/utils/init/postinit.c") "!role_form->rolcanlogin" + Assert-FileContains (Join-Path $BuildDir "src/backend/utils/init/postinit.c") "ResetOliphauntAuthenticatedRoleLatch" + Assert-FileContains (Join-Path $BuildDir "src/backend/commands/variable.c") "oliphaunt_authenticated_role_is_superuser = false" Assert-FileContains (Join-Path $BuildDir "src/bin/initdb/initdb.c") 'getenv("ICU_DATA")' Assert-FileContains (Join-Path $BuildDir "meson_options.txt") "oliphaunt_embedded" Assert-FileContains (Join-Path $BuildDir "meson_options.txt") "oliphaunt_embedded_module_provider" diff --git a/src/runtimes/liboliphaunt/native/bin/check-postgres18-ios-simulator.sh b/src/runtimes/liboliphaunt/native/bin/check-postgres18-ios-simulator.sh index 92cb6ea6..e1af20e5 100755 --- a/src/runtimes/liboliphaunt/native/bin/check-postgres18-ios-simulator.sh +++ b/src/runtimes/liboliphaunt/native/bin/check-postgres18-ios-simulator.sh @@ -194,6 +194,8 @@ compile_probe_objects() { make -C src/backend/libpq pqcomm.o V=1 make -C src/backend/tcop postgres.o V=1 make -C src/backend/storage/ipc ipc.o V=1 + make -C src/backend/commands variable.o V=1 + make -C src/backend/utils/init postinit.o V=1 make -C src/backend/utils/fmgr dfmgr.o V=1 ) > "$make_log" 2>&1 @@ -211,6 +213,10 @@ verify_probe_symbols() { rg -q --fixed-strings "oliphaunt_embedded_main" "$source_root/include/tcop/tcopprot.h" rg -q --fixed-strings "oliphaunt_embedded_kill" "$source_root/port/pqsignal.c" rg -q --fixed-strings "oliphaunt_embedded_raise" "$source_root/port/pqsignal.c" + rg -q --fixed-strings "MyProcPort->oliphaunt_io != NULL" "$source_root/backend/utils/init/postinit.c" + rg -q --fixed-strings "!role_form->rolcanlogin" "$source_root/backend/utils/init/postinit.c" + rg -q --fixed-strings "ResetOliphauntAuthenticatedRoleLatch" "$source_root/backend/utils/init/postinit.c" + rg -q --fixed-strings "oliphaunt_authenticated_role_is_superuser = false" "$source_root/backend/commands/variable.c" rg -q --fixed-strings 'getenv("ICU_DATA")' "$source_root/bin/initdb/initdb.c" rg -q --fixed-strings "oliphaunt_static_extension_magic(file_scanner->static_extension)" "$source_root/backend/utils/fmgr/dfmgr.c" } diff --git a/src/runtimes/liboliphaunt/native/patches/postgresql-18.4/0021-liboliphaunt-authenticate-embedded-role.patch b/src/runtimes/liboliphaunt/native/patches/postgresql-18.4/0021-liboliphaunt-authenticate-embedded-role.patch new file mode 100644 index 00000000..6c0355bd --- /dev/null +++ b/src/runtimes/liboliphaunt/native/patches/postgresql-18.4/0021-liboliphaunt-authenticate-embedded-role.patch @@ -0,0 +1,161 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: liboliphaunt +Date: Sun, 9 Aug 2026 00:00:00 +0000 +Subject: [PATCH] liboliphaunt: authenticate embedded role + +The embedded entrypoint supplies an explicit database role, but InitPostgres +currently follows PostgreSQL's standalone branch and always initializes the +immutable authenticated identity as BOOTSTRAP_SUPERUSERID. Changing role in +SQL cannot form a security boundary because RESET ROLE or RESET SESSION +AUTHORIZATION can restore that bootstrap identity. + +Only for the Oliphaunt host-I/O port, require a LOGIN role and initialize +AuthenticatedUserId and the session identity from that supplied role. Keep +ordinary PostgreSQL standalone startup byte-for-byte equivalent, including +its bootstrap recovery behavior. Once session authorization observes that +role's catalog demotion, latch its cached privilege state off so RESET, +rollback, or DISCARD cannot restore the startup value. +--- + src/backend/commands/variable.c | 41 +++++++++++++++++++++++++++++++++++- + src/backend/utils/init/postinit.c | 42 ++++++++++++++++++++++++++++++------- + src/include/utils/guc_hooks.h | 3 +++ + 3 files changed, 77 insertions(+), 9 deletions(-) + +diff --git a/src/backend/commands/variable.c b/src/backend/commands/variable.c +index 608f10d9..69205a22 100644 +--- a/src/backend/commands/variable.c ++++ b/src/backend/commands/variable.c +@@ -25,6 +25,9 @@ + #include "access/xlogprefetcher.h" + #include "catalog/pg_authid.h" + #include "common/string.h" ++#ifdef OLIPHAUNT_EMBEDDED ++#include "libpq/libpq-be.h" ++#endif + #include "mb/pg_wchar.h" + #include "miscadmin.h" + #include "postmaster/postmaster.h" +@@ -811,6 +814,18 @@ typedef struct + bool is_superuser; + } role_auth_extra; + ++#ifdef OLIPHAUNT_EMBEDDED ++static bool oliphaunt_authenticated_role_latch_initialized = false; ++static bool oliphaunt_authenticated_role_is_superuser = false; ++ ++void ++ResetOliphauntAuthenticatedRoleLatch(void) ++{ ++ oliphaunt_authenticated_role_latch_initialized = false; ++ oliphaunt_authenticated_role_is_superuser = false; ++} ++#endif ++ + bool + check_session_authorization(char **newval, void **extra, GucSource source) + { +@@ -912,12 +927,36 @@ void + assign_session_authorization(const char *newval, void *extra) + { + role_auth_extra *myextra = (role_auth_extra *) extra; ++ bool is_superuser; + + /* Do nothing for the boot_val default of NULL */ + if (!myextra) + return; + +- SetSessionAuthorization(myextra->roleid, myextra->is_superuser); ++ is_superuser = myextra->is_superuser; ++#ifdef OLIPHAUNT_EMBEDDED ++ /* ++ * An Oliphaunt session may demote its authenticated role after startup. ++ * Once an assignment observes that demotion, stale RESET, rollback, or ++ * DISCARD state must never raise its cached superuser state again. Keep ++ * this fail-closed and allocation-free because assign hooks also run late ++ * during transaction abort cleanup. ++ */ ++ if (MyProcPort != NULL && ++ MyProcPort->oliphaunt_io != NULL && ++ myextra->roleid == GetAuthenticatedUserId()) ++ { ++ if (!oliphaunt_authenticated_role_latch_initialized) ++ { ++ oliphaunt_authenticated_role_latch_initialized = true; ++ oliphaunt_authenticated_role_is_superuser = is_superuser; ++ } ++ else if (!is_superuser) ++ oliphaunt_authenticated_role_is_superuser = false; ++ is_superuser = oliphaunt_authenticated_role_is_superuser; ++ } ++#endif ++ SetSessionAuthorization(myextra->roleid, is_superuser); + } + + +diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c +index c86ceefd..41762de6 100644 +--- a/src/backend/utils/init/postinit.c ++++ b/src/backend/utils/init/postinit.c +@@ -870,14 +870,40 @@ InitPostgres(const char *in_dbname, Oid dboid, + } + else if (!IsUnderPostmaster) + { +- InitializeSessionUserIdStandalone(); +- am_superuser = true; +- if (!ThereIsAtLeastOneRole()) +- ereport(WARNING, +- (errcode(ERRCODE_UNDEFINED_OBJECT), +- errmsg("no roles are defined in this database system"), +- errhint("You should immediately run CREATE USER \"%s\" SUPERUSER;.", +- username != NULL ? username : "postgres"))); ++#ifdef OLIPHAUNT_EMBEDDED ++ if (MyProcPort != NULL && MyProcPort->oliphaunt_io != NULL) ++ { ++ HeapTuple role_tuple; ++ Form_pg_authid role_form; ++ ++ ResetOliphauntAuthenticatedRoleLatch(); ++ InitializeSessionUserId(username, useroid, false); ++ role_tuple = SearchSysCache1(AUTHOID, ++ ObjectIdGetDatum(GetAuthenticatedUserId())); ++ if (!HeapTupleIsValid(role_tuple)) ++ elog(FATAL, "cache lookup failed for role %u", ++ GetAuthenticatedUserId()); ++ role_form = (Form_pg_authid) GETSTRUCT(role_tuple); ++ if (!role_form->rolcanlogin) ++ ereport(FATAL, ++ (errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), ++ errmsg("role \"%s\" is not permitted to log in", ++ NameStr(role_form->rolname)))); ++ ReleaseSysCache(role_tuple); ++ am_superuser = superuser(); ++ } ++ else ++#endif ++ { ++ InitializeSessionUserIdStandalone(); ++ am_superuser = true; ++ if (!ThereIsAtLeastOneRole()) ++ ereport(WARNING, ++ (errcode(ERRCODE_UNDEFINED_OBJECT), ++ errmsg("no roles are defined in this database system"), ++ errhint("You should immediately run CREATE USER \"%s\" SUPERUSER;.", ++ username != NULL ? username : "postgres"))); ++ } + } + else if (AmBackgroundWorkerProcess()) + { +diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h +index 82ac8646..0fd28fa4 100644 +--- a/src/include/utils/guc_hooks.h ++++ b/src/include/utils/guc_hooks.h +@@ -97,6 +97,9 @@ extern bool check_primary_slot_name(char **newval, void **extra, + extern bool check_random_seed(double *newval, void **extra, GucSource source); + extern void assign_random_seed(double newval, void *extra); + extern const char *show_random_seed(void); ++#ifdef OLIPHAUNT_EMBEDDED ++extern void ResetOliphauntAuthenticatedRoleLatch(void); ++#endif + extern bool check_recovery_prefetch(int *new_value, void **extra, + GucSource source); + extern void assign_recovery_prefetch(int new_value, void *extra); diff --git a/src/runtimes/liboliphaunt/native/postgres18/source.toml b/src/runtimes/liboliphaunt/native/postgres18/source.toml index 22a583da..c31d1dfe 100644 --- a/src/runtimes/liboliphaunt/native/postgres18/source.toml +++ b/src/runtimes/liboliphaunt/native/postgres18/source.toml @@ -26,4 +26,5 @@ series = [ "0018-liboliphaunt-contain-embedded-proc-signals.patch", "0019-liboliphaunt-link-windows-embedded-modules-to-host.patch", "0020-liboliphaunt-enforce-embedded-signal-boundary.patch", + "0021-liboliphaunt-authenticate-embedded-role.patch", ] diff --git a/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c b/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c index df66e291..528392bf 100644 --- a/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c +++ b/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c @@ -24,8 +24,12 @@ #else #include #include +#include +#include #include #include + +extern char **environ; #endif #undef PG_MAGIC_FUNCTION_NAME @@ -396,6 +400,7 @@ typedef struct StreamAccumulator { size_t len; size_t cap; size_t chunks; + size_t max_chunk; } StreamAccumulator; typedef struct CancelQueryThread { @@ -696,6 +701,9 @@ static int exec_malformed_frame_checks(OliphauntHandle *db) { static int32_t append_stream_chunk(void *context, const uint8_t *data, size_t len) { StreamAccumulator *acc = (StreamAccumulator *)context; acc->chunks++; + if (len > acc->max_chunk) { + acc->max_chunk = len; + } if (len == 0) { return 0; } @@ -716,6 +724,76 @@ static int32_t append_stream_chunk(void *context, const uint8_t *data, size_t le return 0; } +static int exec_stream_queue_hard_ceiling(OliphauntHandle *db) { + static const size_t ceiling = 1024; + const char *previous_limit = getenv("OLIPHAUNT_STREAM_QUEUE_MAX_BYTES"); + char *saved_limit = previous_limit != NULL ? strdup(previous_limit) : NULL; + if (previous_limit != NULL && saved_limit == NULL) { + fprintf(stderr, "failed to save OLIPHAUNT_STREAM_QUEUE_MAX_BYTES\n"); + return 1; + } + if (setenv("OLIPHAUNT_STREAM_QUEUE_MAX_BYTES", "1024", 1) != 0) { + fprintf(stderr, "failed to set OLIPHAUNT_STREAM_QUEUE_MAX_BYTES\n"); + free(saved_limit); + return 1; + } + + unsigned char *query = NULL; + size_t query_len = 0; + push_query( + &query, + &query_len, + "SELECT 'stream-hard-ceiling-ok' AS marker, repeat('x', 65536) AS payload"); + + StreamAccumulator acc = {0}; + fprintf(stderr, "streaming raw protocol with %zu-byte queue ceiling\n", ceiling); + int rc = oliphaunt_exec_protocol_stream(db, query, query_len, append_stream_chunk, &acc); + free(query); + + int status = 0; + if (saved_limit != NULL) { + if (setenv("OLIPHAUNT_STREAM_QUEUE_MAX_BYTES", saved_limit, 1) != 0) { + fprintf(stderr, "failed to restore OLIPHAUNT_STREAM_QUEUE_MAX_BYTES\n"); + status = 1; + } + } else if (unsetenv("OLIPHAUNT_STREAM_QUEUE_MAX_BYTES") != 0) { + fprintf(stderr, "failed to unset OLIPHAUNT_STREAM_QUEUE_MAX_BYTES\n"); + status = 1; + } + free(saved_limit); + + if (rc != 0) { + fprintf(stderr, "bounded oliphaunt_exec_protocol_stream failed: %s\n", oliphaunt_last_error(db)); + status = 1; + } + OliphauntResponse response = { + .data = acc.data, + .len = acc.len, + }; + const unsigned char required_tags[] = {'T', 'D', 'C', 'Z'}; + for (size_t i = 0; i < sizeof(required_tags); i++) { + if (!contains_tag(&response, required_tags[i])) { + fprintf(stderr, "bounded stream response did not contain protocol tag %c\n", required_tags[i]); + status = 1; + } + } + if (!contains_bytes(&response, "stream-hard-ceiling-ok")) { + fprintf(stderr, "bounded stream response lost payload bytes\n"); + status = 1; + } + if (acc.chunks < 2 || acc.max_chunk == 0 || acc.max_chunk > ceiling) { + fprintf( + stderr, + "stream queue ceiling violated: chunks=%zu max_chunk=%zu ceiling=%zu\n", + acc.chunks, + acc.max_chunk, + ceiling); + status = 1; + } + free(acc.data); + return status; +} + static int32_t fail_stream_chunk(void *context, const uint8_t *data, size_t len) { (void)data; (void)len; @@ -1021,6 +1099,76 @@ static int verify_stable_root_lock_file(const char *pgdata) { return 0; } +static int expect_root_locked_in_probe_process(const char *pgdata, const char *runtime_dir) { + OliphauntConfig config = { + .abi_version = OLIPHAUNT_ABI_VERSION, + .pgdata = pgdata, + .runtime_dir = runtime_dir, + .username = "postgres", + .database = "postgres", + .reserved_flags = 0, + }; + OliphauntHandle *probe = NULL; + if (oliphaunt_init(&config, &probe) == 0 || probe != NULL) { + fprintf(stderr, "separate process unexpectedly opened an already-owned native root\n"); + if (probe != NULL) { + oliphaunt_close(probe); + } + return 1; + } + return expect_error_contains( + NULL, + "cross-process native root contention", + "already locked"); +} + +static int verify_cross_process_root_contention( + const char *executable, + const char *pgdata, + const char *runtime_dir) { +#ifdef _WIN32 + (void)executable; + (void)pgdata; + (void)runtime_dir; + fprintf(stderr, "cross-process native root contention probe skipped on Windows\n"); + return 0; +#else + pid_t child = 0; + char *const child_argv[] = { + (char *)executable, + (char *)"--expect-root-locked", + (char *)pgdata, + (char *)runtime_dir, + NULL, + }; + int spawn_rc = posix_spawnp( + &child, + executable, + NULL, + NULL, + child_argv, + environ); + if (spawn_rc != 0) { + fprintf(stderr, "spawn cross-process native root probe failed: %s\n", strerror(spawn_rc)); + return 1; + } + int status = 0; + while (waitpid(child, &status, 0) < 0) { + if (errno == EINTR) { + continue; + } + perror("wait for cross-process native root probe"); + return 1; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "cross-process native root probe failed with status %d\n", status); + return 1; + } + fprintf(stderr, "cross-process native root contention rejected\n"); + return 0; +#endif +} + static void test_tar_write_octal(unsigned char *field, size_t width, unsigned long long value) { memset(field, '0', width); char scratch[32]; @@ -1697,7 +1845,7 @@ static int verify_backup_restore_contract(OliphauntHandle *db, const char *pgdat return 0; } -static int run_cycle(const char *pgdata, const char *runtime_dir) { +static int run_cycle(const char *executable, const char *pgdata, const char *runtime_dir) { static const char *const startup_args[] = { "-c", "application_name=liboliphaunt_smoke", @@ -1742,6 +1890,10 @@ static int run_cycle(const char *pgdata, const char *runtime_dir) { oliphaunt_close(db); return 1; } + if (verify_cross_process_root_contention(executable, pgdata, runtime_dir) != 0) { + oliphaunt_close(db); + return 1; + } if (exec_query_expect_tags(db, "SELECT 1 AS value", select_tags, sizeof(select_tags)) != 0) { oliphaunt_close(db); return 1; @@ -1810,6 +1962,11 @@ static int run_cycle(const char *pgdata, const char *runtime_dir) { return 1; } + if (exec_stream_queue_hard_ceiling(db) != 0) { + oliphaunt_close(db); + return 1; + } + if (exec_stream_callback_failure_recovers(db) != 0) { oliphaunt_close(db); return 1; @@ -1949,9 +2106,267 @@ static int expect_terminal_shutdown_reopen_rejected(const char *pgdata, const ch return expect_error_contains(NULL, "terminal shutdown reopen", "process lifetime has already been used"); } +static int expect_configured_role_rejected( + const char *username, + const char *pgdata, + const char *runtime_dir) { + OliphauntConfig config = { + .abi_version = OLIPHAUNT_ABI_VERSION, + .pgdata = pgdata, + .runtime_dir = runtime_dir, + .username = username, + .database = "postgres", + .reserved_flags = 0, + }; + OliphauntHandle *db = NULL; + fprintf(stderr, "verifying embedded startup rejects configured role: %s\n", username); + if (oliphaunt_init(&config, &db) == 0 || db != NULL) { + fprintf(stderr, "embedded startup ignored the configured database role\n"); + if (db != NULL) { + oliphaunt_close(db); + } + return 1; + } + return expect_error_contains( + NULL, + "missing configured embedded role", + "before ReadyForQuery"); +} + +static int exec_query_expect_error( + OliphauntHandle *db, + const char *sql, + const char *sqlstate) { + unsigned char *query = NULL; + size_t query_len = 0; + push_query(&query, &query_len, sql); + + OliphauntResponse response = {0}; + fprintf(stderr, "executing restricted-role denial probe: %s\n", sql); + int rc = oliphaunt_exec_protocol(db, query, query_len, &response); + free(query); + if (rc != 0) { + fprintf(stderr, "restricted-role protocol probe failed at ABI level: %s\n", oliphaunt_last_error(db)); + return 1; + } + if (!contains_tag(&response, 'E') || + !contains_tag(&response, 'Z') || + !contains_bytes(&response, sqlstate)) { + fprintf( + stderr, + "restricted-role probe did not return ErrorResponse/ReadyForQuery with SQLSTATE %s: %s\n", + sqlstate, + sql); + oliphaunt_free_response(&response); + return 1; + } + oliphaunt_free_response(&response); + return 0; +} + +static int expect_restricted_role_state(OliphauntHandle *db, const char *marker) { + char sql[1024]; + int written = snprintf( + sql, + sizeof(sql), + "SELECT CASE WHEN session_user = 'oliphaunt_broker' " + "AND current_user = 'oliphaunt_broker' " + "AND current_setting('is_superuser') = 'off' " + "AND current_schemas(false) = ARRAY['oliphaunt_broker', 'public']::name[] " + "AND NOT (SELECT rolsuper FROM pg_catalog.pg_roles WHERE rolname = 'oliphaunt_broker') " + "THEN '%s' ELSE 'restricted-role-unsafe' END", + marker); + if (written < 0 || (size_t)written >= sizeof(sql)) { + fprintf(stderr, "restricted-role state query exceeded its fixed buffer\n"); + return 1; + } + return exec_query_expect_bytes(db, sql, marker); +} + +static int probe_restricted_role_boundary(const char *pgdata, const char *runtime_dir) { + static const char *const startup_args[] = { + "-c", + "search_path=\"$user\", public", + }; + OliphauntConfig config = { + .abi_version = OLIPHAUNT_ABI_VERSION, + .pgdata = pgdata, + .runtime_dir = runtime_dir, + .username = "oliphaunt_broker", + .database = "postgres", + .reserved_flags = 0, + .startup_args = startup_args, + .startup_arg_count = sizeof(startup_args) / sizeof(startup_args[0]), + }; + OliphauntHandle *db = NULL; + fprintf(stderr, "opening embedded backend as configured restricted role\n"); + if (oliphaunt_init(&config, &db) != 0 || db == NULL) { + fprintf(stderr, "configured restricted role did not reach ReadyForQuery: %s\n", oliphaunt_last_error(db)); + return 1; + } + if (exec_query_expect_bytes( + db, + "SELECT current_setting('search_path')", + "\"$user\", public") != 0) { + oliphaunt_close(db); + return 1; + } + + const unsigned char command_tags[] = {'C', 'Z'}; + if (exec_query_expect_error( + db, + "BEGIN; " + "ALTER ROLE oliphaunt_broker NOSUPERUSER; " + "SET SESSION AUTHORIZATION oliphaunt_broker; " + "SELECT 1 / 0", + "22012") != 0 || + exec_query_expect_tags(db, "ROLLBACK", command_tags, sizeof(command_tags)) != 0 || + exec_query_expect_bytes( + db, + "SELECT CASE WHEN current_setting('is_superuser') = 'off' " + "AND (SELECT rolsuper FROM pg_catalog.pg_roles WHERE rolname = 'oliphaunt_broker') " + "THEN 'failed-bootstrap-fail-closed' ELSE 'failed-bootstrap-unsafe' END", + "failed-bootstrap-fail-closed") != 0) { + oliphaunt_close(db); + return 1; + } + + fprintf(stderr, "detaching after failed bootstrap and reopening resident backend\n"); + if (oliphaunt_detach(db) != 0) { + fprintf(stderr, "failed-bootstrap detach failed: %s\n", oliphaunt_last_error(db)); + oliphaunt_close(db); + return 1; + } + OliphauntHandle *reopened = NULL; + if (oliphaunt_init(&config, &reopened) != 0 || reopened == NULL) { + fprintf(stderr, "failed-bootstrap resident reopen failed: %s\n", oliphaunt_last_error(NULL)); + oliphaunt_close(db); + return 1; + } + db = reopened; + if (exec_query_expect_bytes( + db, + "SELECT current_setting('search_path')", + "\"$user\", public") != 0) { + oliphaunt_close(db); + return 1; + } + + const char *bootstrap_sql = + "BEGIN; " + "REASSIGN OWNED BY oliphaunt_broker TO postgres; " + "CREATE SCHEMA IF NOT EXISTS oliphaunt_broker AUTHORIZATION oliphaunt_broker; " + "GRANT CONNECT, TEMPORARY ON DATABASE postgres TO oliphaunt_broker; " + "REVOKE CREATE ON DATABASE postgres FROM oliphaunt_broker; " + "GRANT USAGE, CREATE ON SCHEMA oliphaunt_broker TO oliphaunt_broker; " + "REVOKE CREATE ON SCHEMA public FROM PUBLIC, oliphaunt_broker; " + "GRANT USAGE ON SCHEMA public TO oliphaunt_broker; " + "GRANT pg_checkpoint TO oliphaunt_broker; " + "REVOKE EXECUTE ON FUNCTION pg_catalog.pg_relation_filepath(regclass) " + "FROM PUBLIC, oliphaunt_broker; " + "REVOKE EXECUTE ON FUNCTION pg_catalog.pg_tablespace_location(oid) " + "FROM PUBLIC, oliphaunt_broker; " + "ALTER ROLE oliphaunt_broker SET search_path TO \"$user\", public; " + "ALTER ROLE oliphaunt_broker NOSUPERUSER NOCREATEDB NOCREATEROLE " + "INHERIT LOGIN NOREPLICATION NOBYPASSRLS; " + "SET SESSION AUTHORIZATION oliphaunt_broker; " + "SET search_path TO \"$user\", public; " + "COMMIT"; + if (exec_query_expect_tags(db, bootstrap_sql, command_tags, sizeof(command_tags)) != 0) { + oliphaunt_close(db); + return 1; + } + if (exec_query_expect_error(db, "SET ROLE postgres", "42501") != 0) { + oliphaunt_close(db); + return 1; + } + + if (expect_restricted_role_state(db, "restricted-role-safe") != 0 || + exec_query_expect_bytes( + db, + "CREATE TABLE restricted_role_protocol_probe(value text); " + "INSERT INTO restricted_role_protocol_probe VALUES ('restricted-ddl-dml-ok'); " + "SELECT value FROM restricted_role_protocol_probe", + "restricted-ddl-dml-ok") != 0 || + exec_query_expect_tags(db, "CHECKPOINT", command_tags, sizeof(command_tags)) != 0 || + exec_query_expect_error(db, "SHOW data_directory", "42501") != 0 || + exec_query_expect_error(db, "SET SESSION AUTHORIZATION postgres", "42501") != 0 || + exec_query_expect_error(db, "ALTER ROLE oliphaunt_broker SUPERUSER", "42501") != 0 || + exec_query_expect_tags( + db, + "BEGIN; " + "SET LOCAL SESSION AUTHORIZATION oliphaunt_broker; " + "ALTER ROLE oliphaunt_broker PASSWORD NULL; " + "ROLLBACK", + command_tags, + sizeof(command_tags)) != 0 || + expect_restricted_role_state(db, "restricted-rollback-safe") != 0 || + exec_query_expect_tags( + db, + "RESET ROLE; RESET SESSION AUTHORIZATION", + command_tags, + sizeof(command_tags)) != 0 || + exec_query_expect_error(db, "SET ROLE postgres", "42501") != 0 || + expect_restricted_role_state(db, "restricted-reset-safe") != 0 || + exec_query_expect_tags(db, "DISCARD ALL", command_tags, sizeof(command_tags)) != 0 || + exec_query_expect_bytes(db, "SELECT session_user", "oliphaunt_broker") != 0 || + exec_query_expect_bytes(db, "SELECT current_user", "oliphaunt_broker") != 0 || + exec_query_expect_bytes(db, "SELECT current_setting('is_superuser')", "off") != 0 || + exec_query_expect_bytes( + db, + "SELECT current_setting('search_path')", + "\"$user\", public") != 0 || + exec_query_expect_bytes(db, "SELECT current_schemas(false)::text", "{oliphaunt_broker,public}") != 0 || + exec_query_expect_bytes( + db, + "SELECT CASE WHEN NOT (SELECT rolsuper FROM pg_catalog.pg_roles " + "WHERE rolname = 'oliphaunt_broker') THEN 'catalog-nosuper' ELSE 'catalog-super' END", + "catalog-nosuper") != 0 || + expect_restricted_role_state(db, "restricted-discard-safe") != 0 || + exec_query_expect_bytes( + db, + "CREATE TABLE restricted_role_post_discard_probe(value text); " + "INSERT INTO restricted_role_post_discard_probe VALUES ('post-discard-ddl-dml-ok'); " + "SELECT value FROM restricted_role_post_discard_probe", + "post-discard-ddl-dml-ok") != 0 || + exec_query_expect_error(db, "SET ROLE postgres", "42501") != 0 || + exec_query_expect_error(db, "SET SESSION AUTHORIZATION postgres", "42501") != 0) { + oliphaunt_close(db); + return 1; + } + + if (oliphaunt_close(db) != 0) { + fprintf(stderr, "closing restricted-role probe failed: %s\n", oliphaunt_last_error(db)); + return 1; + } + fprintf(stderr, "configured restricted-role protocol probe passed\n"); + return 0; +} + int main(int argc, char **argv) { + if (argc == 4 && strcmp(argv[1], "--expect-root-locked") == 0) { + return expect_root_locked_in_probe_process(argv[2], argv[3]); + } + if (argc == 4 && strcmp(argv[1], "--expect-configured-role-rejected") == 0) { + return expect_configured_role_rejected( + "oliphaunt_missing_authenticated_role", + argv[2], + argv[3]); + } + if (argc == 4 && strcmp(argv[1], "--expect-configured-role-login-rejected") == 0) { + return expect_configured_role_rejected( + "oliphaunt_no_login_authenticated_role", + argv[2], + argv[3]); + } + if (argc == 4 && strcmp(argv[1], "--probe-restricted-role-boundary") == 0) { + return probe_restricted_role_boundary(argv[2], argv[3]); + } if (argc != 3) { - fprintf(stderr, "usage: %s \n", argv[0]); + fprintf( + stderr, + "usage: %s | --expect-configured-role-rejected | --expect-configured-role-login-rejected | --probe-restricted-role-boundary \n", + argv[0]); return 2; } @@ -1970,7 +2385,7 @@ int main(int argc, char **argv) { if (set_pgdata_env_for_smoke(host_pgdata) != 0) { return 1; } - if (run_cycle(argv[1], argv[2]) != 0) { + if (run_cycle(argv[0], argv[1], argv[2]) != 0) { return 1; } if (expect_pgdata_env("oliphaunt_close", host_pgdata) != 0) { diff --git a/src/runtimes/liboliphaunt/native/src/liboliphaunt_native.c b/src/runtimes/liboliphaunt/native/src/liboliphaunt_native.c index f4e4c767..91eaf138 100644 --- a/src/runtimes/liboliphaunt/native/src/liboliphaunt_native.c +++ b/src/runtimes/liboliphaunt/native/src/liboliphaunt_native.c @@ -713,6 +713,21 @@ int32_t oliphaunt_cancel(OliphauntHandle *handle) { if (MyLatch != NULL) { SetLatch(MyLatch); } +#if defined(__ANDROID__) + /* + * Android invokes this ABI from a Binder/JNI thread, while PostgreSQL's + * signal and latch state belongs to the embedded backend pthread. Target + * the backend with PostgreSQL's installed statement-cancel signal so the + * normal handler observes the request in that thread as well. + */ + int signal_rc = pthread_kill(handle->backend_thread, SIGINT); + if (signal_rc != 0) { + snprintf(handle->last_error, sizeof(handle->last_error), + "failed to signal native backend cancellation: %d", signal_rc); + pthread_mutex_unlock(&handle->mutex); + return -1; + } +#endif pthread_cond_broadcast(&handle->input_cond); pthread_cond_broadcast(&handle->output_cond); pthread_mutex_unlock(&handle->mutex); diff --git a/src/runtimes/liboliphaunt/native/src/liboliphaunt_protocol.c b/src/runtimes/liboliphaunt/native/src/liboliphaunt_protocol.c index a4b99e06..b5d0fd59 100644 --- a/src/runtimes/liboliphaunt/native/src/liboliphaunt_protocol.c +++ b/src/runtimes/liboliphaunt/native/src/liboliphaunt_protocol.c @@ -163,7 +163,7 @@ static bool stream_queue_has_room_locked(OliphauntHandle *handle, size_t len, si return true; } if (len > max_bytes) { - return handle->stream_bytes_queued == 0; + return false; } return handle->stream_bytes_queued <= max_bytes - len; } @@ -172,6 +172,11 @@ static int wait_for_stream_queue_room_locked(OliphauntHandle *handle, size_t len size_t max_bytes = handle->stream_queue_max_bytes > 0 ? handle->stream_queue_max_bytes : DEFAULT_STREAM_QUEUE_MAX_BYTES; + if (len > max_bytes) { + set_error(handle, "native liboliphaunt stream chunk exceeds queue limit"); + errno = EOVERFLOW; + return -1; + } while (!stream_queue_has_room_locked(handle, len, max_bytes)) { if (!handle->streaming || handle->stream_failed || handle->backend_exited || handle->closing) { set_error(handle, "native liboliphaunt stream queue closed"); @@ -189,35 +194,43 @@ static int wait_for_stream_queue_room_locked(OliphauntHandle *handle, size_t len } static int enqueue_stream_chunk_locked(OliphauntHandle *handle, const void *buf, size_t len) { - if (len == 0) { - return 0; - } - if (wait_for_stream_queue_room_locked(handle, len) != 0) { - return -1; - } - OliphauntOutputChunk *chunk = (OliphauntOutputChunk *)calloc(1, sizeof(OliphauntOutputChunk)); - if (chunk == NULL) { - set_error(handle, "out of memory enqueuing protocol stream response"); - errno = ENOMEM; - return -1; - } - chunk->data = (unsigned char *)malloc(len); - if (chunk->data == NULL) { - free(chunk); - set_error(handle, "out of memory enqueuing protocol stream response"); - errno = ENOMEM; - return -1; - } - memcpy(chunk->data, buf, len); - chunk->len = len; - if (handle->stream_tail == NULL) { - handle->stream_head = chunk; - handle->stream_tail = chunk; - } else { - handle->stream_tail->next = chunk; - handle->stream_tail = chunk; + const unsigned char *bytes = (const unsigned char *)buf; + size_t max_bytes = handle->stream_queue_max_bytes > 0 + ? handle->stream_queue_max_bytes + : DEFAULT_STREAM_QUEUE_MAX_BYTES; + size_t remaining = len; + while (remaining > 0) { + size_t chunk_len = remaining < max_bytes ? remaining : max_bytes; + if (wait_for_stream_queue_room_locked(handle, chunk_len) != 0) { + return -1; + } + OliphauntOutputChunk *chunk = (OliphauntOutputChunk *)calloc(1, sizeof(OliphauntOutputChunk)); + if (chunk == NULL) { + set_error(handle, "out of memory enqueuing protocol stream response"); + errno = ENOMEM; + return -1; + } + chunk->data = (unsigned char *)malloc(chunk_len); + if (chunk->data == NULL) { + free(chunk); + set_error(handle, "out of memory enqueuing protocol stream response"); + errno = ENOMEM; + return -1; + } + memcpy(chunk->data, bytes, chunk_len); + chunk->len = chunk_len; + if (handle->stream_tail == NULL) { + handle->stream_head = chunk; + handle->stream_tail = chunk; + } else { + handle->stream_tail->next = chunk; + handle->stream_tail = chunk; + } + handle->stream_bytes_queued += chunk_len; + pthread_cond_broadcast(&handle->output_cond); + bytes += chunk_len; + remaining -= chunk_len; } - handle->stream_bytes_queued += len; return 0; } diff --git a/src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs b/src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs index 6568ef53..3d9f5c35 100755 --- a/src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs +++ b/src/runtimes/liboliphaunt/native/tools/check-patch-stack.mjs @@ -202,6 +202,22 @@ const REQUIRED_AUDIT_CHECKS = [ ], posture: 'Embedded backend and extension calls cannot replace or emit host-owned SIGUSR1; other signals delegate to the platform implementation, while frontend tools and normal PostgreSQL builds retain upstream behavior.', }, + { + id: 'embedded-authenticated-role', + requirement: 'Embedded sessions authenticate as the configured database role', + patches: ['0021-liboliphaunt-authenticate-embedded-role.patch'], + evidence: [ + 'MyProcPort->oliphaunt_io != NULL', + 'InitializeSessionUserId(username, useroid, false)', + '!role_form->rolcanlogin', + 'is not permitted to log in', + 'assign_session_authorization', + 'ResetOliphauntAuthenticatedRoleLatch', + 'oliphaunt_authenticated_role_is_superuser = false', + 'InitializeSessionUserIdStandalone', + ], + posture: 'Only the Oliphaunt host-I/O backend resolves its immutable authenticated identity from the configured LOGIN role and latches an observed demotion for that session so stale RESET, rollback, and DISCARD state fail closed without catalog work in GUC cleanup; ordinary standalone PostgreSQL keeps bootstrap-superuser recovery semantics.', + }, ]; const EXPECTED_UPSTREAM_TOUCHPOINTS = new Map([ @@ -220,10 +236,13 @@ const EXPECTED_UPSTREAM_TOUCHPOINTS = new Map([ ['src/backend/storage/ipc/ipc.c', 'Embedded backend cleanup and proc_exit unwinding stay at PostgreSQL lifecycle boundaries.'], ['src/backend/storage/ipc/procsignal.c', 'The one-backend embedded runtime dispatches ProcSignal flags without sending process-directed host signals.'], ['src/backend/tcop/postgres.c', 'Embedded backend entrypoint, protocol lifecycle, cwd restoration, host runtime paths, and host-owned SIGUSR1 disposition.'], + ['src/backend/commands/variable.c', 'Oliphaunt session-authorization assignments monotonically latch an observed authenticated-role demotion without catalog access during transaction cleanup.'], ['src/backend/utils/fmgr/dfmgr.c', 'Static extension lookup reuses PostgreSQL dynamic function manager semantics.'], + ['src/backend/utils/init/postinit.c', 'Oliphaunt host-I/O sessions initialize the immutable authenticated identity from the configured role while ordinary standalone startup remains unchanged.'], ['src/bin/initdb/initdb.c', 'Base runtimes skip ICU-backed collation setup until optional ICU data is present.'], ['src/include/libpq/libpq-be.h', 'Host I/O vtable is attached to PostgreSQL Port state under OLIPHAUNT_EMBEDDED.'], ['src/include/tcop/backend_startup.h', 'Embedded BackendMain may return after its returning PostgresMain call without retaining an invalid pg_noreturn declaration.'], + ['src/include/utils/guc_hooks.h', 'Declares the Oliphaunt-only per-session authenticated-role latch reset used by InitPostgres.'], ['src/include/port.h', 'Embedded mobile builds avoid POSIX shared memory declarations and route embedded backend signal calls through the host-safe provider boundary.'], ['src/include/storage/dsm_impl.h', 'Embedded mobile builds keep DSM on mmap instead of POSIX or SysV shared memory.'], ['src/include/storage/ipc.h', 'Embedded cleanup and proc_exit guard declarations.'], diff --git a/src/runtimes/liboliphaunt/native/tools/run-host-c-smoke.mjs b/src/runtimes/liboliphaunt/native/tools/run-host-c-smoke.mjs index 6b0e037a..7d43a643 100755 --- a/src/runtimes/liboliphaunt/native/tools/run-host-c-smoke.mjs +++ b/src/runtimes/liboliphaunt/native/tools/run-host-c-smoke.mjs @@ -58,8 +58,13 @@ function run(command, args, options = {}) { cwd: options.cwd, env: options.env ?? process.env, encoding: 'utf8', + input: options.input, shell: false, - stdio: options.capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', + stdio: options.capture + ? [options.input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] + : options.input === undefined + ? 'inherit' + : ['pipe', 'inherit', 'inherit'], windowsVerbatimArguments: options.windowsVerbatimArguments ?? false, }); if (result.error) { @@ -366,9 +371,7 @@ function runSmoke(paths, smokeBin, rootArg) { const smokeRoot = process.env.OLIPHAUNT_SMOKE_ROOT ? path.resolve(process.env.OLIPHAUNT_SMOKE_ROOT) : paths.workRoot; - if (!rootArg) { - fs.mkdirSync(smokeRoot, { recursive: true }); - } + fs.mkdirSync(smokeRoot, { recursive: true }); const root = rootArg ? path.resolve(rootArg) : fs.mkdtempSync(path.join(smokeRoot, 'smoke.')); @@ -376,13 +379,75 @@ function runSmoke(paths, smokeBin, rootArg) { const pgdata = path.join(root, '.oliphaunt-pgdata'); const args = [normalizeForC(pgdata), normalizeForC(paths.installDir)]; const env = smokeEnv(paths); + const identityRoot = fs.mkdtempSync(path.join(smokeRoot, 'authenticated-role.')); + const identityPgdata = path.join(identityRoot, 'pgdata'); try { + run(paths.initdb, [ + '-D', + identityPgdata, + '-U', + 'postgres', + '--auth=trust', + '--no-sync', + '--locale-provider=libc', + '--locale=C', + '--encoding=UTF8', + ], { env }); + run(smokeBin, [ + '--expect-configured-role-rejected', + normalizeForC(identityPgdata), + normalizeForC(paths.installDir), + ], { env }); + run(paths.postgres, [ + '--single', + '-D', + identityPgdata, + 'postgres', + ], { + capture: true, + env, + input: [ + 'CREATE ROLE oliphaunt_no_login_authenticated_role SUPERUSER NOLOGIN;', + 'CREATE ROLE oliphaunt_broker SUPERUSER LOGIN;', + 'ALTER ROLE oliphaunt_broker SET search_path TO pg_catalog;', + "ALTER ROLE postgres SET application_name TO 'oliphaunt-role-setting-must-stay-skipped-in-standalone';", + '', + ].join('\n'), + }); + const standaloneSettingsProbe = run(paths.postgres, [ + '--single', + '-D', + identityPgdata, + 'postgres', + ], { + capture: true, + env, + input: [ + "SELECT CASE WHEN current_setting('application_name') <> 'oliphaunt-role-setting-must-stay-skipped-in-standalone' THEN 'oliphaunt-standalone-role-settings-skipped' ELSE 'oliphaunt-standalone-role-settings-loaded' END;", + '', + ].join('\n'), + }); + if (!standaloneSettingsProbe.includes('oliphaunt-standalone-role-settings-skipped')) { + throw new Error('ordinary standalone PostgreSQL unexpectedly loaded pg_db_role_setting'); + } + run(smokeBin, [ + '--expect-configured-role-login-rejected', + normalizeForC(identityPgdata), + normalizeForC(paths.installDir), + ], { env }); + run(smokeBin, [ + '--probe-restricted-role-boundary', + normalizeForC(identityPgdata), + normalizeForC(paths.installDir), + ], { env }); + fs.rmSync(identityRoot, { recursive: true, force: true }); run(smokeBin, args, { env }); run(smokeBin, args, { env }); if (!keepRoot) { fs.rmSync(root, { recursive: true, force: true }); } } catch (error) { + console.error(`authenticated-role smoke root: ${identityRoot}`); console.error(`native smoke root: ${root}`); throw error; } diff --git a/src/sdks/kotlin/build.gradle.kts b/src/sdks/kotlin/build.gradle.kts index c0ee48a8..3f7be4b2 100644 --- a/src/sdks/kotlin/build.gradle.kts +++ b/src/sdks/kotlin/build.gradle.kts @@ -1,7 +1,9 @@ plugins { + alias(libs.plugins.android.application) apply false alias(libs.plugins.android.library) apply false alias(libs.plugins.detekt) apply false alias(libs.plugins.dokka) apply false + alias(libs.plugins.kotlin.android) apply false alias(libs.plugins.kotlin.jvm) apply false alias(libs.plugins.kotlin.multiplatform) apply false alias(libs.plugins.maven.publish) apply false diff --git a/src/sdks/kotlin/gradle/libs.versions.toml b/src/sdks/kotlin/gradle/libs.versions.toml index 2fbf3e6e..fd8893cf 100644 --- a/src/sdks/kotlin/gradle/libs.versions.toml +++ b/src/sdks/kotlin/gradle/libs.versions.toml @@ -10,12 +10,15 @@ dokka = "2.2.0" kover = "0.9.8" [libraries] +kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-android", version.ref = "kotlinx-coroutines" } kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } [plugins] +android-application = { id = "com.android.application", version.ref = "android-gradle-plugin" } android-library = { id = "com.android.library", version.ref = "android-gradle-plugin" } +kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } maven-publish = { id = "com.vanniktech.maven.publish", version.ref = "maven-publish" } diff --git a/src/sdks/kotlin/settings.gradle.kts b/src/sdks/kotlin/settings.gradle.kts index 15bcfa02..7fcd9981 100644 --- a/src/sdks/kotlin/settings.gradle.kts +++ b/src/sdks/kotlin/settings.gradle.kts @@ -40,3 +40,5 @@ rootProject.name = "oliphaunt-kotlin" include(":oliphaunt") include(":oliphaunt-android-gradle-plugin") include(":oliphaunt-maven-artifacts") +include(":android-native-broker-spike") +project(":android-native-broker-spike").projectDir = file("../../../spikes/android-native-broker/app") diff --git a/src/sdks/react-native/tools/mobile-extension-runtime.sh b/src/sdks/react-native/tools/mobile-extension-runtime.sh index f6be35a1..8e7b7b97 100644 --- a/src/sdks/react-native/tools/mobile-extension-runtime.sh +++ b/src/sdks/react-native/tools/mobile-extension-runtime.sh @@ -450,7 +450,11 @@ oliphaunt_dev_installed_runtime_extension_complete() { [ -f "$control_file" ] || return 1 compgen -G "$extension_dir/$extension--*.sql" >/dev/null || return 1 default_version="$(oliphaunt_dev_extension_default_version "$control_file")" - [ -z "$default_version" ] || [ -f "$extension_dir/$extension--$default_version.sql" ] + [ -z "$default_version" ] || + [ -f "$extension_dir/$extension--$default_version.sql" ] || + find "$extension_dir" -maxdepth 1 -type f \ + -name "$extension--*.sql" ! -name "$extension--*--*.sql" \ + -print -quit | grep -q . } oliphaunt_dev_runtime_extension_files() { @@ -467,7 +471,7 @@ oliphaunt_dev_runtime_extension_files() { source_dir="$( oliphaunt_mobile_static_extension_source_dir \ "$root" \ - "$root/target/liboliphaunt-pg18/build" \ + "${OLIPHAUNT_MOBILE_POSTGRES_BUILD_DIR:-$root/target/liboliphaunt-pg18/build}" \ "$extension" )" [ -d "$source_dir" ] || @@ -492,8 +496,14 @@ oliphaunt_dev_runtime_extension_files() { if [ -n "$default_version" ]; then default_install_sql="$source_dir/sql/$extension--$default_version.sql" generated_sql_template="$source_dir/sql/$extension.sql" + # External extensions such as pgvector declare DATA_built and keep their + # canonical install SQL as an unversioned build input. PostgreSQL contrib + # modules may use sql/.sql as a regression test instead, so never + # infer an install script from that filename alone. if ! printf '%s\n' "$sql_files" | grep -Fxq "$default_install_sql" && - [ -f "$generated_sql_template" ]; then + [ -f "$generated_sql_template" ] && + [ -f "$source_dir/Makefile" ] && + grep -Eq '^[[:space:]]*DATA_built[[:space:]]*=' "$source_dir/Makefile"; then sql_files="$( { printf '%s\n' "$sql_files" diff --git a/src/sdks/swift/.release-semantic-inputs.json b/src/sdks/swift/.release-semantic-inputs.json index 27355e82..a74487f5 100644 --- a/src/sdks/swift/.release-semantic-inputs.json +++ b/src/sdks/swift/.release-semantic-inputs.json @@ -94,10 +94,10 @@ "inputs": [ { "path": "tools/release/render_swiftpm_release_package.mjs", - "sha256": "f04e4364aac7c7852ff8957f6f337bff303f133634795ddd88bc85e0cf2ff699" + "sha256": "53db95639e8781b344de445becec25098a8ecf9bd96f61b0ff79858af87b65a0" } ] } ], - "sha256": "036f517f9b6cb431205dce53ba03f7e7bb4c4d12b30a6156524cd3b5878c07cc" + "sha256": "d80a34f4345b7709e6651d11ba25f61b6d1d09a2a6420795e7b7d24f50cacc44" } diff --git a/src/sdks/swift/Package.swift b/src/sdks/swift/Package.swift index 21bf1b80..61efed9e 100644 --- a/src/sdks/swift/Package.swift +++ b/src/sdks/swift/Package.swift @@ -6,12 +6,16 @@ let package = Package( name: "Oliphaunt", platforms: [ .iOS(.v17), - .macOS(.v14) + .macOS(.v14), ], products: [ .library(name: "COliphaunt", targets: ["COliphaunt"]), + .library(name: "OliphauntBrokerProtocol", targets: ["OliphauntBrokerProtocol"]), + .library(name: "OliphauntBrokerXPC", targets: ["OliphauntBrokerXPC"]), + .library(name: "OliphauntIOSBroker", targets: ["OliphauntIOSBroker"]), + .library(name: "OliphauntBrokerExtension", targets: ["OliphauntBrokerExtension"]), .library(name: "Oliphaunt", targets: ["Oliphaunt"]), - .library(name: "OliphauntExtensionSupport", targets: ["OliphauntExtensionSupport"]) + .library(name: "OliphauntExtensionSupport", targets: ["OliphauntExtensionSupport"]), ], dependencies: [ .package(url: "https://github.com/swiftlang/swift-docc-plugin", from: "1.4.0") @@ -21,10 +25,23 @@ let package = Package( name: "COliphaunt", publicHeadersPath: "include" ), + .target(name: "OliphauntBrokerProtocol"), + .target( + name: "OliphauntBrokerXPC", + dependencies: ["OliphauntBrokerProtocol"] + ), .target( name: "Oliphaunt", dependencies: ["COliphaunt"] ), + .target( + name: "OliphauntIOSBroker", + dependencies: ["Oliphaunt", "OliphauntBrokerProtocol", "OliphauntBrokerXPC"] + ), + .target( + name: "OliphauntBrokerExtension", + dependencies: ["COliphaunt", "Oliphaunt", "OliphauntBrokerProtocol"] + ), .target( name: "OliphauntExtensionSupport", dependencies: ["COliphaunt", "Oliphaunt"] @@ -32,6 +49,31 @@ let package = Package( .testTarget( name: "OliphauntTests", dependencies: ["Oliphaunt"] - ) + ), + .testTarget( + name: "OliphauntBrokerProtocolTests", + dependencies: ["OliphauntBrokerProtocol"] + ), + .testTarget( + name: "OliphauntBrokerXPCTests", + dependencies: ["OliphauntBrokerProtocol", "OliphauntBrokerXPC"] + ), + .testTarget( + name: "OliphauntBrokerExtensionTests", + dependencies: [ + "Oliphaunt", + "OliphauntBrokerExtension", + "OliphauntBrokerProtocol", + ] + ), + .testTarget( + name: "OliphauntIOSBrokerTests", + dependencies: [ + "Oliphaunt", + "OliphauntBrokerProtocol", + "OliphauntBrokerXPC", + "OliphauntIOSBroker", + ] + ), ] ) diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/BackendResponseObservation.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/BackendResponseObservation.swift new file mode 100644 index 00000000..a99f02bb --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/BackendResponseObservation.swift @@ -0,0 +1,189 @@ +import Foundation +import OliphauntBrokerProtocol + +enum BrokerBackendTransactionStatus: UInt8, Equatable, Sendable { + case idle = 0x49 + case transaction = 0x54 + case failedTransaction = 0x45 +} + +struct BrokerBackendResponseSnapshot: Equatable, Sendable { + var sawReadyForQuery: Bool + var transactionStatus: BrokerBackendTransactionStatus? + var sawQueryCanceled: Bool +} + +/// Incrementally observes just enough backend protocol to close cancellation +/// races and track transaction ownership. Large row values are skipped rather +/// than accumulated. +private struct BrokerBackendResponseParser { + private static let maximumCapturedErrorBytes = 64 * 1024 + + private var header: [UInt8] = [] + private var currentTag: UInt8? + private var remainingBodyBytes = 0 + private var capturedBody: [UInt8] = [] + private var captureOverflowed = false + private(set) var snapshot = BrokerBackendResponseSnapshot( + sawReadyForQuery: false, + transactionStatus: nil, + sawQueryCanceled: false + ) + + mutating func append(_ data: Data) throws { + try data.withUnsafeBytes { rawBuffer in + let bytes = rawBuffer.bindMemory(to: UInt8.self) + var offset = 0 + while offset < bytes.count { + if currentTag == nil { + let needed = 5 - header.count + let count = min(needed, bytes.count - offset) + header.append(contentsOf: bytes[offset..<(offset + count)]) + offset += count + guard header.count == 5 else { continue } + + let length = + (UInt32(header[1]) << 24) | (UInt32(header[2]) << 16) + | (UInt32(header[3]) << 8) | UInt32(header[4]) + guard length >= 4 else { + throw BrokerError.protocolViolation( + "backend message length is smaller than its header" + ) + } + guard let bodyLength = Int(exactly: length - 4) else { + throw BrokerError.protocolViolation( + "backend message length does not fit this process" + ) + } + currentTag = header[0] + remainingBodyBytes = bodyLength + capturedBody.removeAll(keepingCapacity: true) + captureOverflowed = false + header.removeAll(keepingCapacity: true) + if bodyLength == 0 { + try finishMessage() + } + continue + } + + let count = min(remainingBodyBytes, bytes.count - offset) + if shouldCaptureCurrentMessage, !captureOverflowed { + let available = Self.maximumCapturedErrorBytes - capturedBody.count + if count <= available { + capturedBody.append(contentsOf: bytes[offset..<(offset + count)]) + } else { + if available > 0 { + capturedBody.append(contentsOf: bytes[offset..<(offset + available)]) + } + captureOverflowed = true + } + } + remainingBodyBytes -= count + offset += count + if remainingBodyBytes == 0 { + try finishMessage() + } + } + } + } + + private var shouldCaptureCurrentMessage: Bool { + currentTag == 0x45 || currentTag == 0x5a + } + + private mutating func finishMessage() throws { + guard let tag = currentTag else { return } + defer { + currentTag = nil + remainingBodyBytes = 0 + capturedBody.removeAll(keepingCapacity: true) + captureOverflowed = false + } + + switch tag { + case 0x45 where !captureOverflowed: + if errorSQLState(capturedBody) == "57014" { + snapshot.sawQueryCanceled = true + } + case 0x5a: + guard !captureOverflowed, capturedBody.count == 1, + let status = BrokerBackendTransactionStatus(rawValue: capturedBody[0]) + else { + throw BrokerError.protocolViolation("invalid ReadyForQuery message") + } + snapshot.sawReadyForQuery = true + snapshot.transactionStatus = status + default: + break + } + } + + private func errorSQLState(_ bytes: [UInt8]) -> String? { + var offset = 0 + while offset < bytes.count { + let field = bytes[offset] + offset += 1 + if field == 0 { return nil } + guard let end = bytes[offset...].firstIndex(of: 0) else { return nil } + if field == 0x43 { + return String(bytes: bytes[offset.. BrokerBackendResponseSnapshot { + lock.withResponseLock { parser.snapshot } + } +} + +extension NSLock { + fileprivate func withResponseLock(_ body: () throws -> Result) rethrows -> Result { + lock() + defer { unlock() } + return try body() + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerBackendPrivacyFilter.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerBackendPrivacyFilter.swift new file mode 100644 index 00000000..607f9937 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerBackendPrivacyFilter.swift @@ -0,0 +1,357 @@ +import Foundation + +enum BrokerBackendPrivacyFilterError: Error, Equatable, LocalizedError, Sendable { + case invalidConfiguration + case malformedBackendMessage + case incompleteBackendMessage + case invalidState + + var errorDescription: String? { + switch self { + case .invalidConfiguration: + "invalid backend privacy filter configuration" + case .malformedBackendMessage: + "malformed backend protocol message" + case .incompleteBackendMessage: + "incomplete backend protocol message" + case .invalidState: + "backend privacy filter is no longer available" + } + } +} + +/// Incrementally removes extension-private absolute paths from PostgreSQL +/// ErrorResponse and NoticeResponse messages. Other backend messages are +/// forwarded as their header and incoming body slices without response-wide +/// accumulation. +final class BrokerBackendPrivacyFilter: @unchecked Sendable { + static let maximumBufferedMessageBytes = 64 * 1024 + + private let lock = NSLock() + private var parser: Parser + + init(sensitiveAbsolutePrefixes: [String]) throws { + var variants = Set() + for path in sensitiveAbsolutePrefixes { + guard path.hasPrefix("/"), !path.contains("\0"), path != "/" else { + throw BrokerBackendPrivacyFilterError.invalidConfiguration + } + let url = URL(fileURLWithPath: path) + let candidates = [ + path, + url.standardizedFileURL.path, + url.resolvingSymlinksInPath().standardizedFileURL.path, + ] + guard candidates.allSatisfy({ !$0.isEmpty && $0 != "/" }) else { + throw BrokerBackendPrivacyFilterError.invalidConfiguration + } + variants.formUnion(candidates) + } + guard !variants.isEmpty else { + throw BrokerBackendPrivacyFilterError.invalidConfiguration + } + let prefixes = + variants + .map { Array($0.utf8) } + .sorted { lhs, rhs in + lhs.count == rhs.count + ? lhs.lexicographicallyPrecedes(rhs) + : lhs.count > rhs.count + } + parser = Parser(sensitivePrefixes: prefixes) + } + + func process( + _ data: Data, + emit: (Data) throws -> Void + ) throws { + lock.lock() + defer { lock.unlock() } + do { + try parser.process(data, emit: emit) + } catch { + parser.invalidate() + throw error + } + } + + func finish() throws { + lock.lock() + defer { lock.unlock() } + do { + try parser.finish() + } catch { + parser.invalidate() + throw error + } + } +} + +extension BrokerBackendPrivacyFilter { + fileprivate struct Parser { + private static let errorResponseTag: UInt8 = 0x45 + private static let noticeResponseTag: UInt8 = 0x4e + private static let redaction = Array("[redacted]".utf8) + + private let sensitivePrefixes: [[UInt8]] + private var header: [UInt8] = [] + private var currentTag: UInt8? + private var remainingBodyBytes = 0 + private var bufferedMessage: [UInt8] = [] + private var buffersCurrentMessage = false + private var currentMessageIsOversized = false + private var failed = false + private var finished = false + + init(sensitivePrefixes: [[UInt8]]) { + self.sensitivePrefixes = sensitivePrefixes + header.reserveCapacity(5) + bufferedMessage.reserveCapacity( + BrokerBackendPrivacyFilter.maximumBufferedMessageBytes + ) + } + + mutating func process( + _ data: Data, + emit: (Data) throws -> Void + ) throws { + guard !failed, !finished else { + throw BrokerBackendPrivacyFilterError.invalidState + } + + try data.withUnsafeBytes { rawBuffer in + let bytes = rawBuffer.bindMemory(to: UInt8.self) + var offset = 0 + while offset < bytes.count { + if currentTag == nil { + let count = min(5 - header.count, bytes.count - offset) + header.append(contentsOf: bytes[offset..<(offset + count)]) + offset += count + guard header.count == 5 else { continue } + try beginMessage(emit: emit) + if remainingBodyBytes == 0 { + try completeMessage(emit: emit) + } + continue + } + + let count = min(remainingBodyBytes, bytes.count - offset) + if buffersCurrentMessage { + let available = + BrokerBackendPrivacyFilter.maximumBufferedMessageBytes + - bufferedMessage.count + let capturedCount = min(count, max(0, available)) + if capturedCount > 0 { + bufferedMessage.append( + contentsOf: bytes[offset..<(offset + capturedCount)] + ) + } + } else if count > 0 { + try emit(Data(bytes[offset..<(offset + count)])) + } + remainingBodyBytes -= count + offset += count + if remainingBodyBytes == 0 { + try completeMessage(emit: emit) + } + } + } + } + + mutating func finish() throws { + guard !failed, !finished else { + throw BrokerBackendPrivacyFilterError.invalidState + } + guard header.isEmpty, currentTag == nil else { + failed = true + throw BrokerBackendPrivacyFilterError.incompleteBackendMessage + } + finished = true + } + + mutating func invalidate() { + failed = true + header.removeAll(keepingCapacity: false) + currentTag = nil + remainingBodyBytes = 0 + bufferedMessage.removeAll(keepingCapacity: false) + buffersCurrentMessage = false + currentMessageIsOversized = false + } + + private mutating func beginMessage( + emit: (Data) throws -> Void + ) throws { + let length = + (UInt32(header[1]) << 24) + | (UInt32(header[2]) << 16) + | (UInt32(header[3]) << 8) + | UInt32(header[4]) + guard length >= 4, + let bodyLength = Int(exactly: length - 4) + else { + throw BrokerBackendPrivacyFilterError.malformedBackendMessage + } + + let tag = header[0] + currentTag = tag + remainingBodyBytes = bodyLength + buffersCurrentMessage = + tag == Self.errorResponseTag + || tag == Self.noticeResponseTag + currentMessageIsOversized = + bodyLength + > BrokerBackendPrivacyFilter.maximumBufferedMessageBytes - 5 + + if buffersCurrentMessage { + bufferedMessage.removeAll(keepingCapacity: true) + bufferedMessage.append(contentsOf: header) + } else { + try emit(Data(header)) + } + header.removeAll(keepingCapacity: true) + } + + private mutating func completeMessage( + emit: (Data) throws -> Void + ) throws { + guard let tag = currentTag else { return } + defer { resetMessage() } + guard buffersCurrentMessage else { return } + + let body = bufferedMessage.dropFirst(5) + if currentMessageIsOversized { + try emit(fixedReplacement(tag: tag, capturedBody: body)) + return + } + guard let filtered = filteredMessage(tag: tag, body: body) else { + throw BrokerBackendPrivacyFilterError.malformedBackendMessage + } + try emit(filtered) + } + + private mutating func resetMessage() { + currentTag = nil + remainingBodyBytes = 0 + bufferedMessage.removeAll(keepingCapacity: true) + buffersCurrentMessage = false + currentMessageIsOversized = false + } + + private func filteredMessage(tag: UInt8, body: ArraySlice) -> Data? { + var filteredBody: [UInt8] = [] + filteredBody.reserveCapacity(body.count) + var offset = body.startIndex + var sawTerminator = false + + while offset < body.endIndex { + let fieldTag = body[offset] + offset += 1 + if fieldTag == 0 { + guard offset == body.endIndex else { return nil } + filteredBody.append(0) + sawTerminator = true + break + } + guard let end = body[offset...].firstIndex(of: 0) else { return nil } + let value = body[offset..) -> Bool { + for prefix in sensitivePrefixes where prefix.count <= value.count { + var index = value.startIndex + let finalStart = value.endIndex - prefix.count + while index <= finalStart { + if value[index..<(index + prefix.count)].elementsEqual(prefix) { + return true + } + index += 1 + } + } + return false + } + + private func fixedReplacement(tag: UInt8, capturedBody: ArraySlice) -> Data { + let severity = tag == Self.noticeResponseTag ? "NOTICE" : "ERROR" + let message = + tag == Self.noticeResponseTag + ? "backend notice details redacted" + : "backend error details redacted" + var fields: [(UInt8, [UInt8])] = [ + (0x53, Array(severity.utf8)), + (0x56, Array(severity.utf8)), + ] + if let sqlState = safelyCapturedSQLState(capturedBody) { + fields.append((0x43, sqlState)) + } + fields.append((0x4d, Array(message.utf8))) + return makeMessage(tag: tag, fields: fields) + } + + private func safelyCapturedSQLState(_ body: ArraySlice) -> [UInt8]? { + var offset = body.startIndex + while offset < body.endIndex { + let fieldTag = body[offset] + offset += 1 + if fieldTag == 0 { return nil } + guard let end = body[offset...].firstIndex(of: 0) else { return nil } + if fieldTag == 0x43 { + let value = Array(body[offset.. Data { + var body: [UInt8] = [] + for (fieldTag, value) in fields { + body.append(fieldTag) + body.append(contentsOf: value) + body.append(0) + } + body.append(0) + return makeMessage(tag: tag, body: body) + } + + private func makeMessage(tag: UInt8, body: [UInt8]) -> Data { + let length = UInt32(body.count + 4) + var message = Data([ + tag, + UInt8(truncatingIfNeeded: length >> 24), + UInt8(truncatingIfNeeded: length >> 16), + UInt8(truncatingIfNeeded: length >> 8), + UInt8(truncatingIfNeeded: length), + ]) + message.append(contentsOf: body) + return message + } + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerExtensionStorage.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerExtensionStorage.swift new file mode 100644 index 00000000..69a0d422 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerExtensionStorage.swift @@ -0,0 +1,600 @@ +import CryptoKit +import Foundation +import OliphauntBrokerProtocol + +/// The persistent filesystem layout owned by the broker app-extension process. +/// +/// The host names only the logical root `default`. It never sends a PGDATA path. +/// Resolving that logical name inside the extension is an intentional security and +/// process-isolation boundary. +public struct BrokerExtensionStorage: Equatable, Sendable { + public enum Location: Equatable, Sendable { + case extensionPrivate + case appGroup(identifier: String) + + public var requiresAppGroup: Bool { + if case .appGroup = self { return true } + return false + } + } + + public let location: Location + public let rootURL: URL + + public var manifestURL: URL { + rootURL.appendingPathComponent("manifest.json", isDirectory: false) + } + + public var pgdataURL: URL { + rootURL.appendingPathComponent("pgdata", isDirectory: true) + } + + public var runtimeCacheURL: URL { + rootURL.appendingPathComponent("runtime-cache", isDirectory: true) + } + + public var stagingURL: URL { + rootURL.appendingPathComponent("staging", isDirectory: true) + } + + public init(location: Location, rootURL: URL) throws { + guard rootURL.isFileURL else { + throw BrokerError.invalidConfiguration("broker storage root must be a file URL") + } + let standardized = rootURL.standardizedFileURL + guard !standardized.path.isEmpty, !standardized.path.utf8.contains(0) else { + throw BrokerError.invalidConfiguration("broker storage root is invalid") + } + self.location = location + self.rootURL = standardized + } + + /// Resolves `Library/Application Support/Oliphaunt/default` in the extension's + /// own container. Call this from the app-extension process, not from the host. + public static func extensionPrivate( + fileManager: FileManager = .default + ) throws -> BrokerExtensionStorage { + let applicationSupport = try fileManager.url( + for: .applicationSupportDirectory, + in: .userDomainMask, + appropriateFor: nil, + create: true + ) + return try BrokerExtensionStorage( + location: .extensionPrivate, + rootURL: + applicationSupport + .appendingPathComponent("Oliphaunt", isDirectory: true) + .appendingPathComponent(OliphauntBrokerProtocol.canonicalRootID, isDirectory: true) + ) + } + + /// Explicit fallback for systems where extension-private persistence cannot + /// be made reliable. The caller must hold the matching App Group entitlement. + public static func appGroup( + identifier: String, + fileManager: FileManager = .default + ) throws -> BrokerExtensionStorage { + guard !identifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + !identifier.utf8.contains(0) + else { + throw BrokerError.invalidConfiguration("App Group identifier is invalid") + } + guard + let container = fileManager.containerURL( + forSecurityApplicationGroupIdentifier: identifier + ) + else { + throw BrokerError.invalidConfiguration( + "App Group container is unavailable for \(identifier)" + ) + } + return try BrokerExtensionStorage( + location: .appGroup(identifier: identifier), + rootURL: + container + .appendingPathComponent("Library", isDirectory: true) + .appendingPathComponent("Application Support", isDirectory: true) + .appendingPathComponent("Oliphaunt", isDirectory: true) + .appendingPathComponent(OliphauntBrokerProtocol.canonicalRootID, isDirectory: true) + ) + } + + /// Creates the canonical layout and either validates the durable root + /// manifest or publishes a new one atomically for an empty root. + public func prepare( + postgresMajorVersion: UInt16, + liboliphauntVersion: String, + cABIVersion: UInt32, + selectedPostgresExtensions: [String], + startupConfigurationDigest: String, + dataProtectionPolicy: String = "completeUntilFirstUserAuthentication", + fileManager: FileManager = .default + ) throws -> PreparedBrokerExtensionStorage { + try validateManifestString(liboliphauntVersion, label: "liboliphaunt version") + try validateManifestString( + startupConfigurationDigest, label: "startup configuration digest") + try validateManifestString(dataProtectionPolicy, label: "data-protection policy") + guard dataProtectionPolicy == "completeUntilFirstUserAuthentication" else { + throw BrokerError.invalidConfiguration( + "unsupported broker data-protection policy \(dataProtectionPolicy)" + ) + } + let selectedExtensions = try canonicalExtensions(selectedPostgresExtensions) + + try createProtectedDirectory(rootURL, fileManager: fileManager) + try rejectSymbolicLink(rootURL, fileManager: fileManager) + try createProtectedDirectory(pgdataURL, fileManager: fileManager) + try createProtectedDirectory(runtimeCacheURL, fileManager: fileManager) + try createProtectedDirectory(stagingURL, fileManager: fileManager) + + let expected = ManifestIdentity( + postgresMajorVersion: postgresMajorVersion, + liboliphauntVersion: liboliphauntVersion, + cABIVersion: cABIVersion, + selectedPostgresExtensions: selectedExtensions, + startupConfigurationDigest: startupConfigurationDigest, + dataProtectionPolicy: dataProtectionPolicy + ) + + let manifest: BrokerRootManifest + if fileManager.fileExists(atPath: manifestURL.path) { + manifest = try readManifest(fileManager: fileManager) + try expected.validate(manifest) + } else { + let pgdataEntries = try fileManager.contentsOfDirectory( + at: pgdataURL, + includingPropertiesForKeys: nil, + options: [] + ) + guard pgdataEntries.isEmpty else { + throw BrokerError.invalidConfiguration( + "broker PGDATA exists without manifest.json" + ) + } + manifest = BrokerRootManifest( + postgresMajorVersion: postgresMajorVersion, + liboliphauntVersion: liboliphauntVersion, + cABIVersion: cABIVersion, + rootUUID: UUID(), + selectedPostgresExtensions: selectedExtensions, + startupConfigurationDigest: startupConfigurationDigest, + dataProtectionPolicy: dataProtectionPolicy + ) + try writeManifest(manifest, fileManager: fileManager) + } + + let encoded = try Self.canonicalManifestData(manifest) + return PreparedBrokerExtensionStorage( + storage: self, + manifest: manifest, + manifestDigest: Self.sha256Hex(encoded) + ) + } + + public func readManifest( + fileManager: FileManager = .default + ) throws -> BrokerRootManifest { + try rejectSymbolicLink(manifestURL, fileManager: fileManager) + let attributes = try fileManager.attributesOfItem(atPath: manifestURL.path) + if let size = attributes[.size] as? NSNumber, size.uint64Value > 1024 * 1024 { + throw BrokerError.invalidConfiguration("broker manifest exceeds 1 MiB") + } + do { + let data = try Data(contentsOf: manifestURL, options: []) + let manifest = try JSONDecoder().decode(BrokerRootManifest.self, from: data) + guard manifest.formatVersion == 1 else { + throw BrokerError.invalidConfiguration( + "unsupported broker root manifest format \(manifest.formatVersion)" + ) + } + return manifest + } catch let error as BrokerError { + throw error + } catch { + throw BrokerError.invalidConfiguration( + "cannot decode broker root manifest: \(error)" + ) + } + } + + public func validatePostgresVersion( + _ expectedMajorVersion: UInt16, + fileManager: FileManager = .default + ) throws { + let versionURL = pgdataURL.appendingPathComponent("PG_VERSION", isDirectory: false) + guard fileManager.fileExists(atPath: versionURL.path) else { + throw BrokerError.invalidConfiguration("opened PGDATA has no PG_VERSION") + } + let text = try String(contentsOf: versionURL, encoding: .utf8) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard text == String(expectedMajorVersion) else { + throw BrokerError.invalidConfiguration( + "PGDATA major version \(text) does not match \(expectedMajorVersion)" + ) + } + } + + /// Applies the broker's Data Protection class to every directory and + /// regular file currently owned by this root. + /// + /// Runtime resources and template PGDATA are copied from the signed app + /// bundle. `FileManager.copyItem` preserves their source protection class, + /// so protecting only the destination's top-level directories is not + /// sufficient. Worker startup calls this after runtime hydration and + /// PostgreSQL bootstrap, before publishing Ready. + func enforceDataProtectionRecursively( + fileManager: FileManager = .default + ) throws { + try enforceDataProtectionRecursively(fileManager: fileManager) { url in + try applyDataProtection(to: url, fileManager: fileManager) + } + } + + /// Test seam for proving traversal and ordering on hosts where iOS Data + /// Protection metadata is unavailable. This stays module-internal so URLs + /// cannot cross the broker process boundary. + func enforceDataProtectionRecursively( + fileManager: FileManager, + applyingProtection: (URL) throws -> Void + ) throws { + var enumerationFailed = false + var entries: [URL] = [] + + func validate(_ url: URL) throws { + let attributes = try fileManager.attributesOfItem(atPath: url.path) + guard let type = attributes[.type] as? FileAttributeType, + type == .typeDirectory || type == .typeRegular + else { + throw RecursiveDataProtectionFailure() + } + entries.append(url) + } + + do { + // Preflight the complete tree before the first mutation. A known + // traversal, metadata, symlink, or unsupported-type fault therefore + // cannot leave a partially rewritten protection population. + try validate(rootURL) + guard + let enumerator = fileManager.enumerator( + at: rootURL, + includingPropertiesForKeys: nil, + options: [], + errorHandler: { _, _ in + enumerationFailed = true + return false + } + ) + else { + throw RecursiveDataProtectionFailure() + } + while let item = enumerator.nextObject() { + guard let url = item as? URL else { + throw RecursiveDataProtectionFailure() + } + try validate(url) + } + guard !enumerationFailed else { + throw RecursiveDataProtectionFailure() + } + + // The preflight preserves FileManager's root-first, pre-order + // enumeration, so parent directories are applied before children. + for entry in entries { + try applyingProtection(entry) + } + } catch { + // Never include an underlying Foundation error: it may contain an + // extension-private absolute or relative path. + throw BrokerError.invalidConfiguration( + "cannot enforce broker storage data protection" + ) + } + } + + /// Audits the complete extension-owned root without exposing filesystem + /// names or paths across the process boundary. Call this while the worker is + /// quiesced so PostgreSQL cannot add, remove, or rename entries mid-scan. + public func recursiveProtectionEvidence( + fileManager: FileManager = .default + ) -> BrokerStorageProtectionEvidence { + let expected = FileProtectionType.completeUntilFirstUserAuthentication + var evidence = BrokerStorageProtectionEvidence( + expectedProtection: expected.rawValue + ) + let rootPath = rootURL.standardizedFileURL.path + + func inspect(_ url: URL) { + evidence.entryCount += 1 + do { + let attributes = try fileManager.attributesOfItem(atPath: url.path) + let type = attributes[.type] as? FileAttributeType + switch type { + case .typeDirectory: + evidence.directoryCount += 1 + case .typeRegular: + evidence.regularFileCount += 1 + case .typeSymbolicLink: + evidence.symbolicLinkCount += 1 + default: + evidence.otherEntryCount += 1 + } + + #if os(iOS) && !targetEnvironment(simulator) + let protection = attributes[.protectionKey] as? FileProtectionType + if protection == expected { + evidence.matchingProtectionCount += 1 + } else if protection == nil { + evidence.missingProtectionCount += 1 + } else { + evidence.mismatchedProtectionCount += 1 + } + #else + evidence.protectionMetadataUnavailableCount += 1 + #endif + + guard type == .typeRegular else { return } + if let size = attributes[.size] as? NSNumber { + evidence.regularFileBytes &+= size.uint64Value + } + let relative = url.standardizedFileURL.path.dropFirst(rootPath.count) + .trimmingCharacters(in: CharacterSet(charactersIn: "/")) + let modification = (attributes[.modificationDate] as? Date) + .flatMap(Self.unixNanoseconds) + if relative.hasPrefix("pgdata/base/") { + evidence.relationFileCount += 1 + evidence.newestRelationModificationUnixNanoseconds = max( + evidence.newestRelationModificationUnixNanoseconds ?? 0, + modification ?? 0 + ) + } + if relative.hasPrefix("pgdata/pg_wal/") { + evidence.walFileCount += 1 + evidence.newestWALModificationUnixNanoseconds = max( + evidence.newestWALModificationUnixNanoseconds ?? 0, + modification ?? 0 + ) + } + } catch { + evidence.unreadableEntryCount += 1 + } + } + + inspect(rootURL) + guard + let enumerator = fileManager.enumerator( + at: rootURL, + includingPropertiesForKeys: nil, + options: [], + errorHandler: { _, _ in + evidence.enumerationFailed = true + evidence.unreadableEntryCount += 1 + return false + } + ) + else { + evidence.enumerationFailed = true + return evidence + } + for case let url as URL in enumerator { + inspect(url) + } + return evidence + } + + private static func unixNanoseconds(_ date: Date) -> UInt64? { + let interval = date.timeIntervalSince1970 + guard interval >= 0, interval < Double(UInt64.max) / 1_000_000_000 else { + return nil + } + return UInt64((interval * 1_000_000_000).rounded()) + } + + private func writeManifest( + _ manifest: BrokerRootManifest, + fileManager: FileManager + ) throws { + let data = try Self.canonicalManifestData(manifest) + do { + try data.write(to: manifestURL, options: [.atomic]) + try applyDataProtection(to: manifestURL, fileManager: fileManager) + } catch { + throw BrokerError.invalidConfiguration( + "cannot publish broker root manifest: \(error)" + ) + } + } + + private static func canonicalManifestData(_ manifest: BrokerRootManifest) throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + return try encoder.encode(manifest) + } + + private static func sha256Hex(_ data: Data) -> String { + SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + } + + private func createProtectedDirectory( + _ url: URL, + fileManager: FileManager + ) throws { + if fileManager.fileExists(atPath: url.path) { + try rejectSymbolicLink(url, fileManager: fileManager) + } + try fileManager.createDirectory(at: url, withIntermediateDirectories: true) + try rejectSymbolicLink(url, fileManager: fileManager) + try applyDataProtection(to: url, fileManager: fileManager) + } + + private func applyDataProtection( + to url: URL, + fileManager: FileManager + ) throws { + #if os(iOS) && !targetEnvironment(simulator) + try fileManager.setAttributes( + [.protectionKey: FileProtectionType.completeUntilFirstUserAuthentication], + ofItemAtPath: url.path + ) + #endif + } + + private func rejectSymbolicLink( + _ url: URL, + fileManager: FileManager + ) throws { + guard fileManager.fileExists(atPath: url.path) else { return } + let values = try url.resourceValues(forKeys: [.isSymbolicLinkKey]) + guard values.isSymbolicLink != true else { + throw BrokerError.invalidConfiguration( + "broker storage path must not be a symbolic link: \(url.lastPathComponent)" + ) + } + } +} + +private struct RecursiveDataProtectionFailure: Error {} + +/// Aggregate-only data-protection evidence safe to return to the containing +/// application. It deliberately contains no absolute or relative paths. +public struct BrokerStorageProtectionEvidence: Codable, Equatable, Sendable { + public var expectedProtection: String + public var entryCount = 0 + public var regularFileCount = 0 + public var directoryCount = 0 + public var otherEntryCount = 0 + public var symbolicLinkCount = 0 + public var matchingProtectionCount = 0 + public var missingProtectionCount = 0 + public var mismatchedProtectionCount = 0 + public var protectionMetadataUnavailableCount = 0 + public var unreadableEntryCount = 0 + public var regularFileBytes: UInt64 = 0 + public var relationFileCount = 0 + public var walFileCount = 0 + public var newestRelationModificationUnixNanoseconds: UInt64? + public var newestWALModificationUnixNanoseconds: UInt64? + public var enumerationFailed = false + + public init(expectedProtection: String) { + self.expectedProtection = expectedProtection + } + + public var allEntriesMatchExpectedProtection: Bool { + !enumerationFailed + && unreadableEntryCount == 0 + && missingProtectionCount == 0 + && mismatchedProtectionCount == 0 + && protectionMetadataUnavailableCount == 0 + && symbolicLinkCount == 0 + && entryCount == matchingProtectionCount + } +} + +public struct PreparedBrokerExtensionStorage: Equatable, Sendable { + public let storage: BrokerExtensionStorage + public let manifest: BrokerRootManifest + public let manifestDigest: String + + public init( + storage: BrokerExtensionStorage, + manifest: BrokerRootManifest, + manifestDigest: String + ) { + self.storage = storage + self.manifest = manifest + self.manifestDigest = manifestDigest + } +} + +private struct ManifestIdentity { + let postgresMajorVersion: UInt16 + let liboliphauntVersion: String + let cABIVersion: UInt32 + let selectedPostgresExtensions: [String] + let startupConfigurationDigest: String + let dataProtectionPolicy: String + + func validate(_ manifest: BrokerRootManifest) throws { + guard manifest.formatVersion == 1 else { + throw mismatch("formatVersion", expected: "1", actual: String(manifest.formatVersion)) + } + guard manifest.postgresMajorVersion == postgresMajorVersion else { + throw mismatch( + "postgresMajorVersion", + expected: String(postgresMajorVersion), + actual: String(manifest.postgresMajorVersion) + ) + } + guard manifest.liboliphauntVersion == liboliphauntVersion else { + throw mismatch( + "liboliphauntVersion", + expected: liboliphauntVersion, + actual: manifest.liboliphauntVersion + ) + } + guard manifest.cABIVersion == cABIVersion else { + throw mismatch( + "cABIVersion", + expected: String(cABIVersion), + actual: String(manifest.cABIVersion) + ) + } + guard manifest.selectedPostgresExtensions == selectedPostgresExtensions else { + throw mismatch( + "selectedPostgresExtensions", + expected: selectedPostgresExtensions.joined(separator: ","), + actual: manifest.selectedPostgresExtensions.joined(separator: ",") + ) + } + guard manifest.startupConfigurationDigest == startupConfigurationDigest else { + throw mismatch( + "startupConfigurationDigest", + expected: startupConfigurationDigest, + actual: manifest.startupConfigurationDigest + ) + } + guard manifest.dataProtectionPolicy == dataProtectionPolicy else { + throw mismatch( + "dataProtectionPolicy", + expected: dataProtectionPolicy, + actual: manifest.dataProtectionPolicy + ) + } + } + + private func mismatch(_ field: String, expected: String, actual: String) -> BrokerError { + .invalidConfiguration( + "root manifest \(field) mismatch: expected \(expected), got \(actual)" + ) + } +} + +private func canonicalExtensions(_ extensions: [String]) throws -> [String] { + var result = Set() + for value in extensions { + let name = value.trimmingCharacters(in: .whitespacesAndNewlines) + guard !name.isEmpty, + !name.utf8.contains(0), + name.utf8.allSatisfy({ byte in + (byte >= 65 && byte <= 90) || (byte >= 97 && byte <= 122) + || (byte >= 48 && byte <= 57) || byte == 95 + }) + else { + throw BrokerError.invalidConfiguration( + "invalid PostgreSQL extension identifier \(String(reflecting: value))" + ) + } + result.insert(name) + } + return result.sorted() +} + +private func validateManifestString(_ value: String, label: String) throws { + guard !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + !value.utf8.contains(0) + else { + throw BrokerError.invalidConfiguration("\(label) is invalid") + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerFaultInjection.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerFaultInjection.swift new file mode 100644 index 00000000..583d92db --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerFaultInjection.swift @@ -0,0 +1,157 @@ +import Dispatch +import Foundation +import OliphauntBrokerProtocol + +#if DEBUG + import Darwin + + /// DEBUG-only, one-shot crash and hang hooks for simulator qualification. + /// This type is not present in release builds. + public final class BrokerFaultInjector: @unchecked Sendable { + private struct ArmedFault { + var fault: BrokerWorkerFault + var responseChunkThreshold: Int + var nativeDelay: Duration + } + + private let lock = NSLock() + private var armed: ArmedFault? + private var responseChunks = 0 + + public init() {} + + public func inject( + _ fault: BrokerWorkerFault, + afterResponseChunks: Int = 1, + duringNativeDelay: Duration = .milliseconds(50) + ) { + precondition(afterResponseChunks > 0) + switch fault { + case .abort, .invalidMemoryAccess, .deadlock, .deadlockWithFailStop: + trigger(fault) + default: + lock.withFaultLock { + armed = ArmedFault( + fault: fault, + responseChunkThreshold: afterResponseChunks, + nativeDelay: duringNativeDelay + ) + responseChunks = 0 + } + } + } + + /// Arms the next registered native request to wedge WorkerCore. The + /// InjectFault XPC handler can therefore acknowledge before the actor is + /// deliberately blocked. + func armDeadlockAfterNativeRequestRegistration(failStop: Bool = false) { + lock.withFaultLock { + armed = ArmedFault( + fault: failStop ? .deadlockWithFailStop : .deadlock, + responseChunkThreshold: 1, + nativeDelay: .zero + ) + responseChunks = 0 + } + } + + func beforeNativeDispatch() { + triggerIfArmed(.beforeNativeDispatch) + } + + func afterNativeRequestRegistration() { + triggerIfArmed(.deadlockWithFailStop) + triggerIfArmed(.deadlock) + } + + func beginNativeExecution() -> DispatchWorkItem? { + let delay: Duration? = lock.withFaultLock { + guard armed?.fault == .duringNativeExecution else { return nil } + let value = armed?.nativeDelay + armed = nil + return value + } + guard let delay else { return nil } + let workItem = DispatchWorkItem { trigger(.duringNativeExecution) } + let nanoseconds = max(0, durationNanoseconds(delay)) + DispatchQueue.global(qos: .userInitiated).asyncAfter( + deadline: .now() + .nanoseconds(Int(clamping: nanoseconds)), + execute: workItem + ) + return workItem + } + + func afterResponseChunk() { + let shouldTrigger = lock.withFaultLock { + guard let armed, armed.fault == .afterResponseChunks else { return false } + responseChunks += 1 + if responseChunks >= armed.responseChunkThreshold { + self.armed = nil + return true + } + return false + } + if shouldTrigger { trigger(.afterResponseChunks) } + } + + func afterNativeSuccessBeforeCompleted() { + triggerIfArmed(.afterNativeSuccessBeforeCompleted) + } + + func duringCheckpoint() { + triggerIfArmed(.duringCheckpoint) + } + + private func triggerIfArmed(_ fault: BrokerWorkerFault) { + let shouldTrigger = lock.withFaultLock { + guard armed?.fault == fault else { return false } + armed = nil + return true + } + if shouldTrigger { trigger(fault) } + } + } + + private func trigger(_ fault: BrokerWorkerFault) -> Never { + switch fault { + case .deadlock: + DispatchSemaphore(value: 0).wait() + fatalError("unreachable after broker deadlock fault") + case .deadlockWithFailStop: + DispatchQueue.global(qos: .userInitiated).asyncAfter( + deadline: .now() + .seconds(1) + ) { + Darwin._exit(70) + } + DispatchSemaphore(value: 0).wait() + fatalError("unreachable after broker fail-stop deadlock fault") + case .invalidMemoryAccess: + raise(SIGSEGV) + fatalError("SIGSEGV handler unexpectedly returned") + default: + abort() + } + } + + private func durationNanoseconds(_ duration: Duration) -> Int64 { + let components = duration.components + let seconds = components.seconds.multipliedReportingOverflow(by: 1_000_000_000) + guard !seconds.overflow else { + return components.seconds >= 0 ? Int64.max : Int64.min + } + let attoseconds = components.attoseconds / 1_000_000_000 + let sum = seconds.partialValue.addingReportingOverflow(attoseconds) + guard !sum.overflow else { + return seconds.partialValue >= 0 ? Int64.max : Int64.min + } + return sum.partialValue + } + + extension NSLock { + fileprivate func withFaultLock(_ body: () -> Result) -> Result { + lock() + defer { unlock() } + return body() + } + } +#endif diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerSocketWorker.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerSocketWorker.swift new file mode 100644 index 00000000..c43c1d40 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/BrokerSocketWorker.swift @@ -0,0 +1,439 @@ +import Darwin +import Dispatch +import Foundation +import OliphauntBrokerProtocol + +public enum BrokerSocketError: Error, Equatable, Sendable, CustomStringConvertible { + case invalidFileDescriptor + case notStreamSocket + case alreadyRunning + case stopped + case unexpectedEndOfFile + case systemCall(name: String, code: Int32) + + public var description: String { + switch self { + case .invalidFileDescriptor: "invalid broker socket file descriptor" + case .notStreamSocket: "broker data channel is not a SOCK_STREAM socket" + case .alreadyRunning: "broker socket worker is already running" + case .stopped: "broker socket worker was stopped" + case .unexpectedEndOfFile: "broker socket closed without channelClose" + case .systemCall(let name, let code): + "\(name) failed (errno \(code): \(String(cString: strerror(code))))" + } + } +} + +/// A synchronous, backpressured frame sink over an owned AF_UNIX socket FD. +/// +/// There is no userspace output queue: the native streaming callback blocks on +/// the bounded socket send buffer when the host is slow. This keeps response +/// memory bounded without converting native streaming into whole-result buffering. +public final class BrokerSocketFrameSink: BrokerFrameSink, @unchecked Sendable { + private let stateLock = NSLock() + private let writeLock = NSLock() + private var fileDescriptor: Int32 + private var stopping = false + + public init(ownedFileDescriptor: Int32) throws { + guard ownedFileDescriptor >= 0 else { + throw BrokerSocketError.invalidFileDescriptor + } + do { + var socketType: Int32 = 0 + var socketTypeLength = socklen_t(MemoryLayout.size) + guard + getsockopt( + ownedFileDescriptor, + SOL_SOCKET, + SO_TYPE, + &socketType, + &socketTypeLength + ) == 0 + else { + throw BrokerSocketError.systemCall(name: "getsockopt(SO_TYPE)", code: errno) + } + guard socketType == SOCK_STREAM else { + throw BrokerSocketError.notStreamSocket + } + + let descriptorFlags = fcntl(ownedFileDescriptor, F_GETFD) + guard descriptorFlags >= 0, + fcntl(ownedFileDescriptor, F_SETFD, descriptorFlags | FD_CLOEXEC) == 0 + else { + throw BrokerSocketError.systemCall(name: "fcntl(FD_CLOEXEC)", code: errno) + } + // The host may create both socketpair endpoints as nonblocking before FD + // transfer. This side intentionally uses bounded blocking I/O on its + // private queue, so clear O_NONBLOCK on the received endpoint. + let statusFlags = fcntl(ownedFileDescriptor, F_GETFL) + guard statusFlags >= 0, + fcntl(ownedFileDescriptor, F_SETFL, statusFlags & ~O_NONBLOCK) == 0 + else { + throw BrokerSocketError.systemCall(name: "fcntl(clear O_NONBLOCK)", code: errno) + } + var noSigPipe: Int32 = 1 + guard + setsockopt( + ownedFileDescriptor, + SOL_SOCKET, + SO_NOSIGPIPE, + &noSigPipe, + socklen_t(MemoryLayout.size) + ) == 0 + else { + throw BrokerSocketError.systemCall(name: "setsockopt(SO_NOSIGPIPE)", code: errno) + } + // Keep the kernel buffer comfortably below the 8 MiB userspace target. + // Darwin may adjust/double this value, but this implementation itself + // never holds a second queued copy. + var sendBufferBytes: Int32 = 512 * 1024 + _ = setsockopt( + ownedFileDescriptor, + SOL_SOCKET, + SO_SNDBUF, + &sendBufferBytes, + socklen_t(MemoryLayout.size) + ) + } catch { + _ = Darwin.close(ownedFileDescriptor) + throw error + } + self.fileDescriptor = ownedFileDescriptor + } + + deinit { + requestStop() + finishAndClose() + } + + public func send(_ frame: BrokerFrame) throws { + let bytes = try frame.encoded() + writeLock.lock() + defer { writeLock.unlock() } + + let descriptor = try descriptorForUse() + try bytes.withUnsafeBytes { rawBuffer in + guard let baseAddress = rawBuffer.baseAddress else { return } + var offset = 0 + while offset < rawBuffer.count { + let written = Darwin.send( + descriptor, + baseAddress.advanced(by: offset), + rawBuffer.count - offset, + 0 + ) + if written > 0 { + offset += written + continue + } + if written < 0, errno == EINTR { continue } + let code = written == 0 ? EPIPE : errno + throw BrokerSocketError.systemCall(name: "send", code: code) + } + } + } + + /// Wakes blocked reads/writes without closing the numeric FD out from under + /// an in-progress system call. The owning worker closes it after its loop exits. + public func requestStop() { + let descriptor: Int32? = stateLock.withSocketLock { + guard fileDescriptor >= 0, !stopping else { return nil } + stopping = true + return fileDescriptor + } + if let descriptor { + _ = shutdown(descriptor, SHUT_RDWR) + } + } + + func descriptorForRead() throws -> Int32 { + try descriptorForUse() + } + + func finishAndClose() { + writeLock.lock() + defer { writeLock.unlock() } + let descriptor: Int32? = stateLock.withSocketLock { + guard fileDescriptor >= 0 else { return nil } + let result = fileDescriptor + fileDescriptor = -1 + stopping = true + return result + } + if let descriptor { + _ = Darwin.close(descriptor) + } + } + + private func descriptorForUse() throws -> Int32 { + try stateLock.withSocketLock { + guard fileDescriptor >= 0, !stopping else { + throw BrokerSocketError.stopped + } + return fileDescriptor + } + } +} + +/// Runs bounded blocking socket I/O on a private GCD queue, never the main actor +/// or WorkerCore's actor executor. Frames are handed to WorkerCore serially. +public final class BrokerSocketWorker: @unchecked Sendable { + private let core: WorkerCore + private let sink: BrokerSocketFrameSink + private let epoch: BrokerEpoch + private let protocolVersion: UInt16 + private let ioQueue: DispatchQueue + private let stateLock = NSLock() + private var hasRun = false + private var gracefulStopRequested = false + + public init( + ownedFileDescriptor: Int32, + core: WorkerCore, + epoch: BrokerEpoch, + protocolVersion: UInt16, + queueLabel: String = "dev.oliphaunt.ios-broker.socket" + ) throws { + self.core = core + self.sink = try BrokerSocketFrameSink(ownedFileDescriptor: ownedFileDescriptor) + self.epoch = epoch + self.protocolVersion = protocolVersion + self.ioQueue = DispatchQueue(label: queueLabel, qos: .userInitiated) + } + + deinit { + sink.requestStop() + } + + public func run() async throws { + let mayRun = stateLock.withSocketLock { + guard !hasRun else { return false } + hasRun = true + return true + } + guard mayRun else { throw BrokerSocketError.alreadyRunning } + + try await withCheckedThrowingContinuation { continuation in + ioQueue.async { [self] in + let result: Result + do { + try runBlocking() + result = .success(()) + } catch { + result = .failure(error) + } + continuation.resume(with: result) + } + } + } + + public func stop() { + sink.requestStop() + } + + /// Wakes the worker and treats the externally requested shutdown as a + /// clean detach instead of an interrupted epoch. Extension control glue + /// uses this after accepting a Detach request. + public func stopGracefully() { + stateLock.withSocketLock { + gracefulStopRequested = true + } + sink.requestStop() + } + + private func runBlocking() throws { + var decoder = BrokerSocketIncrementalFrameDecoder( + expectedEpoch: epoch, + expectedProtocolVersion: protocolVersion + ) + var gracefulClose = false + + defer { + sink.requestStop() + sink.finishAndClose() + let shouldDetach = + gracefulClose + || stateLock.withSocketLock { + gracefulStopRequested + } + if shouldDetach { + _ = try? waitForActor { + try await self.core.detach(expectedEpoch: self.epoch) + } + } else { + _ = try? waitForActor { await self.core.interruptCurrentEpoch() } + } + } + + do { + let descriptor = try sink.descriptorForRead() + var readBuffer = [UInt8](repeating: 0, count: 64 * 1024) + while true { + let count = readBuffer.withUnsafeMutableBytes { rawBuffer in + Darwin.recv(descriptor, rawBuffer.baseAddress, rawBuffer.count, 0) + } + if count == 0 { + try decoder.finish() + throw BrokerSocketError.unexpectedEndOfFile + } + if count < 0 { + if errno == EINTR { continue } + if errno == EBADF || errno == ECONNRESET || errno == ENOTCONN { + throw BrokerSocketError.stopped + } + throw BrokerSocketError.systemCall(name: "recv", code: errno) + } + + let frames = try decoder.append(Data(readBuffer[0.. [BrokerFrame] { + var frames: [BrokerFrame] = [] + var offset = 0 + while offset < bytes.count { + if currentHeader == nil { + let needed = Int(OliphauntBrokerProtocol.headerLength) - headerBytes.count + let count = min(needed, bytes.count - offset) + headerBytes.append(bytes.subdata(in: offset..<(offset + count))) + offset += count + guard headerBytes.count == Int(OliphauntBrokerProtocol.headerLength) else { + continue + } + let header = try BrokerFrameHeader.decode( + headerBytes, + expectedEpoch: expectedEpoch, + maximumPayloadLength: OliphauntBrokerProtocol.maximumFramePayload + ) + guard header.protocolVersion == expectedProtocolVersion else { + throw BrokerProtocolError.unsupportedVersion(header.protocolVersion) + } + currentHeader = header + payloadBytes = Data(capacity: Int(header.payloadLength)) + headerBytes.removeAll(keepingCapacity: true) + if header.payloadLength == 0 { + frames.append(try finishFrame()) + } + continue + } + + guard let header = currentHeader else { continue } + let needed = Int(header.payloadLength) - payloadBytes.count + let count = min(needed, bytes.count - offset) + payloadBytes.append(bytes.subdata(in: offset..<(offset + count))) + offset += count + if payloadBytes.count == Int(header.payloadLength) { + frames.append(try finishFrame()) + } + } + return frames + } + + mutating func finish() throws { + guard headerBytes.isEmpty, currentHeader == nil, payloadBytes.isEmpty else { + throw BrokerProtocolError.truncatedFrame + } + } + + private mutating func finishFrame() throws -> BrokerFrame { + guard let header = currentHeader else { + throw BrokerProtocolError.truncatedFrame + } + let frame = try BrokerFrame( + protocolVersion: header.protocolVersion, + frameType: header.frameType, + flags: header.flags, + epoch: header.epoch, + requestID: header.requestID, + payload: payloadBytes + ) + currentHeader = nil + payloadBytes.removeAll(keepingCapacity: false) + return frame + } +} + +private final class BrokerBlockingResult: @unchecked Sendable { + private let lock = NSLock() + private var result: Result? + + func set(_ result: Result) { + lock.withSocketLock { self.result = result } + } + + func take() -> Result { + lock.withSocketLock { + precondition(result != nil) + return result! + } + } +} + +private func waitForActor( + _ operation: @escaping @Sendable () async throws -> Value +) throws -> Value { + let semaphore = DispatchSemaphore(value: 0) + let box = BrokerBlockingResult() + Task.detached { + do { + box.set(.success(try await operation())) + } catch { + box.set(.failure(error)) + } + semaphore.signal() + } + semaphore.wait() + return try box.take().get() +} + +extension NSLock { + fileprivate func withSocketLock(_ body: () throws -> Result) rethrows -> Result { + lock() + defer { unlock() } + return try body() + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/CancellationController.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/CancellationController.swift new file mode 100644 index 00000000..1d0f1e2a --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/CancellationController.swift @@ -0,0 +1,225 @@ +import Foundation +import Oliphaunt +import OliphauntBrokerProtocol + +/// Result of attempting the low-latency cancellation path. +public enum BrokerCancellationSignalResult: Equatable, Sendable { + /// The request is not the request currently executing in native code. + case notRunning + /// This request already has a native cancellation signal in flight. + case alreadyRequested + /// `OliphauntSession.cancel()` accepted the signal. + /// + /// This is deliberately not called "observed". Observation is reported only + /// after the backend emits SQLSTATE 57014 for this request. + case signalSent +} + +public struct BrokerCancellationSnapshot: Equatable, Sendable { + public var requested: Bool + public var signalSent: Bool + public var observedByBackend: Bool + + public init( + requested: Bool, + signalSent: Bool, + observedByBackend: Bool + ) { + self.requested = requested + self.signalSent = signalSent + self.observedByBackend = observedByBackend + } +} + +/// A cancellation fast path that is intentionally independent of `WorkerCore`. +/// +/// XPC glue should call this object directly from its control-message handler. +/// The native-direct session's cancellation witness bypasses the serialized +/// request actor, so a long-running PostgreSQL call cannot starve cancellation. +public final class CancellationController: @unchecked Sendable { + private struct RunningRequest { + let epoch: BrokerEpoch + let requestID: BrokerRequestID + let session: any OliphauntSession + var nativeTransportActive: Bool + var requested: Bool + var signalSent: Bool + var observedByBackend: Bool + } + + private let lock = NSLock() + private var running: RunningRequest? + + public init() {} + + public var activeRequest: (epoch: BrokerEpoch, requestID: BrokerRequestID)? { + lock.withBrokerLock { + running.map { ($0.epoch, $0.requestID) } + } + } + + /// Installs the only native request that may receive an out-of-band cancel. + /// WorkerCore calls this immediately before native dispatch. + func beginNativeRequest( + epoch: BrokerEpoch, + requestID: BrokerRequestID, + session: any OliphauntSession + ) throws { + try lock.withBrokerLock { + guard running == nil else { + throw BrokerError.protocolViolation( + "cancellation target replaced while another request is running" + ) + } + running = RunningRequest( + epoch: epoch, + requestID: requestID, + session: session, + nativeTransportActive: true, + requested: false, + signalSent: false, + observedByBackend: false + ) + } + } + + /// Signals native cancellation without ever entering WorkerCore's actor. + public func requestCancellation( + epoch: BrokerEpoch, + requestID: BrokerRequestID + ) async throws -> BrokerCancellationSignalResult { + let session: (any OliphauntSession)? = lock.withBrokerLock { + guard var target = running, + target.epoch == epoch, + target.requestID == requestID, + target.nativeTransportActive + else { + return nil + } + guard !target.requested else { + return nil + } + target.requested = true + running = target + return target.session + } + + guard let session else { + return lock.withBrokerLock { + guard let target = running, + target.epoch == epoch, + target.requestID == requestID, + target.nativeTransportActive, + target.requested + else { + return .notRunning + } + return .alreadyRequested + } + } + + do { + try await session.cancel() + lock.withBrokerLock { + guard var target = running, + target.epoch == epoch, + target.requestID == requestID + else { return } + target.signalSent = true + running = target + } + return .signalSent + } catch { + lock.withBrokerLock { + guard var target = running, + target.epoch == epoch, + target.requestID == requestID + else { return } + target.requested = false + running = target + } + throw error + } + } + + /// Closes the cancellation race as soon as a complete ReadyForQuery is seen, + /// before a potentially backpressured socket writer returns to native code. + func markNativeTransportComplete( + epoch: BrokerEpoch, + requestID: BrokerRequestID + ) { + lock.withBrokerLock { + guard var target = running, + target.epoch == epoch, + target.requestID == requestID + else { return } + target.nativeTransportActive = false + running = target + } + } + + /// SQLSTATE 57014 is the proof available from the current native API that + /// PostgreSQL observed the cancellation request. + func markCancellationObserved( + epoch: BrokerEpoch, + requestID: BrokerRequestID + ) { + lock.withBrokerLock { + guard var target = running, + target.epoch == epoch, + target.requestID == requestID + else { return } + target.observedByBackend = true + running = target + } + } + + @discardableResult + func finishNativeRequest( + epoch: BrokerEpoch, + requestID: BrokerRequestID + ) -> BrokerCancellationSnapshot { + lock.withBrokerLock { + guard let target = running, + target.epoch == epoch, + target.requestID == requestID + else { + return BrokerCancellationSnapshot( + requested: false, + signalSent: false, + observedByBackend: false + ) + } + running = nil + return BrokerCancellationSnapshot( + requested: target.requested, + signalSent: target.signalSent, + observedByBackend: target.observedByBackend + ) + } + } + + func abandonNativeRequest( + epoch: BrokerEpoch, + requestID: BrokerRequestID + ) { + lock.withBrokerLock { + guard let target = running, + target.epoch == epoch, + target.requestID == requestID + else { return } + running = nil + } + } +} + +extension NSLock { + @discardableResult + fileprivate func withBrokerLock( + _ body: () throws -> Result + ) rethrows -> Result { + lock() + defer { unlock() } + return try body() + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerExtension/WorkerCore.swift b/src/sdks/swift/Sources/OliphauntBrokerExtension/WorkerCore.swift new file mode 100644 index 00000000..2a02755c --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerExtension/WorkerCore.swift @@ -0,0 +1,1595 @@ +import COliphaunt +import Darwin +import Foundation +import Oliphaunt +import OliphauntBrokerProtocol + +public protocol BrokerFrameSink: Sendable { + func send(_ frame: BrokerFrame) throws +} + +public enum BrokerFrameHandlingResult: Equatable, Sendable { + case continueReading + case closeChannel +} + +public enum BrokerWorkerCoreState: Equatable, Sendable { + case created + case starting + case ready + case quiescing + case interrupted + case detached + case failed(String) +} + +public enum BrokerWorkerCancellationDisposition: Equatable, Sendable { + case notCurrent + case canceledBeforeNativeDispatch + case nativeSignal(BrokerCancellationSignalResult) +} + +public struct BrokerWorkerDiagnostics: Equatable, Sendable { + public var state: BrokerWorkerCoreState + public var epoch: BrokerEpoch + public var processID: Int32 + public var rootURL: URL + public var manifestDigest: String? + public var activeRequestID: BrokerRequestID? + public var nativeDispatchStarted: Bool + public var transactionStatus: String + public var capabilities: BrokerCapabilities + + public init( + state: BrokerWorkerCoreState, + epoch: BrokerEpoch, + processID: Int32, + rootURL: URL, + manifestDigest: String?, + activeRequestID: BrokerRequestID?, + nativeDispatchStarted: Bool, + transactionStatus: String, + capabilities: BrokerCapabilities + ) { + self.state = state + self.epoch = epoch + self.processID = processID + self.rootURL = rootURL + self.manifestDigest = manifestDigest + self.activeRequestID = activeRequestID + self.nativeDispatchStarted = nativeDispatchStarted + self.transactionStatus = transactionStatus + self.capabilities = capabilities + } +} + +public struct BrokerWorkerConfiguration: Sendable { + public static let restrictedRoleUsername = "oliphaunt_broker" + public static let restrictedDatabase = "postgres" + public static let restrictedSearchPath = "\"$user\", public" + + public let storage: BrokerExtensionStorage + public let engine: any OliphauntEngine + public let liboliphauntVersion: String + public let cABIVersion: UInt32 + public let postgresMajorVersion: UInt16 + public let startupConfigurationDigest: String + public let selectedPostgresExtensions: [String] + public let durability: OliphauntDurability + public let startupGUCs: [OliphauntStartupGUC] + public let username: String + public let database: String + public let dataProtectionPolicy: String + public let maximumRequestBytes: Int + public let capabilities: BrokerCapabilities + public let runtimeVersionProvider: @Sendable () throws -> String + + public init( + storage: BrokerExtensionStorage, + engine: any OliphauntEngine, + liboliphauntVersion: String, + cABIVersion: UInt32 = UInt32(OLIPHAUNT_ABI_VERSION), + postgresMajorVersion: UInt16 = 18, + startupConfigurationDigest: String, + selectedPostgresExtensions: [String] = [], + durability: OliphauntDurability = .safe, + startupGUCs: [OliphauntStartupGUC] = [], + username: String = Self.restrictedRoleUsername, + database: String = Self.restrictedDatabase, + dataProtectionPolicy: String = "completeUntilFirstUserAuthentication", + maximumRequestBytes: Int = OliphauntBrokerProtocol.defaultMaximumRequestBytes, + capabilities: BrokerCapabilities? = nil, + runtimeVersionProvider: (@Sendable () throws -> String)? = nil + ) throws { + guard maximumRequestBytes >= 5, + maximumRequestBytes <= OliphauntBrokerProtocol.maximumQueuedBytesPerDirection + else { + throw BrokerError.invalidConfiguration( + "maximum request size must be between 5 and \(OliphauntBrokerProtocol.maximumQueuedBytesPerDirection) bytes" + ) + } + guard !liboliphauntVersion.isEmpty, + !startupConfigurationDigest.isEmpty, + !username.isEmpty, + !database.isEmpty + else { + throw BrokerError.invalidConfiguration("worker identity fields must not be empty") + } + guard username == Self.restrictedRoleUsername, + database == Self.restrictedDatabase + else { + throw BrokerError.invalidConfiguration( + "worker must authenticate as the restricted broker role in the postgres database" + ) + } + guard + !startupGUCs.contains(where: { + $0.name.trimmingCharacters(in: .whitespacesAndNewlines) + .caseInsensitiveCompare("search_path") == .orderedSame + }) + else { + throw BrokerError.invalidConfiguration( + "worker search_path is fixed by the restricted broker boundary" + ) + } + + var actual = + capabilities + ?? BrokerCapabilities( + crashRestartable: true, + requiresAppGroup: storage.location.requiresAppGroup + ) + // These are invariants of the v1 extension worker, not caller options. + actual.mode = "nativeBroker" + actual.implementation = "iosExtensionBroker" + actual.minimumOS = "iOS 26" + actual.processIsolated = true + actual.crashRestartable = true + actual.hangRestartable = false + actual.sameRootLogicalReopen = true + actual.rootSwitchable = false + actual.multiRoot = false + actual.independentSessions = false + actual.maxClientSessions = 1 + actual.backgroundContinuable = false + actual.requiresAppGroup = storage.location.requiresAppGroup + actual.protocolRaw = true + actual.protocolStream = true + actual.streamingRequestInput = false + actual.queryCancel = true + actual.backupRestore = false + actual.connectionString = nil + actual.serverMode = false + + self.storage = storage + self.engine = engine + self.liboliphauntVersion = liboliphauntVersion + self.cABIVersion = cABIVersion + self.postgresMajorVersion = postgresMajorVersion + self.startupConfigurationDigest = startupConfigurationDigest + self.selectedPostgresExtensions = selectedPostgresExtensions + self.durability = durability + self.startupGUCs = + startupGUCs + [ + OliphauntStartupGUC("search_path", Self.restrictedSearchPath) + ] + self.username = username + self.database = database + self.dataProtectionPolicy = dataProtectionPolicy + self.maximumRequestBytes = maximumRequestBytes + self.capabilities = actual + self.runtimeVersionProvider = runtimeVersionProvider ?? { liboliphauntVersion } + } + + /// Constructs the production worker engine with runtime materialization kept + /// under the extension-private root. + public static func nativeDirect( + storage: BrokerExtensionStorage, + liboliphauntVersion: String, + startupConfigurationDigest: String, + selectedPostgresExtensions: [String] = [], + durability: OliphauntDurability = .safe, + startupGUCs: [OliphauntStartupGUC] = [], + username: String = Self.restrictedRoleUsername, + database: String = Self.restrictedDatabase + ) throws -> BrokerWorkerConfiguration { + guard + let resources = try OliphauntRuntimeResources.bundled( + containing: selectedPostgresExtensions, + cacheRoot: storage.runtimeCacheURL + ) + else { + throw BrokerError.invalidConfiguration( + "broker app extension does not contain packaged Oliphaunt runtime resources" + ) + } + return try BrokerWorkerConfiguration( + storage: storage, + engine: OliphauntNativeDirectEngine(runtimeResources: resources), + liboliphauntVersion: liboliphauntVersion, + startupConfigurationDigest: startupConfigurationDigest, + selectedPostgresExtensions: selectedPostgresExtensions, + durability: durability, + startupGUCs: startupGUCs, + username: username, + database: database, + runtimeVersionProvider: linkedOliphauntVersion + ) + } +} + +/// Owns the single physical native session in the app-extension process. +/// +/// The actor is deliberately reentrancy-safe: request admission is guarded by an +/// explicit lifecycle even while the native session call is suspended. Running +/// cancellation is delegated to `cancellationController`, which XPC glue can call +/// without entering this actor. +public actor WorkerCore { + private struct ActiveRequest: Sendable { + var lifecycle: BrokerRequestLifecycle + var assembler: BrokerFrontendRequestAssembler + } + + public private(set) var epoch: BrokerEpoch + public nonisolated let cancellationController: CancellationController + + private let configuration: BrokerWorkerConfiguration + private var state: BrokerWorkerCoreState = .created + private var preparedStorage: PreparedBrokerExtensionStorage? + private var selectedProtocolVersion: UInt16? + private var session: (any OliphauntSession)? + private var sessionCloseInProgress = false + private var deadlineRecoveryID: UUID? + private var startAttemptID: UUID? + private var activeRequest: ActiveRequest? + private var acceptingRequests = false + private var transactionStatus: BrokerBackendTransactionStatus = .idle + + #if DEBUG + private nonisolated let faultInjector = BrokerFaultInjector() + #endif + + public init(configuration: BrokerWorkerConfiguration) { + self.configuration = configuration + self.epoch = .fresh() + self.cancellationController = CancellationController() + } + + public func start(hello: BrokerHello) async throws -> BrokerReady { + let previousState = state + let previousEpoch = epoch + switch state { + case .created: + break + case .detached, .interrupted: + guard activeRequest == nil, + session == nil, + !sessionCloseInProgress, + startAttemptID == nil + else { + throw BrokerError.workerInterrupted(epoch: epoch) + } + epoch = .fresh() + case .ready, .quiescing, .starting: + throw BrokerError.rejected(.invalidRequest("a data channel is already active")) + case .failed(let reason): + throw BrokerError.brokerUnavailableWithReason(reason) + } + + let attemptID = UUID() + startAttemptID = attemptID + state = .starting + acceptingRequests = false + + do { + let version = try BrokerHandshake.validate( + hello, + actualABI: configuration.cABIVersion, + actualRuntimeVersion: configuration.liboliphauntVersion, + residentRootID: OliphauntBrokerProtocol.canonicalRootID, + startupConfigurationDigest: configuration.startupConfigurationDigest + ) + try validateRequestedCapabilities(hello.requestedCapabilities) + let prepared: PreparedBrokerExtensionStorage + do { + prepared = try configuration.storage.prepare( + postgresMajorVersion: configuration.postgresMajorVersion, + liboliphauntVersion: configuration.liboliphauntVersion, + cABIVersion: configuration.cABIVersion, + selectedPostgresExtensions: configuration.selectedPostgresExtensions, + startupConfigurationDigest: configuration.startupConfigurationDigest, + dataProtectionPolicy: configuration.dataProtectionPolicy + ) + } catch { + throw BrokerError.rejected(.rootOpen) + } + + let nativeConfiguration = OliphauntConfiguration( + mode: .nativeDirect, + root: configuration.storage.rootURL, + durability: configuration.durability, + runtimeFootprint: .smallMobile, + startupGUCs: configuration.startupGUCs, + username: configuration.username, + database: configuration.database, + extensions: prepared.manifest.selectedPostgresExtensions + ) + let opened: any OliphauntSession + do { + opened = try await configuration.engine.open(configuration: nativeConfiguration) + } catch { + throw BrokerError.rejected(.rootOpen) + } + do { + try validateStartAttempt(attemptID) + do { + try await establishRestrictedRoleBoundary( + opened, + selectedExtensions: prepared.manifest.selectedPostgresExtensions + ) + } catch { + // ALTER ROLE is transactional. A post-demotion validation + // failure must explicitly abandon the transaction before + // this session is closed so no partial bootstrap can be + // observed by a later resident lease. + try? await executeControlSQL("ROLLBACK", session: opened) + throw BrokerError.brokerUnavailable + } + try validateStartAttempt(attemptID) + let actualRuntimeVersion: String + do { + actualRuntimeVersion = try configuration.runtimeVersionProvider() + } catch { + throw BrokerError.brokerUnavailable + } + guard actualRuntimeVersion == configuration.liboliphauntVersion else { + throw BrokerError.runtimeMismatch( + expected: configuration.liboliphauntVersion, + actual: actualRuntimeVersion + ) + } + do { + try await validateNativeCapabilities(opened) + } catch { + throw BrokerError.brokerUnavailable + } + try validateStartAttempt(attemptID) + do { + try await healthCheck(opened) + } catch { + throw BrokerError.brokerUnavailable + } + try validateStartAttempt(attemptID) + do { + try configuration.storage.validatePostgresVersion( + configuration.postgresMajorVersion + ) + } catch { + throw BrokerError.rejected(.rootOpen) + } + do { + // Runtime/template resources are copied before native open, + // and bootstrap may create additional PGDATA files. Repair + // that complete population only after all startup writes, + // then fail closed on an immediate iOS audit before Ready. + try configuration.storage.enforceDataProtectionRecursively() + #if os(iOS) && !targetEnvironment(simulator) + guard + configuration.storage.recursiveProtectionEvidence() + .allEntriesMatchExpectedProtection + else { + throw BrokerError.rejected(.rootOpen) + } + #endif + } catch { + // Do not expose extension-private paths carried by a + // traversal or filesystem error across the broker boundary. + throw BrokerError.rejected(.rootOpen) + } + } catch { + try? await opened.close() + throw error + } + + try validateStartAttempt(attemptID) + preparedStorage = prepared + selectedProtocolVersion = version + session = opened + transactionStatus = .idle + acceptingRequests = true + startAttemptID = nil + state = .ready + return readyValue() + } catch { + if startAttemptID == attemptID { + startAttemptID = nil + if state == .starting { + // Handshake/configuration failures are scoped to this + // incoming generation. They must not poison the resident + // process: the host may correct the mismatch and issue a + // fresh Hello over the same ExtensionKit process. Any + // session opened by this attempt was closed above before + // the stable pre-start state and epoch are restored. + epoch = previousEpoch + state = previousState + } + } + throw error + } + } + + public func handle( + _ frame: BrokerFrame, + sink: any BrokerFrameSink + ) async throws -> BrokerFrameHandlingResult { + guard frame.header.epoch == epoch else { + throw BrokerProtocolError.staleEpoch(expected: epoch, actual: frame.header.epoch) + } + if let selectedProtocolVersion, + frame.header.protocolVersion != selectedProtocolVersion + { + throw BrokerProtocolError.unsupportedVersion(frame.header.protocolVersion) + } + + switch frame.header.frameType { + case .ping: + guard frame.payload.isEmpty else { + return try protocolViolation("ping payload must be empty", sink: sink) + } + try sink.send(try makeFrame(.pong)) + return .continueReading + + case .channelClose: + guard frame.payload.isEmpty else { + return try protocolViolation("channelClose payload must be empty", sink: sink) + } + acceptingRequests = false + return .closeChannel + + case .requestBegin: + try beginRequest(frame, sink: sink) + return .continueReading + + case .requestBytes: + try appendRequestBytes(frame, sink: sink) + return .continueReading + + case .requestEnd: + try await finishAndExecuteRequest(frame, sink: sink) + return .continueReading + + case .cancelRequested: + let requestID = try BrokerRequestID(validating: frame.header.requestID) + _ = try await cancelRequest(epoch: epoch, requestID: requestID) + return .continueReading + + case .responseBytes, .completed, .rejected, .outcomeUnknown, + .cancelObserved, .pong, .protocolError: + return try protocolViolation( + "host sent extension-only frame \(frame.header.frameType)", + sink: sink + ) + } + } + + /// Handles pre-dispatch cancellation and provides a convenience path for + /// running work. XPC should normally call `cancellationController` directly + /// first so running cancellation never waits behind this actor. + public func cancelRequest( + epoch requestedEpoch: BrokerEpoch, + requestID: BrokerRequestID + ) async throws -> BrokerWorkerCancellationDisposition { + guard requestedEpoch == epoch, + var request = activeRequest, + request.lifecycle.requestID == requestID + else { + return .notCurrent + } + + if !request.lifecycle.nativeDispatchStarted { + _ = request.lifecycle.requestCancellation() + // Keep a canceled receiving request installed until RequestEnd. + // The host may already have queued additional SOCK_STREAM bytes; + // discarding those frames preserves channel synchronization while + // still proving that the request never reached liboliphaunt. + activeRequest = request + return .canceledBeforeNativeDispatch + } + + if case .running = request.lifecycle.state { + _ = request.lifecycle.requestCancellation() + activeRequest = request + } + let result = try await cancellationController.requestCancellation( + epoch: requestedEpoch, + requestID: requestID + ) + return .nativeSignal(result) + } + + public func checkpoint(expectedEpoch: BrokerEpoch) async throws { + try validateExpectedEpoch(expectedEpoch) + guard state == .ready || state == .quiescing else { + throw BrokerError.databaseClosed + } + guard activeRequest == nil else { + throw BrokerError.rejected( + .invalidRequest("cannot checkpoint while a request is active")) + } + guard let session else { throw BrokerError.databaseClosed } + #if DEBUG + faultInjector.duringCheckpoint() + #endif + try await executeControlSQL("CHECKPOINT", session: session) + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [.ready, .quiescing] + ) + } + + public func prepareForBackground( + expectedEpoch: BrokerEpoch, + deadline: Date + ) async throws -> OliphauntBackgroundPreparationResult { + try validateExpectedEpoch(expectedEpoch) + guard state == .ready || state == .quiescing else { + throw BrokerError.databaseClosed + } + state = .quiescing + acceptingRequests = false + + var canceledActiveWork = false + if let request = activeRequest { + canceledActiveWork = true + if request.lifecycle.nativeDispatchStarted { + _ = try? await cancellationController.requestCancellation( + epoch: expectedEpoch, + requestID: request.lifecycle.requestID + ) + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [.quiescing] + ) + } else { + var canceled = request + _ = canceled.lifecycle.requestCancellation() + activeRequest = nil + } + } + + while activeRequest != nil, Date() < deadline { + try await Task.sleep(for: .milliseconds(10)) + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [.quiescing] + ) + } + guard activeRequest == nil else { + return OliphauntBackgroundPreparationResult( + cancelledActiveWork: canceledActiveWork, + checkpointed: false, + skippedCheckpointReason: .activeWork + ) + } + guard Date() < deadline, let session else { + return OliphauntBackgroundPreparationResult( + cancelledActiveWork: canceledActiveWork, + checkpointed: false, + skippedCheckpointReason: .activeWork + ) + } + + // Keep a small reply/serialization reserve inside the caller's hard + // deadline. The host owns the outer transport timer; the worker never + // begins unbounded native control work at that boundary. + let controlDeadline = deadline.addingTimeInterval(-0.25) + if transactionStatus != .idle { + guard Date() < controlDeadline else { + beginDeadlineRecovery( + session: session, + operation: nil, + expectedEpoch: expectedEpoch + ) + throw BrokerError.workerInterrupted(epoch: expectedEpoch) + } + try await executeControlSQL( + "ROLLBACK", + session: session, + completingBefore: controlDeadline, + expectedEpoch: expectedEpoch + ) + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [.quiescing] + ) + transactionStatus = .idle + } + guard Date() < controlDeadline else { + return OliphauntBackgroundPreparationResult( + cancelledActiveWork: canceledActiveWork, + checkpointed: false, + skippedCheckpointReason: .activeWork + ) + } + + #if DEBUG + faultInjector.duringCheckpoint() + #endif + try await executeControlSQL( + "CHECKPOINT", + session: session, + completingBefore: controlDeadline, + expectedEpoch: expectedEpoch + ) + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [.quiescing] + ) + return OliphauntBackgroundPreparationResult( + cancelledActiveWork: canceledActiveWork, + checkpointed: true + ) + } + + public func resumeFromBackground(expectedEpoch: BrokerEpoch) async throws { + try validateExpectedEpoch(expectedEpoch) + guard state == .quiescing, let session else { + throw BrokerError.databaseClosed + } + do { + try await healthCheck(session) + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [.quiescing] + ) + acceptingRequests = true + state = .ready + } catch { + if epoch == expectedEpoch, state == .quiescing { + acceptingRequests = false + state = .failed(String(describing: error)) + } + throw error + } + } + + /// Marks the current channel stale. The XPC interruption handler should call + /// the CancellationController directly first, close the FD, then call here. + public func interruptCurrentEpoch() async { + acceptingRequests = false + state = .interrupted + if var request = activeRequest { + _ = request.lifecycle.establishTerminal(request.lifecycle.lossResult()) + if request.lifecycle.nativeDispatchStarted { + // Native work may already have committed. Keep its terminal + // outcome installed until the suspended execution unwinds so a + // replacement epoch can never overlap or replay it. + activeRequest = request + return + } + + // The channel is already gone, so unlike an in-band cancellation + // there are no remaining upload frames to drain. This request is + // proven not-started and must not pin the resident worker forever. + activeRequest = nil + } + await closeCurrentSessionForRecovery() + } + + public func detach(expectedEpoch: BrokerEpoch) async throws { + try validateExpectedEpoch(expectedEpoch) + acceptingRequests = false + guard state == .ready || state == .quiescing else { + throw BrokerError.databaseClosed + } + guard activeRequest == nil else { + throw BrokerError.rejected(.invalidRequest("cannot detach while a request is active")) + } + let detachState = state + if let session { + guard !sessionCloseInProgress else { + throw BrokerError.workerInterrupted(epoch: expectedEpoch) + } + sessionCloseInProgress = true + do { + try await session.close() + } catch { + sessionCloseInProgress = false + if epoch == expectedEpoch, state == .interrupted { + await closeCurrentSessionForRecovery() + } + throw error + } + sessionCloseInProgress = false + do { + try validateControlContinuation( + expectedEpoch: expectedEpoch, + allowedStates: [detachState] + ) + } catch { + self.session = nil + throw error + } + } + session = nil + selectedProtocolVersion = nil + state = .detached + } + + public func diagnostics(expectedEpoch: BrokerEpoch) throws -> BrokerWorkerDiagnostics { + try validateExpectedEpoch(expectedEpoch) + return BrokerWorkerDiagnostics( + state: state, + epoch: epoch, + processID: Int32(ProcessInfo.processInfo.processIdentifier), + rootURL: configuration.storage.rootURL, + manifestDigest: preparedStorage?.manifestDigest, + activeRequestID: activeRequest?.lifecycle.requestID, + nativeDispatchStarted: activeRequest?.lifecycle.nativeDispatchStarted ?? false, + transactionStatus: transactionStatus.description, + capabilities: configuration.capabilities + ) + } + + #if DEBUG + public func injectFault( + _ fault: BrokerWorkerFault, + expectedEpoch: BrokerEpoch + ) throws { + try validateExpectedEpoch(expectedEpoch) + switch fault { + case .deadlock, .deadlockWithFailStop: + // Acknowledge the XPC control request while WorkerCore is still + // responsive. The next registered native request triggers the + // actor deadlock after the cancellation fast path is installed. + faultInjector.armDeadlockAfterNativeRequestRegistration( + failStop: fault == .deadlockWithFailStop + ) + default: + faultInjector.inject(fault) + } + } + #endif + + /// Backup/restore remain unavailable until the C ABI has bounded streaming + /// source/sink APIs. The existing whole-Data archive path is intentionally not + /// exposed through the extension broker. + public func rejectBackupOrRestore() throws { + throw BrokerError.rejected(.unsupportedCapability(.backupRestore)) + } + + private func beginRequest( + _ frame: BrokerFrame, + sink: any BrokerFrameSink + ) throws { + let requestID = try BrokerRequestID(validating: frame.header.requestID) + guard frame.payload.isEmpty else { + try sendRejection( + requestID: requestID, + reason: .invalidRequest("requestBegin payload must be empty"), + sink: sink + ) + return + } + guard state == .ready, acceptingRequests, session != nil else { + try sendRejection(requestID: requestID, reason: .queueClosed, sink: sink) + return + } + guard activeRequest == nil else { + throw BrokerProtocolError.illegalFrame( + frameType: .requestBegin, + state: activeRequest?.lifecycle.state.description ?? "active" + ) + } + var lifecycle = BrokerRequestLifecycle(epoch: epoch, requestID: requestID) + try lifecycle.beginReceiving() + activeRequest = ActiveRequest( + lifecycle: lifecycle, + assembler: BrokerFrontendRequestAssembler( + maximumRequestBytes: configuration.maximumRequestBytes + ) + ) + } + + private func appendRequestBytes( + _ frame: BrokerFrame, + sink: any BrokerFrameSink + ) throws { + let requestID = try BrokerRequestID(validating: frame.header.requestID) + guard var request = activeRequest, + request.lifecycle.requestID == requestID + else { + throw BrokerProtocolError.illegalFrame( + frameType: .requestBytes, + state: activeRequest?.lifecycle.state.description ?? "idle" + ) + } + if request.lifecycle.state == .terminal(.canceled) { + return + } + do { + try request.assembler.append(frame.payload) + activeRequest = request + } catch { + _ = request.lifecycle.establishTerminal( + .rejected(.invalidRequest(String(describing: error))) + ) + activeRequest = nil + try sendRejection( + requestID: requestID, + reason: .invalidRequest(String(describing: error)), + sink: sink + ) + } + } + + private func finishAndExecuteRequest( + _ frame: BrokerFrame, + sink: any BrokerFrameSink + ) async throws { + let requestID = try BrokerRequestID(validating: frame.header.requestID) + guard frame.payload.isEmpty else { + try sendRejection( + requestID: requestID, + reason: .invalidRequest("requestEnd payload must be empty"), + sink: sink + ) + activeRequest = nil + return + } + guard var request = activeRequest, + request.lifecycle.requestID == requestID + else { + throw BrokerProtocolError.illegalFrame( + frameType: .requestEnd, + state: activeRequest?.lifecycle.state.description ?? "idle" + ) + } + if request.lifecycle.state == .terminal(.canceled) { + activeRequest = nil + try sendRejection(requestID: requestID, reason: .canceled, sink: sink) + return + } + + let bytes: Data + do { + bytes = try request.assembler.finish() + try request.lifecycle.finishReceiving() + try request.lifecycle.beginNativeDispatch() + activeRequest = request + } catch { + _ = request.lifecycle.establishTerminal( + .rejected(.invalidRequest(String(describing: error))) + ) + activeRequest = nil + try sendRejection( + requestID: requestID, + reason: .invalidRequest(String(describing: error)), + sink: sink + ) + return + } + try await executeNativeRequest(bytes, requestID: requestID, sink: sink) + } + + private func executeNativeRequest( + _ bytes: Data, + requestID: BrokerRequestID, + sink: any BrokerFrameSink + ) async throws { + guard let session else { + throw BrokerError.databaseClosed + } + #if DEBUG + faultInjector.beforeNativeDispatch() + let nativeFaultWorkItem = faultInjector.beginNativeExecution() + #endif + + let observer = BrokerBackendResponseObserver( + epoch: epoch, + requestID: requestID, + cancellationController: cancellationController + ) + do { + let privacyFilter = try makeBackendPrivacyFilter() + try cancellationController.beginNativeRequest( + epoch: epoch, + requestID: requestID, + session: session + ) + #if DEBUG + faultInjector.afterNativeRequestRegistration() + #endif + let version = selectedProtocolVersion ?? OliphauntBrokerProtocol.maximumVersion + let requestEpoch = epoch + #if DEBUG + let injector = faultInjector + #endif + try await session.execProtocolStream(bytes) { chunk in + try privacyFilter.process(chunk) { filteredChunk in + // Observe ReadyForQuery before a backpressured write so a + // late cancel cannot poison the following request. + try observer.observe(filteredChunk) + try sendResponseBytes( + filteredChunk, + protocolVersion: version, + epoch: requestEpoch, + requestID: requestID, + sink: sink + ) + } + #if DEBUG + injector.afterResponseChunk() + #endif + } + // A native callback may end between backend-frame fragments. Never + // accept Completed/Ready while a potentially sensitive E/N message + // remains buffered or a generic backend frame is truncated. + try privacyFilter.finish() + #if DEBUG + nativeFaultWorkItem?.cancel() + #endif + + let observation = observer.snapshot() + guard observation.sawReadyForQuery, + let finalTransactionStatus = observation.transactionStatus + else { + throw BrokerError.protocolViolation( + "native response ended without ReadyForQuery" + ) + } + let cancellation = cancellationController.finishNativeRequest( + epoch: epoch, + requestID: requestID + ) + transactionStatus = finalTransactionStatus + + guard state != .interrupted, + var request = activeRequest, + request.lifecycle.requestID == requestID, + !request.lifecycle.state.isTerminal + else { + throw BrokerError.workerInterrupted(epoch: epoch) + } + + if cancellation.observedByBackend || observation.sawQueryCanceled { + try sink.send(try makeFrame(.cancelObserved, requestID: requestID)) + } + #if DEBUG + faultInjector.afterNativeSuccessBeforeCompleted() + #endif + _ = request.lifecycle.establishTerminal(.completed) + activeRequest = request + try sink.send(try makeFrame(.completed, requestID: requestID)) + activeRequest = nil + } catch { + #if DEBUG + nativeFaultWorkItem?.cancel() + #endif + cancellationController.abandonNativeRequest(epoch: epoch, requestID: requestID) + if var request = activeRequest, request.lifecycle.requestID == requestID { + _ = request.lifecycle.establishTerminal(.outcomeUnknown) + activeRequest = request + } + try? sink.send(try makeFrame(.outcomeUnknown, requestID: requestID)) + activeRequest = nil + // A thrown native stream or socket write makes this request's + // outcome unknown, but it does not prove permanent process damage. + // Tear down the physical session and allow a fresh epoch to reopen + // the same root. The failed request is never replayed. + state = .interrupted + acceptingRequests = false + await closeCurrentSessionForRecovery() + throw error + } + } + + private func makeBackendPrivacyFilter() throws -> BrokerBackendPrivacyFilter { + var prefixes = [ + configuration.storage.rootURL.path, + configuration.storage.pgdataURL.path, + configuration.storage.runtimeCacheURL.path, + configuration.storage.stagingURL.path, + NSHomeDirectory(), + NSTemporaryDirectory(), + Bundle.main.bundleURL.path, + ] + if let resourceURL = Bundle.main.resourceURL { + prefixes.append(resourceURL.path) + } + return try BrokerBackendPrivacyFilter( + sensitiveAbsolutePrefixes: prefixes + ) + } + + /// Removes the physical session before suspending in `close()` and keeps + /// `start` gated until the close finishes. Both the XPC interruption handler + /// and socket worker can report the same loss, so this also prevents a + /// duplicate close or a same-root reopen overlapping the old session. + private func closeCurrentSessionForRecovery() async { + guard !sessionCloseInProgress, let session else { return } + self.session = nil + sessionCloseInProgress = true + defer { sessionCloseInProgress = false } + try? await session.close() + } + + /// Executes rollback/checkpoint against an internal deadline without + /// waiting for an uncooperative native call after that deadline. A timeout + /// makes the epoch unusable, requests cancellation out of band, and keeps a + /// replacement epoch gated until the old call and close have unwound. + private func executeControlSQL( + _ sql: String, + session: any OliphauntSession, + completingBefore deadline: Date, + expectedEpoch: BrokerEpoch + ) async throws { + let request = try OliphauntProtocol.simpleQuery(sql) + let operation = Task.detached(priority: .userInitiated) { + () -> Result in + do { + return .success(try await session.execProtocolRaw(request)) + } catch { + return .failure(error) + } + } + let gate = BrokerControlDeadlineGate() + Task.detached(priority: .userInitiated) { + gate.resolve(.completed(await operation.value)) + } + Task.detached(priority: .userInitiated) { + let remaining = max(0, deadline.timeIntervalSinceNow) + if remaining > 0 { + try? await Task.sleep(for: .seconds(remaining)) + } + gate.resolve(.expired) + } + + switch await gate.wait() { + case .completed(let result) where Date() <= deadline: + let response = try result.get() + _ = try parseOliphauntQueryResponse(response) + case .completed, .expired: + beginDeadlineRecovery( + session: session, + operation: operation, + expectedEpoch: expectedEpoch + ) + throw BrokerError.workerInterrupted(epoch: expectedEpoch) + } + } + + private func beginDeadlineRecovery( + session: any OliphauntSession, + operation: Task, Never>?, + expectedEpoch: BrokerEpoch + ) { + guard epoch == expectedEpoch else { return } + acceptingRequests = false + state = .interrupted + self.session = nil + sessionCloseInProgress = true + let recoveryID = UUID() + deadlineRecoveryID = recoveryID + + Task.detached(priority: .userInitiated) { + try? await session.cancel() + } + Task.detached(priority: .utility) { [weak self] in + if let operation { + _ = await operation.value + } + try? await session.close() + await self?.finishDeadlineRecovery(recoveryID) + } + } + + private func finishDeadlineRecovery(_ recoveryID: UUID) { + guard deadlineRecoveryID == recoveryID else { return } + deadlineRecoveryID = nil + sessionCloseInProgress = false + } + + private func validateStartAttempt(_ attemptID: UUID) throws { + guard startAttemptID == attemptID, state == .starting else { + throw BrokerError.workerInterrupted(epoch: epoch) + } + } + + private func validateExpectedEpoch(_ expectedEpoch: BrokerEpoch) throws { + guard epoch == expectedEpoch else { + throw BrokerError.workerInterrupted(epoch: expectedEpoch) + } + } + + private func validateControlContinuation( + expectedEpoch: BrokerEpoch, + allowedStates: [BrokerWorkerCoreState] + ) throws { + try validateExpectedEpoch(expectedEpoch) + guard allowedStates.contains(state) else { + throw BrokerError.workerInterrupted(epoch: expectedEpoch) + } + } + + private func executeControlSQL( + _ sql: String, + session: any OliphauntSession + ) async throws { + let response = try await session.execProtocolRaw(try OliphauntProtocol.simpleQuery(sql)) + _ = try parseOliphauntQueryResponse(response) + } + + private func establishRestrictedRoleBoundary( + _ session: any OliphauntSession, + selectedExtensions: [String] + ) async throws { + let role = BrokerWorkerConfiguration.restrictedRoleUsername + let quotedRoleLiteral = postgresStringLiteral(role) + let quotedRoleIdentifier = postgresIdentifier(role) + let bootstrapRole = "postgres" + let quotedBootstrapRoleLiteral = postgresStringLiteral(bootstrapRole) + let quotedBootstrapRoleIdentifier = postgresIdentifier(bootstrapRole) + let database = BrokerWorkerConfiguration.restrictedDatabase + let quotedDatabaseIdentifier = postgresIdentifier(database) + let extensionArray = + selectedExtensions + .sorted() + .map(postgresStringLiteral) + .joined(separator: ", ") + + let sql = """ + BEGIN; + DO $oliphaunt_broker_bootstrap$ + DECLARE + extension_name text; + membership_name text; + membership_names text[]; + role_is_superuser boolean; + BEGIN + IF session_user <> \(quotedRoleLiteral) + OR current_user <> \(quotedRoleLiteral) THEN + RAISE EXCEPTION 'broker restricted role identity mismatch'; + END IF; + + SELECT rolsuper + INTO STRICT role_is_superuser + FROM pg_catalog.pg_roles + WHERE rolname = \(quotedRoleLiteral); + + IF role_is_superuser THEN + FOREACH extension_name IN ARRAY ARRAY[\(extensionArray)]::text[] LOOP + EXECUTE format( + 'CREATE EXTENSION IF NOT EXISTS %I', + extension_name + ); + END LOOP; + + REASSIGN OWNED BY \(quotedRoleIdentifier) + TO \(quotedBootstrapRoleIdentifier); + CREATE SCHEMA IF NOT EXISTS \(quotedRoleIdentifier) + AUTHORIZATION \(quotedRoleIdentifier); + ALTER SCHEMA \(quotedRoleIdentifier) + OWNER TO \(quotedRoleIdentifier); + GRANT CONNECT, TEMPORARY + ON DATABASE \(quotedDatabaseIdentifier) + TO \(quotedRoleIdentifier); + REVOKE CREATE + ON DATABASE \(quotedDatabaseIdentifier) + FROM \(quotedRoleIdentifier); + GRANT USAGE, CREATE + ON SCHEMA \(quotedRoleIdentifier) + TO \(quotedRoleIdentifier); + REVOKE CREATE + ON SCHEMA public + FROM PUBLIC, \(quotedRoleIdentifier); + GRANT USAGE ON SCHEMA public TO \(quotedRoleIdentifier); + GRANT pg_checkpoint TO \(quotedRoleIdentifier); + REVOKE EXECUTE + ON FUNCTION pg_catalog.pg_relation_filepath(regclass) + FROM PUBLIC, \(quotedRoleIdentifier); + REVOKE EXECUTE + ON FUNCTION pg_catalog.pg_tablespace_location(oid) + FROM PUBLIC, \(quotedRoleIdentifier); + + SELECT coalesce( + array_agg(parent.rolname::text ORDER BY parent.rolname), + ARRAY[]::text[] + ) + INTO membership_names + FROM pg_catalog.pg_auth_members membership + JOIN pg_catalog.pg_roles parent ON parent.oid = membership.roleid + JOIN pg_catalog.pg_roles member ON member.oid = membership.member + WHERE member.rolname = \(quotedRoleLiteral) + AND parent.rolname <> 'pg_checkpoint'; + + FOREACH membership_name IN ARRAY membership_names LOOP + EXECUTE format( + 'REVOKE %I FROM %I', + membership_name, + \(quotedRoleLiteral) + ); + END LOOP; + + ALTER ROLE \(quotedRoleIdentifier) + NOSUPERUSER NOCREATEDB NOCREATEROLE + INHERIT LOGIN NOREPLICATION NOBYPASSRLS; + END IF; + + ALTER ROLE \(quotedRoleIdentifier) + SET search_path TO "$user", public; + END + $oliphaunt_broker_bootstrap$; + SET SESSION AUTHORIZATION \(quotedRoleIdentifier); + SET search_path TO "$user", public; + + DO $oliphaunt_broker_validate$ + DECLARE + direct_memberships text[]; + effective_memberships text[]; + role_is_safe boolean; + BEGIN + SELECT + NOT rolsuper + AND NOT rolcreatedb + AND NOT rolcreaterole + AND rolinherit + AND rolcanlogin + AND NOT rolreplication + AND NOT rolbypassrls + AND coalesce( + rolconfig @> ARRAY['search_path="$user", public'], + false + ) + INTO STRICT role_is_safe + FROM pg_catalog.pg_roles + WHERE rolname = \(quotedRoleLiteral); + + IF session_user <> \(quotedRoleLiteral) + OR current_user <> \(quotedRoleLiteral) + OR current_setting('is_superuser') <> 'off' + OR current_schemas(false) + <> ARRAY[\(quotedRoleLiteral), 'public']::name[] + OR NOT role_is_safe THEN + RAISE EXCEPTION 'broker restricted role validation failed'; + END IF; + + SELECT coalesce( + array_agg(parent.rolname::text ORDER BY parent.rolname), + ARRAY[]::text[] + ) + INTO direct_memberships + FROM pg_catalog.pg_auth_members membership + JOIN pg_catalog.pg_roles parent ON parent.oid = membership.roleid + JOIN pg_catalog.pg_roles member ON member.oid = membership.member + WHERE member.rolname = \(quotedRoleLiteral); + + IF direct_memberships <> ARRAY['pg_checkpoint']::text[] THEN + RAISE EXCEPTION 'broker restricted role membership validation failed'; + END IF; + + WITH RECURSIVE effective_role_oids(oid) AS ( + SELECT oid + FROM pg_catalog.pg_roles + WHERE rolname = \(quotedRoleLiteral) + UNION + SELECT membership.roleid + FROM effective_role_oids effective + JOIN pg_catalog.pg_auth_members membership + ON membership.member = effective.oid + ) + SELECT coalesce( + array_agg( + role.rolname::text + ORDER BY role.rolname::text COLLATE "C" + ), + ARRAY[]::text[] + ) + INTO effective_memberships + FROM effective_role_oids effective + JOIN pg_catalog.pg_roles role ON role.oid = effective.oid; + + IF effective_memberships + <> ARRAY[\(quotedRoleLiteral), 'pg_checkpoint']::text[] THEN + RAISE EXCEPTION 'broker effective role membership validation failed'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_database database + JOIN pg_catalog.pg_roles owner ON owner.oid = database.datdba + WHERE database.datname = current_database() + AND owner.rolname = \(quotedBootstrapRoleLiteral) + ) THEN + RAISE EXCEPTION 'broker database owner validation failed'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace namespace + JOIN pg_catalog.pg_roles owner ON owner.oid = namespace.nspowner + WHERE namespace.nspname = \(quotedRoleLiteral) + AND owner.rolname = \(quotedRoleLiteral) + ) THEN + RAISE EXCEPTION 'broker schema owner validation failed'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM unnest(ARRAY[\(extensionArray)]::text[]) requested(name) + LEFT JOIN pg_catalog.pg_extension installed + ON installed.extname = requested.name + LEFT JOIN pg_catalog.pg_roles owner + ON owner.oid = installed.extowner + WHERE installed.oid IS NULL + OR owner.rolname <> \(quotedBootstrapRoleLiteral) + ) THEN + RAISE EXCEPTION 'broker selected extension owner validation failed'; + END IF; + + IF NOT pg_catalog.has_database_privilege( + \(quotedRoleLiteral), current_database(), 'CONNECT' + ) + OR NOT pg_catalog.has_database_privilege( + \(quotedRoleLiteral), current_database(), 'TEMPORARY' + ) + OR pg_catalog.has_database_privilege( + \(quotedRoleLiteral), current_database(), 'CREATE' + ) + OR NOT pg_catalog.has_schema_privilege( + \(quotedRoleLiteral), \(quotedRoleLiteral), 'USAGE' + ) + OR NOT pg_catalog.has_schema_privilege( + \(quotedRoleLiteral), \(quotedRoleLiteral), 'CREATE' + ) + OR NOT pg_catalog.has_schema_privilege( + \(quotedRoleLiteral), 'public', 'USAGE' + ) + OR pg_catalog.has_schema_privilege( + \(quotedRoleLiteral), 'public', 'CREATE' + ) THEN + RAISE EXCEPTION 'broker runtime privilege validation failed'; + END IF; + + IF pg_catalog.pg_has_role( + \(quotedRoleLiteral), 'pg_database_owner', 'USAGE' + ) + OR pg_catalog.pg_has_role( + \(quotedRoleLiteral), 'pg_database_owner', 'MEMBER' + ) + OR pg_catalog.pg_has_role( + \(quotedRoleLiteral), 'pg_database_owner', 'SET' + ) THEN + RAISE EXCEPTION 'broker database-owner role validation failed'; + END IF; + + IF pg_catalog.has_function_privilege( + \(quotedRoleLiteral), + 'pg_catalog.pg_relation_filepath(regclass)', + 'EXECUTE' + ) + OR pg_catalog.has_function_privilege( + \(quotedRoleLiteral), + 'pg_catalog.pg_tablespace_location(oid)', + 'EXECUTE' + ) THEN + RAISE EXCEPTION 'broker path function privilege validation failed'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_tablespace + WHERE spcname NOT IN ('pg_default', 'pg_global') + ) THEN + RAISE EXCEPTION 'broker tablespace validation failed'; + END IF; + END + $oliphaunt_broker_validate$; + COMMIT; + """ + try await executeControlSQL(sql, session: session) + } + + private func healthCheck(_ session: any OliphauntSession) async throws { + let response = try await session.execProtocolRaw( + try OliphauntProtocol.simpleQuery("SELECT 1 AS broker_health") + ) + let result = try parseOliphauntQueryResponse(response) + guard result.rows.count == 1, + try result.rows[0].text(0) == "1" + else { + throw BrokerError.protocolViolation("broker health check returned an unexpected row") + } + } + + private func validateNativeCapabilities(_ session: any OliphauntSession) async throws { + let capabilities = await session.capabilities() + guard capabilities.protocolRaw, + capabilities.protocolStream, + capabilities.queryCancel + else { + throw BrokerError.invalidConfiguration( + "linked liboliphaunt lacks raw protocol, streaming response, or cancellation support" + ) + } + } + + private func validateRequestedCapabilities( + _ requested: Set + ) throws { + let unsupported = requested.subtracting(configuration.capabilities.enabled) + .sorted { $0.rawValue < $1.rawValue } + if let first = unsupported.first { + throw BrokerError.rejected(.unsupportedCapability(first)) + } + } + + private func readyValue() -> BrokerReady { + precondition(state == .ready) + precondition(session != nil) + guard let preparedStorage, let selectedProtocolVersion else { + preconditionFailure("ready worker has no prepared storage or protocol version") + } + return BrokerReady( + selectedProtocolVersion: selectedProtocolVersion, + epoch: epoch, + extensionPID: Int32(ProcessInfo.processInfo.processIdentifier), + runtimeVersion: configuration.liboliphauntVersion, + abiVersion: configuration.cABIVersion, + postgresMajorVersion: configuration.postgresMajorVersion, + rootManifestDigest: preparedStorage.manifestDigest, + actualCapabilities: configuration.capabilities, + actualRuntimeConfiguration: BrokerRuntimeConfiguration( + rootID: OliphauntBrokerProtocol.canonicalRootID, + startupConfigurationDigest: configuration.startupConfigurationDigest, + selectedExtensions: preparedStorage.manifest.selectedPostgresExtensions, + footprintProfile: "smallMobile" + ) + ) + } + + private func sendRejection( + requestID: BrokerRequestID, + reason: BrokerRejectionReason, + sink: any BrokerFrameSink + ) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + try sink.send( + try makeFrame( + .rejected, + requestID: requestID, + payload: try encoder.encode(reason) + )) + } + + private func protocolViolation( + _ reason: String, + sink: any BrokerFrameSink + ) throws -> BrokerFrameHandlingResult { + try sink.send(try makeFrame(.protocolError, payload: Data(reason.utf8))) + return .closeChannel + } + + private func makeFrame( + _ type: BrokerFrameType, + requestID: BrokerRequestID? = nil, + payload: Data = Data() + ) throws -> BrokerFrame { + try BrokerFrame( + protocolVersion: selectedProtocolVersion ?? OliphauntBrokerProtocol.maximumVersion, + frameType: type, + epoch: epoch, + requestID: requestID?.rawValue ?? 0, + payload: payload + ) + } +} + +private enum BrokerControlDeadlineOutcome: Sendable { + case completed(Result) + case expired +} + +private final class BrokerControlDeadlineGate: @unchecked Sendable { + private let lock = NSLock() + private var outcome: BrokerControlDeadlineOutcome? + private var continuation: CheckedContinuation? + + func wait() async -> BrokerControlDeadlineOutcome { + await withCheckedContinuation { continuation in + lock.lock() + if let outcome { + lock.unlock() + continuation.resume(returning: outcome) + } else { + self.continuation = continuation + lock.unlock() + } + } + } + + func resolve(_ outcome: BrokerControlDeadlineOutcome) { + lock.lock() + guard self.outcome == nil else { + lock.unlock() + return + } + self.outcome = outcome + let continuation = self.continuation + self.continuation = nil + lock.unlock() + continuation?.resume(returning: outcome) + } +} + +private func sendResponseBytes( + _ bytes: Data, + protocolVersion: UInt16, + epoch: BrokerEpoch, + requestID: BrokerRequestID, + sink: any BrokerFrameSink +) throws { + var offset = 0 + while offset < bytes.count { + let end = min(offset + OliphauntBrokerProtocol.maximumFramePayload, bytes.count) + try sink.send( + try BrokerFrame( + protocolVersion: protocolVersion, + frameType: .responseBytes, + epoch: epoch, + requestID: requestID.rawValue, + payload: bytes.subdata(in: offset.. String { + "\"\(value.replacingOccurrences(of: "\"", with: "\"\""))\"" +} + +private func postgresStringLiteral(_ value: String) -> String { + "'\(value.replacingOccurrences(of: "'", with: "''"))'" +} + +extension BrokerBackendTransactionStatus { + fileprivate var description: String { + switch self { + case .idle: "idle" + case .transaction: "transaction" + case .failedTransaction: "failedTransaction" + } + } +} + +extension BrokerError { + fileprivate static func brokerUnavailableWithReason(_ reason: String) -> BrokerError { + .invalidConfiguration("worker is unavailable: \(reason)") + } +} + +private func linkedOliphauntVersion() throws -> String { + dlerror() + guard + let symbol = dlsym( + UnsafeMutableRawPointer(bitPattern: -2), // Darwin's RTLD_DEFAULT + "oliphaunt_version" + ) + else { + let detail = dlerror().map { String(cString: $0) } ?? "symbol not found" + throw BrokerError.invalidConfiguration( + "cannot inspect linked liboliphaunt version: \(detail)" + ) + } + typealias VersionFunction = @convention(c) () -> UnsafePointer? + let function = unsafeBitCast(symbol, to: VersionFunction.self) + guard let value = function() else { + throw BrokerError.invalidConfiguration("linked liboliphaunt returned a null version") + } + let version = String(cString: value) + guard !version.isEmpty else { + throw BrokerError.invalidConfiguration("linked liboliphaunt returned an empty version") + } + return version +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerFrame.swift b/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerFrame.swift new file mode 100644 index 00000000..95fb0e54 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerFrame.swift @@ -0,0 +1,387 @@ +import Foundation + +public enum BrokerFrameType: UInt8, Codable, CaseIterable, Sendable { + case requestBegin = 1 + case requestBytes = 2 + case requestEnd = 3 + case responseBytes = 4 + case completed = 5 + case rejected = 6 + case outcomeUnknown = 7 + case cancelRequested = 8 + case cancelObserved = 9 + case ping = 10 + case pong = 11 + case protocolError = 12 + case channelClose = 13 + + public var requiresRequestID: Bool { + switch self { + case .ping, .pong, .protocolError, .channelClose: + false + default: + true + } + } +} + +public struct BrokerFrameFlags: OptionSet, Hashable, Codable, Sendable { + public let rawValue: UInt8 + + public init(rawValue: UInt8) { + self.rawValue = rawValue + } + + public static let none: BrokerFrameFlags = [] + public static let knownMask: UInt8 = 0 +} + +public struct BrokerFrameHeader: Equatable, Sendable { + public var protocolVersion: UInt16 + public var frameType: BrokerFrameType + public var flags: BrokerFrameFlags + public var epoch: BrokerEpoch + public var requestID: UInt64 + public var payloadLength: UInt32 + + public init( + protocolVersion: UInt16 = OliphauntBrokerProtocol.maximumVersion, + frameType: BrokerFrameType, + flags: BrokerFrameFlags = .none, + epoch: BrokerEpoch, + requestID: UInt64, + payloadLength: UInt32 + ) { + self.protocolVersion = protocolVersion + self.frameType = frameType + self.flags = flags + self.epoch = epoch + self.requestID = requestID + self.payloadLength = payloadLength + } + + public func encoded() throws -> Data { + try validate(maximumPayloadLength: OliphauntBrokerProtocol.maximumFramePayload) + var result = Data(capacity: Int(OliphauntBrokerProtocol.headerLength)) + result.append(contentsOf: [ + OliphauntBrokerProtocol.magic.0, + OliphauntBrokerProtocol.magic.1, + OliphauntBrokerProtocol.magic.2, + OliphauntBrokerProtocol.magic.3, + ]) + result.appendNetwork(protocolVersion) + result.appendNetwork(OliphauntBrokerProtocol.headerLength) + result.append(frameType.rawValue) + result.append(flags.rawValue) + result.appendNetwork(UInt16(0)) + result.appendUUID(epoch.rawValue) + result.appendNetwork(requestID) + result.appendNetwork(payloadLength) + precondition(result.count == Int(OliphauntBrokerProtocol.headerLength)) + return result + } + + public static func decode( + _ bytes: Data, + expectedEpoch: BrokerEpoch? = nil, + maximumPayloadLength: Int = OliphauntBrokerProtocol.maximumFramePayload + ) throws -> BrokerFrameHeader { + guard bytes.count >= Int(OliphauntBrokerProtocol.headerLength) else { + throw BrokerProtocolError.truncatedFrame + } + guard bytes[0] == OliphauntBrokerProtocol.magic.0, + bytes[1] == OliphauntBrokerProtocol.magic.1, + bytes[2] == OliphauntBrokerProtocol.magic.2, + bytes[3] == OliphauntBrokerProtocol.magic.3 + else { + throw BrokerProtocolError.invalidMagic + } + let protocolVersion = try bytes.networkUInt16(at: 4) + guard OliphauntBrokerProtocol.supports(version: protocolVersion) else { + throw BrokerProtocolError.unsupportedVersion(protocolVersion) + } + let headerLength = try bytes.networkUInt16(at: 6) + guard headerLength == OliphauntBrokerProtocol.headerLength else { + throw BrokerProtocolError.invalidHeaderLength(headerLength) + } + guard let frameType = BrokerFrameType(rawValue: bytes[8]) else { + throw BrokerProtocolError.unknownFrameType(bytes[8]) + } + let flags = BrokerFrameFlags(rawValue: bytes[9]) + guard flags.rawValue & ~BrokerFrameFlags.knownMask == 0 else { + throw BrokerProtocolError.invalidFlags(flags.rawValue) + } + let reserved = try bytes.networkUInt16(at: 10) + guard reserved == 0 else { + throw BrokerProtocolError.nonzeroReserved(reserved) + } + let epoch = BrokerEpoch(try bytes.uuid(at: 12)) + if let expectedEpoch, epoch != expectedEpoch { + throw BrokerProtocolError.staleEpoch(expected: expectedEpoch, actual: epoch) + } + let requestID = try bytes.networkUInt64(at: 28) + let payloadLength = try bytes.networkUInt32(at: 36) + let header = BrokerFrameHeader( + protocolVersion: protocolVersion, + frameType: frameType, + flags: flags, + epoch: epoch, + requestID: requestID, + payloadLength: payloadLength + ) + try header.validate(maximumPayloadLength: maximumPayloadLength) + return header + } + + public func validate(maximumPayloadLength: Int) throws { + guard OliphauntBrokerProtocol.supports(version: protocolVersion) else { + throw BrokerProtocolError.unsupportedVersion(protocolVersion) + } + guard flags.rawValue & ~BrokerFrameFlags.knownMask == 0 else { + throw BrokerProtocolError.invalidFlags(flags.rawValue) + } + if frameType.requiresRequestID { + guard requestID != 0 else { + throw BrokerProtocolError.invalidRequestIDForFrame( + frameType: frameType, + requestID: requestID + ) + } + } else if requestID != 0 { + throw BrokerProtocolError.invalidRequestIDForFrame( + frameType: frameType, + requestID: requestID + ) + } + guard maximumPayloadLength >= 0 else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(payloadLength), + maximum: maximumPayloadLength + ) + } + guard UInt64(payloadLength) <= UInt64(maximumPayloadLength) else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(payloadLength), + maximum: maximumPayloadLength + ) + } + } +} + +public struct BrokerFrame: Equatable, Sendable { + public var header: BrokerFrameHeader + public var payload: Data + + public init( + protocolVersion: UInt16 = OliphauntBrokerProtocol.maximumVersion, + frameType: BrokerFrameType, + flags: BrokerFrameFlags = .none, + epoch: BrokerEpoch, + requestID: UInt64, + payload: Data = Data() + ) throws { + guard payload.count <= Int(UInt32.max) else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(payload.count), + maximum: Int(UInt32.max) + ) + } + header = BrokerFrameHeader( + protocolVersion: protocolVersion, + frameType: frameType, + flags: flags, + epoch: epoch, + requestID: requestID, + payloadLength: UInt32(payload.count) + ) + self.payload = payload + try header.validate(maximumPayloadLength: OliphauntBrokerProtocol.maximumFramePayload) + } + + init(header: BrokerFrameHeader, payload: Data) { + self.header = header + self.payload = payload + } + + public func encoded() throws -> Data { + guard payload.count == Int(header.payloadLength) else { + throw BrokerProtocolError.protocolLengthMismatchForFrame( + declared: header.payloadLength, + actual: payload.count + ) + } + var result = try header.encoded() + result.append(payload) + return result + } +} + +public struct BrokerFrameDecoder: Sendable { + private var buffer = Data() + private var readOffset = 0 + public var expectedEpoch: BrokerEpoch? + public var maximumPayloadLength: Int + public var maximumBufferedBytes: Int + + public init( + expectedEpoch: BrokerEpoch? = nil, + maximumPayloadLength: Int = OliphauntBrokerProtocol.maximumFramePayload, + maximumBufferedBytes: Int = OliphauntBrokerProtocol.maximumQueuedBytesPerDirection + ) { + self.expectedEpoch = expectedEpoch + self.maximumPayloadLength = maximumPayloadLength + self.maximumBufferedBytes = maximumBufferedBytes + } + + public mutating func append(_ bytes: Data) throws -> [BrokerFrame] { + if !bytes.isEmpty { + let unread = buffer.count - readOffset + let (combined, overflow) = unread.addingReportingOverflow(bytes.count) + guard !overflow else { + throw BrokerProtocolError.arithmeticOverflow + } + // A stream read may finish one maximum-sized frame and include the + // beginning of subsequent frames. Bound aggregate buffered bytes, + // but let the header parser enforce the per-frame payload limit. + guard combined <= maximumBufferedBytes else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(combined), + maximum: maximumBufferedBytes + ) + } + compactIfNeeded(force: readOffset > 0) + buffer.append(bytes) + } + + var frames: [BrokerFrame] = [] + while buffer.count - readOffset >= Int(OliphauntBrokerProtocol.headerLength) { + let headerEnd = readOffset + Int(OliphauntBrokerProtocol.headerLength) + let headerData = buffer.subdata(in: readOffset..= frameLength else { + break + } + let payloadStart = headerEnd + let payloadEnd = readOffset + frameLength + frames.append( + BrokerFrame( + header: header, + payload: buffer.subdata(in: payloadStart.. [BrokerFrame] { + let frames = try append(Data()) + guard buffer.count == readOffset else { + throw BrokerProtocolError.truncatedFrame + } + return frames + } + + public mutating func reset(expectedEpoch: BrokerEpoch?) { + buffer.removeAll(keepingCapacity: true) + readOffset = 0 + self.expectedEpoch = expectedEpoch + } + + private mutating func compactIfNeeded(force: Bool) { + guard readOffset > 0, force || readOffset >= 64 * 1024 else { + return + } + if readOffset == buffer.count { + buffer.removeAll(keepingCapacity: true) + } else { + buffer.removeSubrange(0.. BrokerProtocolError { + .illegalFrame( + frameType: .protocolError, + state: "payload length declared \(declared), actual \(actual)" + ) + } +} + +extension Data { + fileprivate mutating func appendNetwork(_ value: UInt16) { + append(UInt8((value >> 8) & 0xff)) + append(UInt8(value & 0xff)) + } + + fileprivate mutating func appendNetwork(_ value: UInt32) { + append(UInt8((value >> 24) & 0xff)) + append(UInt8((value >> 16) & 0xff)) + append(UInt8((value >> 8) & 0xff)) + append(UInt8(value & 0xff)) + } + + fileprivate mutating func appendNetwork(_ value: UInt64) { + for shift in stride(from: 56, through: 0, by: -8) { + append(UInt8((value >> UInt64(shift)) & 0xff)) + } + } + + fileprivate mutating func appendUUID(_ value: UUID) { + var uuid = value.uuid + Swift.withUnsafeBytes(of: &uuid) { append(contentsOf: $0) } + } + + fileprivate func networkUInt16(at offset: Int) throws -> UInt16 { + guard offset >= 0, count - offset >= 2 else { + throw BrokerProtocolError.truncatedFrame + } + return (UInt16(self[offset]) << 8) | UInt16(self[offset + 1]) + } + + fileprivate func networkUInt32(at offset: Int) throws -> UInt32 { + guard offset >= 0, count - offset >= 4 else { + throw BrokerProtocolError.truncatedFrame + } + return (UInt32(self[offset]) << 24) | (UInt32(self[offset + 1]) << 16) + | (UInt32(self[offset + 2]) << 8) | UInt32(self[offset + 3]) + } + + fileprivate func networkUInt64(at offset: Int) throws -> UInt64 { + guard offset >= 0, count - offset >= 8 else { + throw BrokerProtocolError.truncatedFrame + } + var result: UInt64 = 0 + for index in offset..<(offset + 8) { + result = (result << 8) | UInt64(self[index]) + } + return result + } + + fileprivate func uuid(at offset: Int) throws -> UUID { + guard offset >= 0, count - offset >= 16 else { + throw BrokerProtocolError.truncatedFrame + } + let values = Array(self[offset..<(offset + 16)]) + return UUID( + uuid: ( + values[0], values[1], values[2], values[3], + values[4], values[5], values[6], values[7], + values[8], values[9], values[10], values[11], + values[12], values[13], values[14], values[15] + )) + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerProtocol.swift b/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerProtocol.swift new file mode 100644 index 00000000..895e9d4a --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerProtocol.swift @@ -0,0 +1,445 @@ +import Foundation + +public enum OliphauntBrokerProtocol { + public static let magic: (UInt8, UInt8, UInt8, UInt8) = (0x4f, 0x4c, 0x50, 0x42) + public static let headerLength: UInt16 = 40 + public static let minimumVersion: UInt16 = 1 + public static let maximumVersion: UInt16 = 1 + public static let maximumFramePayload = 256 * 1024 + public static let maximumQueuedBytesPerDirection = 8 * 1024 * 1024 + public static let defaultMaximumRequestBytes = 8 * 1024 * 1024 + public static let canonicalRootID = "default" + + public static func supports(version: UInt16) -> Bool { + version >= minimumVersion && version <= maximumVersion + } +} + +public struct BrokerEpoch: Hashable, Codable, Sendable, CustomStringConvertible { + public let rawValue: UUID + + public init(_ rawValue: UUID) { + self.rawValue = rawValue + } + + public static func fresh() -> BrokerEpoch { + BrokerEpoch(UUID()) + } + + public var description: String { + rawValue.uuidString.lowercased() + } +} + +public struct BrokerRequestID: Hashable, Codable, Sendable, Comparable, CustomStringConvertible { + public let rawValue: UInt64 + + public init(validating rawValue: UInt64) throws { + guard rawValue != 0 else { + throw BrokerProtocolError.invalidRequestID(rawValue) + } + self.rawValue = rawValue + } + + init(unchecked rawValue: UInt64) { + self.rawValue = rawValue + } + + public static func < (lhs: BrokerRequestID, rhs: BrokerRequestID) -> Bool { + lhs.rawValue < rhs.rawValue + } + + public var description: String { + String(rawValue) + } +} + +public struct BrokerRequestIDSequence: Sendable { + private var nextValue: UInt64 + + public init(startingAt: UInt64 = 1) throws { + guard startingAt != 0 else { + throw BrokerProtocolError.invalidRequestID(startingAt) + } + nextValue = startingAt + } + + public mutating func next() throws -> BrokerRequestID { + guard nextValue != 0 else { + throw BrokerProtocolError.requestIDSpaceExhausted + } + let result = BrokerRequestID(unchecked: nextValue) + if nextValue == UInt64.max { + nextValue = 0 + } else { + nextValue += 1 + } + return result + } +} + +public enum BrokerControlMessageKind: String, Codable, CaseIterable, Sendable { + case hello + case ready + case rejected + case attachDataChannel + case cancel + case cancelObserved + case checkpoint + case prepareForBackground + case resumeFromBackground + case detach + case diagnostics + case injectFault +} + +public enum BrokerControlKey { + public static let message = "message" + public static let minimumProtocolVersion = "minimumProtocolVersion" + public static let maximumProtocolVersion = "maximumProtocolVersion" + public static let selectedProtocolVersion = "selectedProtocolVersion" + public static let expectedABI = "expectedABI" + public static let expectedRuntimeVersion = "expectedRuntimeVersion" + public static let rootID = "rootID" + public static let startupConfigurationDigest = "startupConfigurationDigest" + public static let requestedCapabilities = "requestedCapabilities" + public static let dataChannel = "dataChannel" + public static let epoch = "epoch" + public static let requestID = "requestID" + public static let extensionPID = "extensionPID" + public static let runtimeVersion = "runtimeVersion" + public static let abiVersion = "abiVersion" + public static let postgresMajorVersion = "postgresMajorVersion" + public static let rootManifestDigest = "rootManifestDigest" + public static let actualCapabilities = "actualCapabilities" + public static let actualRuntimeConfiguration = "actualRuntimeConfiguration" + public static let deadlineUnixNanoseconds = "deadlineUnixNanoseconds" + public static let fault = "fault" + public static let reason = "reason" + public static let error = "error" +} + +public struct BrokerHello: Equatable, Codable, Sendable { + public var minimumProtocolVersion: UInt16 + public var maximumProtocolVersion: UInt16 + public var expectedABI: UInt32 + public var expectedRuntimeVersion: String? + public var rootID: String + public var startupConfigurationDigest: String + public var requestedCapabilities: Set + + public init( + minimumProtocolVersion: UInt16 = OliphauntBrokerProtocol.minimumVersion, + maximumProtocolVersion: UInt16 = OliphauntBrokerProtocol.maximumVersion, + expectedABI: UInt32, + expectedRuntimeVersion: String? = nil, + rootID: String = OliphauntBrokerProtocol.canonicalRootID, + startupConfigurationDigest: String, + requestedCapabilities: Set + ) { + self.minimumProtocolVersion = minimumProtocolVersion + self.maximumProtocolVersion = maximumProtocolVersion + self.expectedABI = expectedABI + self.expectedRuntimeVersion = expectedRuntimeVersion + self.rootID = rootID + self.startupConfigurationDigest = startupConfigurationDigest + self.requestedCapabilities = requestedCapabilities + } +} + +public struct BrokerReady: Equatable, Codable, Sendable { + public var selectedProtocolVersion: UInt16 + public var epoch: BrokerEpoch + public var extensionPID: Int32 + public var runtimeVersion: String + public var abiVersion: UInt32 + public var postgresMajorVersion: UInt16 + public var rootManifestDigest: String + public var actualCapabilities: BrokerCapabilities + public var actualRuntimeConfiguration: BrokerRuntimeConfiguration + + public init( + selectedProtocolVersion: UInt16, + epoch: BrokerEpoch, + extensionPID: Int32, + runtimeVersion: String, + abiVersion: UInt32, + postgresMajorVersion: UInt16, + rootManifestDigest: String, + actualCapabilities: BrokerCapabilities, + actualRuntimeConfiguration: BrokerRuntimeConfiguration + ) { + self.selectedProtocolVersion = selectedProtocolVersion + self.epoch = epoch + self.extensionPID = extensionPID + self.runtimeVersion = runtimeVersion + self.abiVersion = abiVersion + self.postgresMajorVersion = postgresMajorVersion + self.rootManifestDigest = rootManifestDigest + self.actualCapabilities = actualCapabilities + self.actualRuntimeConfiguration = actualRuntimeConfiguration + } +} + +public struct BrokerRuntimeConfiguration: Equatable, Codable, Sendable { + public var rootID: String + public var startupConfigurationDigest: String + public var selectedExtensions: [String] + public var footprintProfile: String + + public init( + rootID: String, + startupConfigurationDigest: String, + selectedExtensions: [String], + footprintProfile: String = "smallMobile" + ) { + self.rootID = rootID + self.startupConfigurationDigest = startupConfigurationDigest + self.selectedExtensions = selectedExtensions + self.footprintProfile = footprintProfile + } +} + +public enum BrokerCapability: String, Codable, CaseIterable, Sendable { + case processIsolated + case crashRestartable + case hangRestartable + case sameRootLogicalReopen + case rootSwitchable + case multiRoot + case independentSessions + case backgroundContinuable + case protocolRaw + case protocolStream + case streamingRequestInput + case queryCancel + case backupRestore +} + +public struct BrokerCapabilities: Equatable, Codable, Sendable { + public var mode: String + public var implementation: String + public var minimumOS: String + public var processIsolated: Bool + public var crashRestartable: Bool + public var hangRestartable: Bool + public var sameRootLogicalReopen: Bool + public var rootSwitchable: Bool + public var multiRoot: Bool + public var independentSessions: Bool + public var maxClientSessions: Int + public var backgroundContinuable: Bool + public var requiresAppGroup: Bool + public var protocolRaw: Bool + public var protocolStream: Bool + public var streamingRequestInput: Bool + public var queryCancel: Bool + public var backupRestore: Bool + public var connectionString: String? + public var serverMode: Bool + + public init( + mode: String = "nativeBroker", + implementation: String = "iosExtensionBroker", + minimumOS: String = "iOS 26", + processIsolated: Bool = true, + crashRestartable: Bool = true, + hangRestartable: Bool = false, + sameRootLogicalReopen: Bool = true, + rootSwitchable: Bool = false, + multiRoot: Bool = false, + independentSessions: Bool = false, + maxClientSessions: Int = 1, + backgroundContinuable: Bool = false, + requiresAppGroup: Bool = false, + protocolRaw: Bool = true, + protocolStream: Bool = true, + streamingRequestInput: Bool = false, + queryCancel: Bool = true, + backupRestore: Bool = false, + connectionString: String? = nil, + serverMode: Bool = false + ) { + self.mode = mode + self.implementation = implementation + self.minimumOS = minimumOS + self.processIsolated = processIsolated + self.crashRestartable = crashRestartable + self.hangRestartable = hangRestartable + self.sameRootLogicalReopen = sameRootLogicalReopen + self.rootSwitchable = rootSwitchable + self.multiRoot = multiRoot + self.independentSessions = independentSessions + self.maxClientSessions = maxClientSessions + self.backgroundContinuable = backgroundContinuable + self.requiresAppGroup = requiresAppGroup + self.protocolRaw = protocolRaw + self.protocolStream = protocolStream + self.streamingRequestInput = streamingRequestInput + self.queryCancel = queryCancel + self.backupRestore = backupRestore + self.connectionString = connectionString + self.serverMode = serverMode + } + + public var enabled: Set { + var result = Set() + if processIsolated { result.insert(.processIsolated) } + if crashRestartable { result.insert(.crashRestartable) } + if hangRestartable { result.insert(.hangRestartable) } + if sameRootLogicalReopen { result.insert(.sameRootLogicalReopen) } + if rootSwitchable { result.insert(.rootSwitchable) } + if multiRoot { result.insert(.multiRoot) } + if independentSessions { result.insert(.independentSessions) } + if backgroundContinuable { result.insert(.backgroundContinuable) } + if protocolRaw { result.insert(.protocolRaw) } + if protocolStream { result.insert(.protocolStream) } + if streamingRequestInput { result.insert(.streamingRequestInput) } + if queryCancel { result.insert(.queryCancel) } + if backupRestore { result.insert(.backupRestore) } + return result + } +} + +public enum BrokerRejectionReason: Equatable, Codable, Sendable { + case invalidRequest(String) + case canceled + case queueClosed + case rootOpen + case unsupportedCapability(BrokerCapability) +} + +public enum BrokerError: Error, Equatable, Codable, Sendable, CustomStringConvertible { + case brokerUnavailable + case unsupportedOS + case extensionMissing + case incompatibleProtocol(minimum: UInt16, maximum: UInt16) + case incompatibleABI(expected: UInt32, actual: UInt32) + case runtimeMismatch(expected: String, actual: String) + case rootMismatch(expected: String, actual: String) + case invalidConfiguration(String) + case notStarted + case rejected(BrokerRejectionReason) + case outcomeUnknown(epoch: BrokerEpoch, requestID: BrokerRequestID) + case canceled + case deadlineExceeded + case workerInterrupted(epoch: BrokerEpoch?) + case protocolViolation(String) + case databaseClosed + + public var description: String { + switch self { + case .brokerUnavailable: "iOS broker is unavailable" + case .unsupportedOS: "iOS broker requires iOS 26 or newer" + case .extensionMissing: "broker app extension is missing" + case .incompatibleProtocol(let minimum, let maximum): + "no compatible broker protocol version in \(minimum)...\(maximum)" + case .incompatibleABI(let expected, let actual): + "liboliphaunt ABI mismatch: expected \(expected), got \(actual)" + case .runtimeMismatch(let expected, let actual): + "liboliphaunt runtime mismatch: expected \(expected), got \(actual)" + case .rootMismatch(let expected, let actual): + "broker root mismatch: expected \(expected), got \(actual)" + case .invalidConfiguration(let reason): "invalid broker configuration: \(reason)" + case .notStarted: "request did not start" + case .rejected(let reason): "broker rejected request: \(reason)" + case .outcomeUnknown(let epoch, let requestID): + "request outcome is unknown (epoch \(epoch), request \(requestID))" + case .canceled: "request was canceled before dispatch" + case .deadlineExceeded: "broker request deadline exceeded" + case .workerInterrupted(let epoch): + "broker worker was interrupted\(epoch.map { " (epoch \($0))" } ?? "")" + case .protocolViolation(let reason): "broker protocol violation: \(reason)" + case .databaseClosed: "database is closed" + } + } +} + +public enum BrokerHandshake { + public static func negotiateVersion(_ hello: BrokerHello) throws -> UInt16 { + guard hello.minimumProtocolVersion <= hello.maximumProtocolVersion else { + throw BrokerError.invalidConfiguration("minimum protocol version exceeds maximum") + } + let minimum = max(hello.minimumProtocolVersion, OliphauntBrokerProtocol.minimumVersion) + let maximum = min(hello.maximumProtocolVersion, OliphauntBrokerProtocol.maximumVersion) + guard minimum <= maximum else { + throw BrokerError.incompatibleProtocol( + minimum: hello.minimumProtocolVersion, + maximum: hello.maximumProtocolVersion + ) + } + return maximum + } + + public static func validate( + _ hello: BrokerHello, + actualABI: UInt32, + actualRuntimeVersion: String, + residentRootID: String?, + startupConfigurationDigest: String + ) throws -> UInt16 { + let version = try negotiateVersion(hello) + guard hello.expectedABI == actualABI else { + throw BrokerError.incompatibleABI(expected: hello.expectedABI, actual: actualABI) + } + if let expected = hello.expectedRuntimeVersion, expected != actualRuntimeVersion { + throw BrokerError.runtimeMismatch(expected: expected, actual: actualRuntimeVersion) + } + guard hello.rootID == OliphauntBrokerProtocol.canonicalRootID else { + throw BrokerError.rootMismatch( + expected: OliphauntBrokerProtocol.canonicalRootID, + actual: hello.rootID + ) + } + if let residentRootID, residentRootID != hello.rootID { + throw BrokerError.rootMismatch(expected: residentRootID, actual: hello.rootID) + } + guard hello.startupConfigurationDigest == startupConfigurationDigest else { + throw BrokerError.invalidConfiguration("startup-configuration digest mismatch") + } + return version + } +} + +public enum BrokerProtocolError: Error, Equatable, Sendable, CustomStringConvertible { + case invalidMagic + case unsupportedVersion(UInt16) + case invalidHeaderLength(UInt16) + case unknownFrameType(UInt8) + case invalidFlags(UInt8) + case nonzeroReserved(UInt16) + case staleEpoch(expected: BrokerEpoch, actual: BrokerEpoch) + case invalidRequestID(UInt64) + case invalidRequestIDForFrame(frameType: BrokerFrameType, requestID: UInt64) + case payloadTooLarge(actual: UInt64, maximum: Int) + case arithmeticOverflow + case illegalFrame(frameType: BrokerFrameType, state: String) + case truncatedFrame + case malformedFrontendProtocol(String) + case requestIDSpaceExhausted + + public var description: String { + switch self { + case .invalidMagic: "invalid OLPB frame magic" + case .unsupportedVersion(let version): "unsupported broker protocol version \(version)" + case .invalidHeaderLength(let length): "unsupported broker header length \(length)" + case .unknownFrameType(let raw): "unknown broker frame type \(raw)" + case .invalidFlags(let flags): "unknown broker frame flags 0x\(String(flags, radix: 16))" + case .nonzeroReserved(let reserved): "broker frame reserved field is nonzero: \(reserved)" + case .staleEpoch(let expected, let actual): + "stale broker epoch \(actual); current epoch is \(expected)" + case .invalidRequestID(let requestID): "invalid broker request ID \(requestID)" + case .invalidRequestIDForFrame(let frameType, let requestID): + "request ID \(requestID) is invalid for \(frameType)" + case .payloadTooLarge(let actual, let maximum): + "broker frame/request payload \(actual) exceeds limit \(maximum)" + case .arithmeticOverflow: "broker frame length arithmetic overflow" + case .illegalFrame(let frameType, let state): + "broker frame \(frameType) is illegal while request is \(state)" + case .truncatedFrame: "broker channel ended with a truncated frame" + case .malformedFrontendProtocol(let reason): + "malformed PostgreSQL frontend protocol: \(reason)" + case .requestIDSpaceExhausted: "broker request ID space is exhausted for this epoch" + } + } +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerStateMachines.swift b/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerStateMachines.swift new file mode 100644 index 00000000..d7f28510 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerProtocol/BrokerStateMachines.swift @@ -0,0 +1,240 @@ +import Foundation + +public enum IOSBrokerManagerState: Equatable, Sendable { + case unavailable + case idle + case launching + case binding + case recovering + case ready(BrokerEpoch) + case quiescing(BrokerEpoch) + case interrupted(BrokerEpoch) + case closing + + public var epoch: BrokerEpoch? { + switch self { + case .ready(let epoch), .quiescing(let epoch), .interrupted(let epoch): epoch + default: nil + } + } +} + +public enum BrokerRequestState: Equatable, Sendable, CustomStringConvertible { + case queued + case receiving + case readyToDispatch + case running + case cancelRequested + case terminal(BrokerTerminalResult) + + public var description: String { + switch self { + case .queued: "queued" + case .receiving: "receiving" + case .readyToDispatch: "readyToDispatch" + case .running: "running" + case .cancelRequested: "cancelRequested" + case .terminal(let result): "terminal(\(result))" + } + } + + public var isTerminal: Bool { + if case .terminal = self { return true } + return false + } +} + +public enum BrokerTerminalResult: Equatable, Sendable { + case completed + case rejected(BrokerRejectionReason) + case outcomeUnknown + case canceled + case notStarted +} + +public struct BrokerRequestLifecycle: Equatable, Sendable { + public let epoch: BrokerEpoch + public let requestID: BrokerRequestID + public private(set) var state: BrokerRequestState + public private(set) var nativeDispatchStarted: Bool + + public init(epoch: BrokerEpoch, requestID: BrokerRequestID) { + self.epoch = epoch + self.requestID = requestID + state = .queued + nativeDispatchStarted = false + } + + public mutating func beginReceiving() throws { + try transition(from: [.queued], to: .receiving, frameType: .requestBegin) + } + + public mutating func finishReceiving() throws { + try transition(from: [.receiving], to: .readyToDispatch, frameType: .requestEnd) + } + + public mutating func beginNativeDispatch() throws { + try transition(from: [.readyToDispatch], to: .running, frameType: .requestEnd) + nativeDispatchStarted = true + } + + @discardableResult + public mutating func requestCancellation() -> Bool { + switch state { + case .queued, .receiving, .readyToDispatch: + state = .terminal(.canceled) + return true + case .running: + state = .cancelRequested + return true + case .cancelRequested, .terminal: + return false + } + } + + @discardableResult + public mutating func establishTerminal(_ result: BrokerTerminalResult) -> Bool { + guard !state.isTerminal else { + return false + } + state = .terminal(result) + return true + } + + public func lossResult() -> BrokerTerminalResult { + nativeDispatchStarted ? .outcomeUnknown : .notStarted + } + + private mutating func transition( + from allowed: [BrokerRequestState], + to newState: BrokerRequestState, + frameType: BrokerFrameType + ) throws { + guard allowed.contains(state) else { + throw BrokerProtocolError.illegalFrame(frameType: frameType, state: state.description) + } + state = newState + } +} + +public struct BrokerFrontendRequestAssembler: Sendable { + public let maximumRequestBytes: Int + private var bytes = Data() + private var scanOffset = 0 + + public init(maximumRequestBytes: Int = OliphauntBrokerProtocol.defaultMaximumRequestBytes) { + precondition(maximumRequestBytes >= 5) + self.maximumRequestBytes = maximumRequestBytes + } + + public var byteCount: Int { bytes.count } + + public mutating func append(_ chunk: Data) throws { + let (newCount, overflow) = bytes.count.addingReportingOverflow(chunk.count) + guard !overflow else { + throw BrokerProtocolError.arithmeticOverflow + } + guard newCount <= maximumRequestBytes else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(newCount), + maximum: maximumRequestBytes + ) + } + bytes.append(chunk) + try scanCompleteMessages() + } + + public mutating func finish() throws -> Data { + try scanCompleteMessages() + guard !bytes.isEmpty else { + throw BrokerProtocolError.malformedFrontendProtocol("empty request") + } + guard scanOffset == bytes.count else { + let remaining = bytes.count - scanOffset + throw BrokerProtocolError.malformedFrontendProtocol( + remaining < 5 ? "truncated message header" : "truncated message body" + ) + } + return bytes + } + + public mutating func reset() { + bytes.removeAll(keepingCapacity: true) + scanOffset = 0 + } + + private mutating func scanCompleteMessages() throws { + while bytes.count - scanOffset >= 5 { + let lengthOffset = scanOffset + 1 + let messageLength = + (UInt32(bytes[lengthOffset]) << 24) | (UInt32(bytes[lengthOffset + 1]) << 16) + | (UInt32(bytes[lengthOffset + 2]) << 8) | UInt32(bytes[lengthOffset + 3]) + guard messageLength >= 4 else { + throw BrokerProtocolError.malformedFrontendProtocol( + "message length is smaller than protocol header" + ) + } + let (totalLength, overflow) = Int(messageLength).addingReportingOverflow(1) + guard !overflow else { + throw BrokerProtocolError.arithmeticOverflow + } + guard totalLength <= maximumRequestBytes else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(totalLength), + maximum: maximumRequestBytes + ) + } + guard totalLength <= bytes.count - scanOffset else { + return + } + scanOffset += totalLength + } + } +} + +public struct BrokerRootManifest: Equatable, Codable, Sendable { + public var formatVersion: UInt32 + public var postgresMajorVersion: UInt16 + public var liboliphauntVersion: String + public var cABIVersion: UInt32 + public var rootUUID: UUID + public var selectedPostgresExtensions: [String] + public var startupConfigurationDigest: String + public var dataProtectionPolicy: String + + public init( + formatVersion: UInt32 = 1, + postgresMajorVersion: UInt16, + liboliphauntVersion: String, + cABIVersion: UInt32, + rootUUID: UUID, + selectedPostgresExtensions: [String], + startupConfigurationDigest: String, + dataProtectionPolicy: String + ) { + self.formatVersion = formatVersion + self.postgresMajorVersion = postgresMajorVersion + self.liboliphauntVersion = liboliphauntVersion + self.cABIVersion = cABIVersion + self.rootUUID = rootUUID + self.selectedPostgresExtensions = selectedPostgresExtensions + self.startupConfigurationDigest = startupConfigurationDigest + self.dataProtectionPolicy = dataProtectionPolicy + } +} + +public enum BrokerWorkerFault: String, Codable, CaseIterable, Sendable { + case beforeNativeDispatch + case duringNativeExecution + case afterNativeSuccessBeforeCompleted + case afterResponseChunks + case duringCheckpoint + case duringBackup + case duringRestore + case abort + case invalidMemoryAccess + case deadlock + #if DEBUG + case deadlockWithFailStop + #endif +} diff --git a/src/sdks/swift/Sources/OliphauntBrokerXPC/IOSBrokerXPC.swift b/src/sdks/swift/Sources/OliphauntBrokerXPC/IOSBrokerXPC.swift new file mode 100644 index 00000000..0b1e361b --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntBrokerXPC/IOSBrokerXPC.swift @@ -0,0 +1,667 @@ +import Darwin +import Foundation +import OliphauntBrokerProtocol +import XPC + +/// An explicitly owned descriptor. Initializers take ownership and `close()` is +/// idempotent. `xpc_fd_create` and `xpc_fd_dup` each create a new descriptor; +/// callers therefore never exchange bare integer descriptor values. +public final class IOSBrokerOwnedFileDescriptor: @unchecked Sendable { + private let lock = NSLock() + private var rawDescriptor: Int32 + + public init(takingOwnershipOf descriptor: Int32) throws { + guard descriptor >= 0 else { + throw POSIXError(.EBADF) + } + rawDescriptor = descriptor + } + + deinit { + close() + } + + public var isOpen: Bool { + lock.lock() + defer { lock.unlock() } + return rawDescriptor >= 0 + } + + /// The returned value is borrowed and remains owned by this object. + public func borrowedDescriptor() throws -> Int32 { + lock.lock() + defer { lock.unlock() } + guard rawDescriptor >= 0 else { + throw POSIXError(.EBADF) + } + return rawDescriptor + } + + /// Transfers ownership to the caller and prevents this object from closing + /// the descriptor. This is used when the XPC duplicate is adopted by the + /// extension's socket worker. + public func takeDescriptor() throws -> Int32 { + lock.lock() + defer { lock.unlock() } + guard rawDescriptor >= 0 else { + throw POSIXError(.EBADF) + } + let descriptor = rawDescriptor + rawDescriptor = -1 + return descriptor + } + + @discardableResult + public func close() -> Bool { + lock.lock() + let descriptor = rawDescriptor + rawDescriptor = -1 + lock.unlock() + guard descriptor >= 0 else { + return false + } + Darwin.close(descriptor) + return true + } +} + +public struct IOSBrokerControlEnvelope: Sendable { + public var kind: BrokerControlMessageKind + public var epoch: BrokerEpoch? + public var requestID: BrokerRequestID? + public var deadlineUnixNanoseconds: UInt64? + public var fault: BrokerWorkerFault? + + public init( + kind: BrokerControlMessageKind, + epoch: BrokerEpoch? = nil, + requestID: BrokerRequestID? = nil, + deadlineUnixNanoseconds: UInt64? = nil, + fault: BrokerWorkerFault? = nil + ) { + self.kind = kind + self.epoch = epoch + self.requestID = requestID + self.deadlineUnixNanoseconds = deadlineUnixNanoseconds + self.fault = fault + } +} + +/// Path-free diagnostics decoded by the shared XPC layer. The host adapter +/// maps this wire value to its public diagnostics type; keeping the wire DTO in +/// this protocol-only module prevents the extension from importing the host +/// manager/session implementation. +public struct IOSBrokerWireCheckpointMemorySample: Equatable, Sendable { + public var sequence: UInt64 + public var startedAtUptimeNanoseconds: UInt64 + public var sampledAtUptimeNanoseconds: UInt64 + public var completedAtUptimeNanoseconds: UInt64 + public var physFootprintBytes: UInt64 + public var residentBytes: UInt64 + public var availableMemoryBytes: UInt64 + + public init( + sequence: UInt64, + startedAtUptimeNanoseconds: UInt64, + sampledAtUptimeNanoseconds: UInt64, + completedAtUptimeNanoseconds: UInt64, + physFootprintBytes: UInt64, + residentBytes: UInt64, + availableMemoryBytes: UInt64 + ) { + self.sequence = sequence + self.startedAtUptimeNanoseconds = startedAtUptimeNanoseconds + self.sampledAtUptimeNanoseconds = sampledAtUptimeNanoseconds + self.completedAtUptimeNanoseconds = completedAtUptimeNanoseconds + self.physFootprintBytes = physFootprintBytes + self.residentBytes = residentBytes + self.availableMemoryBytes = availableMemoryBytes + } +} + +public struct IOSBrokerWireDiagnostics: Equatable, Sendable { + public var state: String + public var epoch: BrokerEpoch + public var extensionProcessIdentifier: Int32 + public var manifestDigest: String? + public var activeRequestID: BrokerRequestID? + public var nativeDispatchStarted: Bool + public var transactionStatus: String + public var capabilities: BrokerCapabilities + public var currentPhysFootprintBytes: UInt64? + public var currentResidentBytes: UInt64? + public var availableMemoryBytes: UInt64? + public var checkpointInProgress: Bool + public var checkpointMemorySample: IOSBrokerWireCheckpointMemorySample? + public var storageProtectionEvidenceJSON: String? + public var extensionEntryPreOpenPhysFootprintBytes: UInt64? + public var extensionEntryPreOpenResidentBytes: UInt64? + public var openedIdlePhysFootprintBytes: UInt64? + public var openedIdleResidentBytes: UInt64? + + public init( + state: String, + epoch: BrokerEpoch, + extensionProcessIdentifier: Int32, + manifestDigest: String?, + activeRequestID: BrokerRequestID?, + nativeDispatchStarted: Bool, + transactionStatus: String, + capabilities: BrokerCapabilities, + currentPhysFootprintBytes: UInt64?, + currentResidentBytes: UInt64?, + availableMemoryBytes: UInt64?, + checkpointInProgress: Bool, + checkpointMemorySample: IOSBrokerWireCheckpointMemorySample? = nil, + storageProtectionEvidenceJSON: String?, + extensionEntryPreOpenPhysFootprintBytes: UInt64?, + extensionEntryPreOpenResidentBytes: UInt64?, + openedIdlePhysFootprintBytes: UInt64?, + openedIdleResidentBytes: UInt64? + ) { + self.state = state + self.epoch = epoch + self.extensionProcessIdentifier = extensionProcessIdentifier + self.manifestDigest = manifestDigest + self.activeRequestID = activeRequestID + self.nativeDispatchStarted = nativeDispatchStarted + self.transactionStatus = transactionStatus + self.capabilities = capabilities + self.currentPhysFootprintBytes = currentPhysFootprintBytes + self.currentResidentBytes = currentResidentBytes + self.availableMemoryBytes = availableMemoryBytes + self.checkpointInProgress = checkpointInProgress + self.checkpointMemorySample = checkpointMemorySample + self.storageProtectionEvidenceJSON = storageProtectionEvidenceJSON + self.extensionEntryPreOpenPhysFootprintBytes = + extensionEntryPreOpenPhysFootprintBytes + self.extensionEntryPreOpenResidentBytes = extensionEntryPreOpenResidentBytes + self.openedIdlePhysFootprintBytes = openedIdlePhysFootprintBytes + self.openedIdleResidentBytes = openedIdleResidentBytes + } +} + +/// The primitive-only lightweight-XPC control schema shared by a host harness +/// and its extension entry point. Potentially large PostgreSQL bytes are +/// deliberately absent; they travel only through the framed socket. +@available(iOS 26.0, macOS 26.0, *) +public enum IOSBrokerXPC { + public static let successKey = "success" + public static let cancelledActiveWorkKey = "cancelledActiveWork" + public static let checkpointedKey = "checkpointed" + public static let rejectionKey = "rejection" + public static let stateKey = "state" + public static let manifestDigestKey = "manifestDigest" + public static let activeRequestIDKey = "activeRequestID" + public static let nativeDispatchStartedKey = "nativeDispatchStarted" + public static let transactionStatusKey = "transactionStatus" + public static let capabilitiesKey = "capabilities" + public static let currentPhysFootprintBytesKey = "currentPhysFootprintBytes" + public static let currentResidentBytesKey = "currentResidentBytes" + public static let availableMemoryBytesKey = "availableMemoryBytes" + public static let checkpointInProgressKey = "checkpointInProgress" + public static let checkpointMemorySampleSequenceKey = "checkpointMemorySampleSequence" + public static let checkpointMemorySampleStartedAtUptimeNanosecondsKey = + "checkpointMemorySampleStartedAtUptimeNanoseconds" + public static let checkpointMemorySampledAtUptimeNanosecondsKey = + "checkpointMemorySampledAtUptimeNanoseconds" + public static let checkpointMemorySampleCompletedAtUptimeNanosecondsKey = + "checkpointMemorySampleCompletedAtUptimeNanoseconds" + public static let checkpointMemorySamplePhysFootprintBytesKey = + "checkpointMemorySamplePhysFootprintBytes" + public static let checkpointMemorySampleResidentBytesKey = + "checkpointMemorySampleResidentBytes" + public static let checkpointMemorySampleAvailableMemoryBytesKey = + "checkpointMemorySampleAvailableMemoryBytes" + public static let storageProtectionEvidenceJSONKey = "storageProtectionEvidenceJSON" + public static let extensionEntryPreOpenPhysFootprintBytesKey = + "extensionEntryPreOpenPhysFootprintBytes" + public static let extensionEntryPreOpenResidentBytesKey = + "extensionEntryPreOpenResidentBytes" + public static let openedIdlePhysFootprintBytesKey = "openedIdlePhysFootprintBytes" + public static let openedIdleResidentBytesKey = "openedIdleResidentBytes" + + public static func makeHello( + _ hello: BrokerHello, + dataChannel descriptor: IOSBrokerOwnedFileDescriptor + ) throws -> XPCDictionary { + var dictionary = XPCDictionary() + dictionary[BrokerControlKey.message] = BrokerControlMessageKind.hello.rawValue + dictionary[BrokerControlKey.minimumProtocolVersion] = UInt64(hello.minimumProtocolVersion) + dictionary[BrokerControlKey.maximumProtocolVersion] = UInt64(hello.maximumProtocolVersion) + dictionary[BrokerControlKey.expectedABI] = UInt64(hello.expectedABI) + if let expectedRuntimeVersion = hello.expectedRuntimeVersion { + dictionary[BrokerControlKey.expectedRuntimeVersion] = expectedRuntimeVersion + } + dictionary[BrokerControlKey.rootID] = hello.rootID + dictionary[BrokerControlKey.startupConfigurationDigest] = + hello.startupConfigurationDigest + dictionary[BrokerControlKey.requestedCapabilities] = try encodeJSON( + hello.requestedCapabilities + ) + dictionary[BrokerControlKey.dataChannel] = try box(descriptor) + return dictionary + } + + public static func decodeHello( + _ dictionary: XPCDictionary + ) throws -> (hello: BrokerHello, dataChannel: IOSBrokerOwnedFileDescriptor) { + guard try messageKind(in: dictionary) == .hello else { + throw BrokerError.protocolViolation("expected Hello control message") + } + let minimum = try uint16(dictionary, BrokerControlKey.minimumProtocolVersion) + let maximum = try uint16(dictionary, BrokerControlKey.maximumProtocolVersion) + let expectedABI = try uint32(dictionary, BrokerControlKey.expectedABI) + let expectedRuntimeVersion: String? = dictionary[BrokerControlKey.expectedRuntimeVersion] + let rootID = try string(dictionary, BrokerControlKey.rootID) + let digest = try string(dictionary, BrokerControlKey.startupConfigurationDigest) + let encodedCapabilities = try string( + dictionary, + BrokerControlKey.requestedCapabilities + ) + let capabilities: Set = try decodeJSON(encodedCapabilities) + return ( + BrokerHello( + minimumProtocolVersion: minimum, + maximumProtocolVersion: maximum, + expectedABI: expectedABI, + expectedRuntimeVersion: expectedRuntimeVersion, + rootID: rootID, + startupConfigurationDigest: digest, + requestedCapabilities: capabilities + ), + try duplicateDescriptor(in: dictionary, key: BrokerControlKey.dataChannel) + ) + } + + public static func makeReady(_ ready: BrokerReady) throws -> XPCDictionary { + var dictionary = XPCDictionary() + dictionary[BrokerControlKey.message] = BrokerControlMessageKind.ready.rawValue + dictionary[BrokerControlKey.selectedProtocolVersion] = + UInt64(ready.selectedProtocolVersion) + dictionary[BrokerControlKey.epoch] = ready.epoch.description + dictionary[BrokerControlKey.extensionPID] = Int64(ready.extensionPID) + dictionary[BrokerControlKey.runtimeVersion] = ready.runtimeVersion + dictionary[BrokerControlKey.abiVersion] = UInt64(ready.abiVersion) + dictionary[BrokerControlKey.postgresMajorVersion] = + UInt64(ready.postgresMajorVersion) + dictionary[BrokerControlKey.rootManifestDigest] = ready.rootManifestDigest + dictionary[BrokerControlKey.actualCapabilities] = try encodeJSON( + ready.actualCapabilities + ) + dictionary[BrokerControlKey.actualRuntimeConfiguration] = try encodeJSON( + ready.actualRuntimeConfiguration + ) + return dictionary + } + + public static func decodeReady(_ dictionary: XPCDictionary) throws -> BrokerReady { + let kind = try messageKind(in: dictionary) + if kind == .rejected { + throw try decodeError(dictionary) + } + guard kind == .ready else { + throw BrokerError.protocolViolation("expected Ready, received \(kind.rawValue)") + } + + let epochString = try string(dictionary, BrokerControlKey.epoch) + guard let epochUUID = UUID(uuidString: epochString) else { + throw BrokerError.protocolViolation("Ready contains an invalid epoch UUID") + } + let pidValue = try int64(dictionary, BrokerControlKey.extensionPID) + guard let pid = Int32(exactly: pidValue) else { + throw BrokerError.protocolViolation("Ready contains an invalid worker PID") + } + let encodedCapabilities = try string(dictionary, BrokerControlKey.actualCapabilities) + let encodedRuntimeConfiguration = try string( + dictionary, + BrokerControlKey.actualRuntimeConfiguration + ) + return BrokerReady( + selectedProtocolVersion: try uint16( + dictionary, + BrokerControlKey.selectedProtocolVersion + ), + epoch: BrokerEpoch(epochUUID), + extensionPID: pid, + runtimeVersion: try string(dictionary, BrokerControlKey.runtimeVersion), + abiVersion: try uint32(dictionary, BrokerControlKey.abiVersion), + postgresMajorVersion: try uint16( + dictionary, + BrokerControlKey.postgresMajorVersion + ), + rootManifestDigest: try string(dictionary, BrokerControlKey.rootManifestDigest), + actualCapabilities: try decodeJSON(encodedCapabilities), + actualRuntimeConfiguration: try decodeJSON(encodedRuntimeConfiguration) + ) + } + + public static func makeRejected(_ rejection: BrokerRejectionReason) throws -> XPCDictionary { + var dictionary = XPCDictionary() + dictionary[BrokerControlKey.message] = BrokerControlMessageKind.rejected.rawValue + dictionary[rejectionKey] = try encodeJSON(rejection) + dictionary[BrokerControlKey.reason] = String(describing: rejection) + return dictionary + } + + /// Returns the path-free error that may cross the extension boundary. + /// Intrinsically typed handshake failures retain their identity. Free-form + /// strings are either explicitly allowlisted or replaced with stable text. + public static func extensionBoundaryError(_ error: any Error) -> BrokerError { + guard let brokerError = error as? BrokerError else { + if error is BrokerProtocolError { + return .protocolViolation("extension control message was invalid") + } + return .brokerUnavailable + } + + switch brokerError { + case .invalidConfiguration(let reason): + if pathFreeConfigurationReasons.contains(reason) { + return brokerError + } + return .invalidConfiguration("extension configuration was rejected") + case .protocolViolation: + return .protocolViolation("extension control message was invalid") + case .rejected(.invalidRequest(let reason)): + if pathFreeInvalidRequestReasons.contains(reason) { + return brokerError + } + return .rejected(.invalidRequest("extension rejected the request")) + default: + return brokerError + } + } + + /// Encodes a structured, path-free broker failure for handshake and control + /// replies. ABI/runtime/root/protocol failures retain their typed identity. + public static func makeError(_ error: BrokerError) throws -> XPCDictionary { + try makeBoundaryError(extensionBoundaryError(error)) + } + + /// Encodes an arbitrary extension error without reflecting its description. + public static func makeError(_ error: any Error) throws -> XPCDictionary { + try makeBoundaryError(extensionBoundaryError(error)) + } + + private static func makeBoundaryError(_ error: BrokerError) throws -> XPCDictionary { + var dictionary = XPCDictionary() + dictionary[BrokerControlKey.message] = BrokerControlMessageKind.rejected.rawValue + dictionary[BrokerControlKey.error] = try encodeJSON(error) + dictionary[BrokerControlKey.reason] = error.description + return dictionary + } + + public static func decodeError(_ dictionary: XPCDictionary) throws -> BrokerError { + if let encoded: String = dictionary[BrokerControlKey.error] { + return try decodeJSON(encoded) + } + return .rejected(try decodeRejection(dictionary)) + } + + public static func decodeRejection( + _ dictionary: XPCDictionary + ) throws -> BrokerRejectionReason { + if let encoded: String = dictionary[rejectionKey] { + return try decodeJSON(encoded) + } + let reason: String = dictionary[BrokerControlKey.reason] ?? "unspecified rejection" + return .invalidRequest(reason) + } + + public static func decodeWorkerDiagnostics( + _ dictionary: XPCDictionary + ) throws -> IOSBrokerWireDiagnostics { + guard try messageKind(in: dictionary) == .diagnostics else { + throw BrokerError.protocolViolation("expected Diagnostics control reply") + } + let epochText = try string(dictionary, BrokerControlKey.epoch) + guard let epochUUID = UUID(uuidString: epochText) else { + throw BrokerError.protocolViolation("diagnostics contains an invalid epoch") + } + let pidValue = try int64(dictionary, BrokerControlKey.extensionPID) + guard let pid = Int32(exactly: pidValue) else { + throw BrokerError.protocolViolation("diagnostics contains an invalid worker PID") + } + let activeRequestID: BrokerRequestID? + if let rawRequestID: UInt64 = dictionary[activeRequestIDKey] { + activeRequestID = try BrokerRequestID(validating: rawRequestID) + } else { + activeRequestID = nil + } + guard let nativeDispatchStarted: Bool = dictionary[nativeDispatchStartedKey] else { + throw BrokerError.protocolViolation( + "diagnostics is missing nativeDispatchStarted" + ) + } + let encodedCapabilities = try string(dictionary, capabilitiesKey) + return IOSBrokerWireDiagnostics( + state: try string(dictionary, stateKey), + epoch: BrokerEpoch(epochUUID), + extensionProcessIdentifier: pid, + manifestDigest: dictionary[manifestDigestKey], + activeRequestID: activeRequestID, + nativeDispatchStarted: nativeDispatchStarted, + transactionStatus: try string(dictionary, transactionStatusKey), + capabilities: try decodeJSON(encodedCapabilities), + currentPhysFootprintBytes: dictionary[currentPhysFootprintBytesKey], + currentResidentBytes: dictionary[currentResidentBytesKey], + availableMemoryBytes: dictionary[availableMemoryBytesKey], + checkpointInProgress: dictionary[checkpointInProgressKey] ?? false, + checkpointMemorySample: try decodeCheckpointMemorySample(dictionary), + storageProtectionEvidenceJSON: dictionary[storageProtectionEvidenceJSONKey], + extensionEntryPreOpenPhysFootprintBytes: dictionary[ + extensionEntryPreOpenPhysFootprintBytesKey + ], + extensionEntryPreOpenResidentBytes: dictionary[ + extensionEntryPreOpenResidentBytesKey + ], + openedIdlePhysFootprintBytes: dictionary[openedIdlePhysFootprintBytesKey], + openedIdleResidentBytes: dictionary[openedIdleResidentBytesKey] + ) + } + + public static func makeControl(_ envelope: IOSBrokerControlEnvelope) -> XPCDictionary { + var dictionary = XPCDictionary() + dictionary[BrokerControlKey.message] = envelope.kind.rawValue + if let epoch = envelope.epoch { + dictionary[BrokerControlKey.epoch] = epoch.description + } + if let requestID = envelope.requestID { + dictionary[BrokerControlKey.requestID] = requestID.rawValue + } + if let deadline = envelope.deadlineUnixNanoseconds { + dictionary[BrokerControlKey.deadlineUnixNanoseconds] = deadline + } + if let fault = envelope.fault { + dictionary[BrokerControlKey.fault] = fault.rawValue + } + return dictionary + } + + public static func decodeControl( + _ dictionary: XPCDictionary + ) throws + -> IOSBrokerControlEnvelope + { + let kind = try messageKind(in: dictionary) + let epoch: BrokerEpoch? + if let value: String = dictionary[BrokerControlKey.epoch] { + guard let uuid = UUID(uuidString: value) else { + throw BrokerError.protocolViolation("control message has an invalid epoch") + } + epoch = BrokerEpoch(uuid) + } else { + epoch = nil + } + let requestID: BrokerRequestID? + if let raw: UInt64 = dictionary[BrokerControlKey.requestID] { + requestID = try BrokerRequestID(validating: raw) + } else { + requestID = nil + } + let deadline: UInt64? = dictionary[BrokerControlKey.deadlineUnixNanoseconds] + let fault: BrokerWorkerFault? + if let rawFault: String = dictionary[BrokerControlKey.fault] { + guard let value = BrokerWorkerFault(rawValue: rawFault) else { + throw BrokerError.protocolViolation("control message has an unknown fault") + } + fault = value + } else { + fault = nil + } + return IOSBrokerControlEnvelope( + kind: kind, + epoch: epoch, + requestID: requestID, + deadlineUnixNanoseconds: deadline, + fault: fault + ) + } + + public static func makeAcknowledgement( + _ kind: BrokerControlMessageKind, + success: Bool = true + ) -> XPCDictionary { + var dictionary = XPCDictionary() + dictionary[BrokerControlKey.message] = kind.rawValue + dictionary[successKey] = success + return dictionary + } + + public static func messageKind( + in dictionary: XPCDictionary + ) throws -> BrokerControlMessageKind { + let value = try string(dictionary, BrokerControlKey.message) + guard let kind = BrokerControlMessageKind(rawValue: value) else { + throw BrokerError.protocolViolation("unknown control message '\(value)'") + } + return kind + } + + public static func duplicateDescriptor( + in dictionary: XPCDictionary, + key: String + ) throws -> IOSBrokerOwnedFileDescriptor { + guard let boxed = dictionary[key, as: XPC_TYPE_FD] else { + throw BrokerError.protocolViolation("missing XPC file descriptor '\(key)'") + } + let duplicate = xpc_fd_dup(boxed) + guard duplicate >= 0 else { + throw POSIXError(.EBADF) + } + return try IOSBrokerOwnedFileDescriptor(takingOwnershipOf: duplicate) + } + + private static func box(_ descriptor: IOSBrokerOwnedFileDescriptor) throws -> xpc_object_t { + let rawDescriptor = try descriptor.borrowedDescriptor() + guard let boxed = xpc_fd_create(rawDescriptor) else { + throw POSIXError(.EBADF) + } + return boxed + } + + private static func string(_ dictionary: XPCDictionary, _ key: String) throws -> String { + guard let value: String = dictionary[key] else { + throw BrokerError.protocolViolation("missing string control field '\(key)'") + } + return value + } + + private static func uint16(_ dictionary: XPCDictionary, _ key: String) throws -> UInt16 { + let raw = try uint64(dictionary, key) + guard let value = UInt16(exactly: raw) else { + throw BrokerError.protocolViolation("control field '\(key)' exceeds UInt16") + } + return value + } + + private static func uint32(_ dictionary: XPCDictionary, _ key: String) throws -> UInt32 { + let raw = try uint64(dictionary, key) + guard let value = UInt32(exactly: raw) else { + throw BrokerError.protocolViolation("control field '\(key)' exceeds UInt32") + } + return value + } + + private static func uint64(_ dictionary: XPCDictionary, _ key: String) throws -> UInt64 { + guard let value: UInt64 = dictionary[key] else { + throw BrokerError.protocolViolation("missing unsigned control field '\(key)'") + } + return value + } + + private static func int64(_ dictionary: XPCDictionary, _ key: String) throws -> Int64 { + guard let value: Int64 = dictionary[key] else { + throw BrokerError.protocolViolation("missing signed control field '\(key)'") + } + return value + } + + private static func decodeCheckpointMemorySample( + _ dictionary: XPCDictionary + ) throws -> IOSBrokerWireCheckpointMemorySample? { + let values: [UInt64?] = [ + dictionary[checkpointMemorySampleSequenceKey], + dictionary[checkpointMemorySampleStartedAtUptimeNanosecondsKey], + dictionary[checkpointMemorySampledAtUptimeNanosecondsKey], + dictionary[checkpointMemorySampleCompletedAtUptimeNanosecondsKey], + dictionary[checkpointMemorySamplePhysFootprintBytesKey], + dictionary[checkpointMemorySampleResidentBytesKey], + dictionary[checkpointMemorySampleAvailableMemoryBytesKey], + ] + if values.allSatisfy({ $0 == nil }) { + return nil + } + guard + let sequence = values[0], + let startedAtUptimeNanoseconds = values[1], + let sampledAtUptimeNanoseconds = values[2], + let completedAtUptimeNanoseconds = values[3], + let physFootprintBytes = values[4], + let residentBytes = values[5], + let availableMemoryBytes = values[6] + else { + throw BrokerError.protocolViolation( + "diagnostics contains incomplete checkpoint memory evidence" + ) + } + return IOSBrokerWireCheckpointMemorySample( + sequence: sequence, + startedAtUptimeNanoseconds: startedAtUptimeNanoseconds, + sampledAtUptimeNanoseconds: sampledAtUptimeNanoseconds, + completedAtUptimeNanoseconds: completedAtUptimeNanoseconds, + physFootprintBytes: physFootprintBytes, + residentBytes: residentBytes, + availableMemoryBytes: availableMemoryBytes + ) + } + + private static let pathFreeConfigurationReasons: Set = [ + "minimum protocol version exceeds maximum", + "startup-configuration digest mismatch", + ] + + private static let pathFreeInvalidRequestReasons: Set = [ + "a broker data channel is already active", + "a data channel is already active", + "cannot checkpoint while a request is active", + "cannot detach while a request is active", + ] + + private static func encodeJSON(_ value: T) throws -> String { + let data = try JSONEncoder().encode(value) + guard let encoded = String(data: data, encoding: .utf8) else { + throw BrokerError.protocolViolation("failed to encode UTF-8 control field") + } + return encoded + } + + private static func decodeJSON(_ value: String) throws -> T { + do { + return try JSONDecoder().decode(T.self, from: Data(value.utf8)) + } catch { + throw BrokerError.protocolViolation("invalid JSON control field: \(error)") + } + } +} diff --git a/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerDataChannel.swift b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerDataChannel.swift new file mode 100644 index 00000000..7ace2b68 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerDataChannel.swift @@ -0,0 +1,497 @@ +import Darwin +import Dispatch +import Foundation +import OliphauntBrokerProtocol +import OliphauntBrokerXPC + +enum IOSBrokerDataPlaneFailure: Error, Sendable { + case rejected(BrokerRejectionReason) + case outcomeUnknown(String) + case protocolViolation(String) +} + +/// Loss of the socket transport, distinct from a well-formed peer violating the +/// framed broker protocol. Data requests still translate this into +/// `outcomeUnknown`; lifecycle recovery may safely use it as an interruption +/// signal because it never replays caller SQL. +enum IOSBrokerTransportFailure: Error, Equatable, Sendable { + case socketWrite(Int32) + case socketRead(Int32) + case unexpectedEOF +} + +enum IOSBrokerBackendTransactionStatus: UInt8, Sendable { + case idle = 0x49 // I + case inTransaction = 0x54 // T + case failedTransaction = 0x45 // E +} + +struct IOSBrokerSocketPair { + let host: IOSBrokerDataChannel + let extensionEndpoint: IOSBrokerOwnedFileDescriptor + + static func make() throws -> IOSBrokerSocketPair { + var descriptors: [Int32] = [-1, -1] + guard socketpair(AF_UNIX, SOCK_STREAM, 0, &descriptors) == 0 else { + throw POSIXError(POSIXError.Code(rawValue: errno) ?? .EIO) + } + do { + try configure(descriptors[0]) + try configure(descriptors[1]) + let extensionEndpoint = try IOSBrokerOwnedFileDescriptor( + takingOwnershipOf: descriptors[1] + ) + return IOSBrokerSocketPair( + host: IOSBrokerDataChannel(takingOwnershipOf: descriptors[0]), + extensionEndpoint: extensionEndpoint + ) + } catch { + Darwin.close(descriptors[0]) + Darwin.close(descriptors[1]) + throw error + } + } + + private static func configure(_ descriptor: Int32) throws { + let descriptorFlags = fcntl(descriptor, F_GETFD) + guard descriptorFlags >= 0, + fcntl(descriptor, F_SETFD, descriptorFlags | FD_CLOEXEC) >= 0 + else { + throw POSIXError(POSIXError.Code(rawValue: errno) ?? .EIO) + } + let statusFlags = fcntl(descriptor, F_GETFL) + guard statusFlags >= 0, + fcntl(descriptor, F_SETFL, statusFlags | O_NONBLOCK) >= 0 + else { + throw POSIXError(POSIXError.Code(rawValue: errno) ?? .EIO) + } + var enabled: Int32 = 1 + guard + setsockopt( + descriptor, + SOL_SOCKET, + SO_NOSIGPIPE, + &enabled, + socklen_t(MemoryLayout.size(ofValue: enabled)) + ) == 0 + else { + throw POSIXError(POSIXError.Code(rawValue: errno) ?? .EIO) + } + } +} + +/// One epoch's host endpoint. DispatchIO supplies bounded, nonblocking I/O on a +/// private queue, and closing the channel interrupts outstanding reads/writes. +final class IOSBrokerDataChannel: @unchecked Sendable { + private let queue = DispatchQueue(label: "dev.oliphaunt.ios-broker.data-channel") + private let channel: DispatchIO + private let closeLock = NSLock() + private var isClosed = false + + init(takingOwnershipOf descriptor: Int32) { + channel = DispatchIO( + type: .stream, + fileDescriptor: descriptor, + queue: queue + ) { _ in + Darwin.close(descriptor) + } + channel.setLimit(lowWater: 1) + channel.setLimit( + highWater: Int(OliphauntBrokerProtocol.headerLength) + + OliphauntBrokerProtocol.maximumFramePayload + ) + } + + deinit { + close() + } + + func close() { + closeLock.lock() + guard !isClosed else { + closeLock.unlock() + return + } + isClosed = true + closeLock.unlock() + channel.close(flags: .stop) + } + + func healthCheck(epoch: BrokerEpoch, protocolVersion: UInt16) async throws { + let ping = try BrokerFrame( + protocolVersion: protocolVersion, + frameType: .ping, + epoch: epoch, + requestID: 0 + ) + try await write(ping) + let reply = try await readFrame(expectedEpoch: epoch) + guard reply.header.frameType == .pong, + reply.header.requestID == 0, + reply.payload.isEmpty + else { + throw IOSBrokerDataPlaneFailure.protocolViolation( + "health check did not receive an empty Pong" + ) + } + } + + func execute( + requestID: BrokerRequestID, + epoch: BrokerEpoch, + protocolVersion: UInt16, + bytes: Data, + maximumRequestBytes: Int, + onChunk: @escaping @Sendable (Data) throws -> Void + ) async throws -> IOSBrokerBackendTransactionStatus { + var assembler = BrokerFrontendRequestAssembler( + maximumRequestBytes: maximumRequestBytes + ) + do { + try assembler.append(bytes) + _ = try assembler.finish() + } catch { + throw IOSBrokerDataPlaneFailure.protocolViolation(String(describing: error)) + } + + let rawRequestID = requestID.rawValue + do { + try await write( + try BrokerFrame( + protocolVersion: protocolVersion, + frameType: .requestBegin, + epoch: epoch, + requestID: rawRequestID + )) + + var offset = 0 + while offset < bytes.count { + let end = min( + bytes.count, + offset + OliphauntBrokerProtocol.maximumFramePayload + ) + try await write( + try BrokerFrame( + protocolVersion: protocolVersion, + frameType: .requestBytes, + epoch: epoch, + requestID: rawRequestID, + payload: bytes.subdata(in: offset.. BrokerFrame { + let headerBytes = try await readExactly(Int(OliphauntBrokerProtocol.headerLength)) + let header: BrokerFrameHeader + do { + header = try BrokerFrameHeader.decode( + headerBytes, + expectedEpoch: expectedEpoch, + maximumPayloadLength: OliphauntBrokerProtocol.maximumFramePayload + ) + } catch { + throw IOSBrokerDataPlaneFailure.protocolViolation(String(describing: error)) + } + let payload = try await readExactly(Int(header.payloadLength)) + return try BrokerFrame( + protocolVersion: header.protocolVersion, + frameType: header.frameType, + flags: header.flags, + epoch: header.epoch, + requestID: header.requestID, + payload: payload + ) + } + + private func readExactly(_ count: Int) async throws -> Data { + guard count > 0 else { + return Data() + } + return try await withCheckedThrowingContinuation { continuation in + let state = IOReadState(expectedCount: count, continuation: continuation) + channel.read(offset: 0, length: count, queue: queue) { done, data, error in + state.consume(done: done, data: data, error: error) + } + } + } + + private func decodeRejection(_ payload: Data) -> BrokerRejectionReason { + if let decoded = try? JSONDecoder().decode(BrokerRejectionReason.self, from: payload) { + return decoded + } + return .invalidRequest( + String(data: payload, encoding: .utf8) ?? "worker rejected request" + ) + } +} + +/// Incrementally observes backend framing without retaining message bodies. +/// ReadyForQuery (`Z`) is protocol metadata, so transaction ownership can be +/// pinned without parsing or classifying SQL. +struct IOSBrokerBackendResponseObserver { + private var header = Data() + private var messageType: UInt8 = 0 + private var remainingBodyBytes: Int? + private var bodyOffset = 0 + private var lastCompletedMessageType: UInt8? + private(set) var lastReadyStatus: IOSBrokerBackendTransactionStatus? + + init() { + header.reserveCapacity(5) + } + + mutating func append(_ bytes: Data) throws { + var offset = 0 + while offset < bytes.count { + if remainingBodyBytes == nil { + let count = min(5 - header.count, bytes.count - offset) + header.append(bytes.subdata(in: offset..<(offset + count))) + offset += count + guard header.count == 5 else { continue } + + messageType = header[0] + let length = + (UInt32(header[1]) << 24) | (UInt32(header[2]) << 16) + | (UInt32(header[3]) << 8) | UInt32(header[4]) + guard length >= 4 else { + throw IOSBrokerDataPlaneFailure.protocolViolation( + "backend message length is smaller than its header" + ) + } + let bodyLength = Int(length - 4) + if messageType == 0x5A, bodyLength != 1 { + throw IOSBrokerDataPlaneFailure.protocolViolation( + "ReadyForQuery has an invalid length" + ) + } + header.removeAll(keepingCapacity: true) + remainingBodyBytes = bodyLength + bodyOffset = 0 + if bodyLength == 0 { + lastCompletedMessageType = messageType + remainingBodyBytes = nil + } + continue + } + + guard let remaining = remainingBodyBytes else { continue } + let count = min(remaining, bytes.count - offset) + if messageType == 0x5A, bodyOffset == 0, count > 0 { + guard let status = IOSBrokerBackendTransactionStatus(rawValue: bytes[offset]) else { + throw IOSBrokerDataPlaneFailure.protocolViolation( + "ReadyForQuery has an unknown transaction status" + ) + } + lastReadyStatus = status + } + offset += count + bodyOffset += count + let nextRemaining = remaining - count + if nextRemaining == 0 { + lastCompletedMessageType = messageType + } + remainingBodyBytes = nextRemaining == 0 ? nil : nextRemaining + } + } + + func finish() throws -> IOSBrokerBackendTransactionStatus { + guard header.isEmpty, remainingBodyBytes == nil else { + throw IOSBrokerDataPlaneFailure.protocolViolation( + "Completed arrived in the middle of a PostgreSQL backend message" + ) + } + guard lastCompletedMessageType == 0x5A, let lastReadyStatus else { + throw IOSBrokerDataPlaneFailure.protocolViolation( + "Completed arrived without a terminal ReadyForQuery" + ) + } + return lastReadyStatus + } +} + +private final class IOWriteState: @unchecked Sendable { + private let lock = NSLock() + private var finished = false + private let continuation: CheckedContinuation + + init(continuation: CheckedContinuation) { + self.continuation = continuation + } + + func consume(done: Bool, error: Int32) { + guard done || error != 0 else { + return + } + lock.lock() + guard !finished else { + lock.unlock() + return + } + finished = true + lock.unlock() + if error == 0 { + continuation.resume() + } else { + continuation.resume( + throwing: IOSBrokerTransportFailure.socketWrite(error) + ) + } + } +} + +private final class IOReadState: @unchecked Sendable { + private let lock = NSLock() + private let expectedCount: Int + private var bytes = Data() + private var finished = false + private let continuation: CheckedContinuation + + init( + expectedCount: Int, + continuation: CheckedContinuation + ) { + self.expectedCount = expectedCount + self.continuation = continuation + bytes.reserveCapacity(expectedCount) + } + + func consume(done: Bool, data: DispatchData?, error: Int32) { + lock.lock() + guard !finished else { + lock.unlock() + return + } + if let data, !data.isEmpty { + bytes.append(contentsOf: data) + } + if bytes.count > expectedCount { + finished = true + lock.unlock() + continuation.resume( + throwing: BrokerError.protocolViolation("socket read exceeded requested length") + ) + return + } + if error != 0 { + finished = true + lock.unlock() + continuation.resume( + throwing: IOSBrokerTransportFailure.socketRead(error) + ) + return + } + if bytes.count == expectedCount { + let result = bytes + finished = true + lock.unlock() + continuation.resume(returning: result) + return + } + if done { + finished = true + lock.unlock() + continuation.resume( + throwing: IOSBrokerTransportFailure.unexpectedEOF + ) + return + } + lock.unlock() + } +} diff --git a/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerManager.swift b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerManager.swift new file mode 100644 index 00000000..04c69812 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerManager.swift @@ -0,0 +1,1571 @@ +import ExtensionFoundation +import Foundation +import Oliphaunt +import OliphauntBrokerProtocol +import OliphauntBrokerXPC +import XPC + +@available(iOS 26.0, macOS 26.0, *) +private final class IOSBrokerProcessHandle: @unchecked Sendable { + let value: AppExtensionProcess + + init(_ value: AppExtensionProcess) { + self.value = value + } + + func invalidate() { + value.invalidate() + } +} + +struct IOSBrokerInputBudget: Sendable { + enum ReservationState: Equatable, Sendable { + case queued + case active + } + + private struct Reservation: Sendable { + let bytes: Int + var state: ReservationState + } + + let maximumBytes: Int + private var reservations: [BrokerRequestID: Reservation] = [:] + private(set) var accountedBytes = 0 + + init( + maximumBytes: Int = OliphauntBrokerProtocol.maximumQueuedBytesPerDirection + ) { + precondition(maximumBytes >= 0) + self.maximumBytes = maximumBytes + } + + mutating func reserve(_ bytes: Int, for requestID: BrokerRequestID) -> Bool { + guard bytes >= 0, reservations[requestID] == nil else { + return false + } + let (newTotal, overflow) = accountedBytes.addingReportingOverflow(bytes) + guard !overflow, newTotal <= maximumBytes else { + return false + } + reservations[requestID] = Reservation(bytes: bytes, state: .queued) + accountedBytes = newTotal + return true + } + + mutating func activate(_ requestID: BrokerRequestID) { + guard var reservation = reservations[requestID] else { + preconditionFailure("activating an unaccounted broker request") + } + precondition(reservation.state == .queued, "broker request activated more than once") + reservation.state = .active + reservations[requestID] = reservation + } + + mutating func release(_ requestID: BrokerRequestID) { + guard let reservation = reservations.removeValue(forKey: requestID) else { + preconditionFailure("releasing an unaccounted broker request") + } + precondition(accountedBytes >= reservation.bytes) + accountedBytes -= reservation.bytes + } + + mutating func reset() { + reservations.removeAll(keepingCapacity: true) + accountedBytes = 0 + } + + func state(for requestID: BrokerRequestID) -> ReservationState? { + reservations[requestID]?.state + } +} + +struct IOSBrokerLaunchMetrics: Equatable, Sendable { + private(set) var attemptCount: UInt64 = 0 + private(set) var successfulCount: UInt64 = 0 + + mutating func recordProcessInitializationAttempt() { + attemptCount &+= 1 + } + + mutating func recordReadyValidatedLaunch() { + precondition( + successfulCount < attemptCount, + "a broker launch cannot succeed without a process initialization attempt" + ) + successfulCount &+= 1 + } +} + +struct IOSBrokerResumeRetryPolicy: Sendable { + private(set) var retryCount = 0 + + mutating func consumeRetry(for error: any Error) -> Bool { + guard retryCount == 0, + let brokerError = error as? BrokerError, + case .workerInterrupted = brokerError + else { + return false + } + retryCount = 1 + return true + } +} + +struct IOSBrokerLaunchAcquisitionRetryPolicy: Sendable { + private(set) var retryCount = 0 + + mutating func consumeRetry( + for error: any Error, + recoveringInterruptedEpoch: Bool + ) -> Bool { + guard recoveringInterruptedEpoch, + retryCount == 0, + Self.containsDeadProcessAssertion(error) + else { + return false + } + retryCount = 1 + return true + } + + private static func containsDeadProcessAssertion(_ error: any Error) -> Bool { + var current: NSError? = error as NSError + var visited: Set = [] + + while let currentError = current { + guard visited.insert(ObjectIdentifier(currentError)).inserted else { + return false + } + if currentError.domain == "RBSAssertionErrorDomain", + currentError.code == 2 + { + return true + } + current = currentError.userInfo[NSUnderlyingErrorKey] as? NSError + } + return false + } +} + +struct IOSBrokerInterruptedLaunchExpectation: Equatable, Sendable { + let staleEpoch: BrokerEpoch + let knownDeadProcessIdentifier: Int32? + + func validate( + recoveredEpoch: BrokerEpoch, + recoveredProcessIdentifier: Int32 + ) throws { + guard recoveredEpoch != staleEpoch else { + throw BrokerError.protocolViolation( + "worker recovery reused the interrupted worker epoch" + ) + } + if let knownDeadProcessIdentifier, + recoveredProcessIdentifier == knownDeadProcessIdentifier + { + throw BrokerError.protocolViolation( + "worker recovery reused the known-dead extension process" + ) + } + } +} + +struct IOSBrokerResumeRecoveryExpectation: Equatable, Sendable { + let staleEpoch: BrokerEpoch? + let rootManifestDigest: String? + + func validate( + recoveredEpoch: BrokerEpoch, + recoveredRootManifestDigest: String + ) throws { + if let staleEpoch, recoveredEpoch == staleEpoch { + throw BrokerError.protocolViolation( + "resume recovery reused the interrupted worker epoch" + ) + } + if let rootManifestDigest, + recoveredRootManifestDigest != rootManifestDigest + { + throw BrokerError.rootMismatch( + expected: rootManifestDigest, + actual: recoveredRootManifestDigest + ) + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +public actor IOSBrokerManager { + public static let shared = IOSBrokerManager() + + public private(set) var state: IOSBrokerManagerState = .idle + + private var process: IOSBrokerProcessHandle? + private var controlSession: XPCSession? + private var dataChannel: IOSBrokerDataChannel? + private var ready: BrokerReady? + private var currentLaunchID: UUID? + private var lastExtensionPID: Int32? + private var residentRootManifestDigest: String? + + private var residentIdentity: ResidentIdentity? + private var brokerConfiguration: IOSBrokerConfiguration? + private var handles: Set = [] + private var queue: [PendingOperation] = [] + private var inputBudget = IOSBrokerInputBudget() + private var inFlight: [BrokerRequestID: PendingOperation] = [:] + private var active: PendingOperation? + private var transactionOwner: UUID? + private var drainRunning = false + private var detachWhenIdle = false + private var admissionsPaused = false + private var requestIDs = try! BrokerRequestIDSequence() + private var launchMetrics = IOSBrokerLaunchMetrics() + private var interruptionCount: UInt64 = 0 + + private let controlQueue = DispatchQueue( + label: "dev.oliphaunt.ios-broker.control", + qos: .userInitiated + ) + + public init() {} + + deinit { + // XPCSession treats releasing an activated session without an explicit + // cancel as API misuse. Actor teardown can happen while unwinding a + // failed launch, before the normal detach path has run. + controlSession?.cancel(reason: "broker manager deinitialized") + process?.invalidate() + dataChannel?.close() + } + + public func open( + configuration proposedConfiguration: IOSBrokerConfiguration, + databaseConfiguration: OliphauntConfiguration + ) async throws -> IOSBrokerSession { + let configuration = try proposedConfiguration.validated() + let identity = try ResidentIdentity( + broker: configuration, + database: databaseConfiguration + ) + + if let residentIdentity, residentIdentity != identity { + throw BrokerError.invalidConfiguration( + "broker v1 already owns the canonical root with a different runtime configuration" + ) + } + if let brokerConfiguration, brokerConfiguration != configuration { + throw BrokerError.invalidConfiguration( + "the application-scoped broker manager already has different host settings" + ) + } + guard !admissionsPaused else { + throw BrokerError.rejected(.queueClosed) + } + + residentIdentity = identity + brokerConfiguration = configuration + let handleID = UUID() + handles.insert(handleID) + detachWhenIdle = false + + do { + _ = try await enqueue( + kind: .ensureReady, + handleID: handleID, + queuedBytes: 0, + deadline: nil + ) + return IOSBrokerSession( + manager: self, + handleID: handleID, + maximumRawResponseBytes: configuration.maximumRawResponseBytes + ) + } catch { + handles.remove(handleID) + if handles.isEmpty { + detachWhenIdle = true + await detachIfUnused() + } + throw error + } + } + + public func diagnostics() -> IOSBrokerDiagnostics { + IOSBrokerDiagnostics( + state: state, + epoch: ready?.epoch ?? state.epoch, + extensionProcessIdentifier: ready?.extensionPID ?? lastExtensionPID, + logicalHandleCount: handles.count, + queuedOperationCount: queue.count, + activeRequestID: active?.requestID, + launchAttemptCount: launchMetrics.attemptCount, + launchCount: launchMetrics.successfulCount, + interruptionCount: interruptionCount, + admissionsPaused: admissionsPaused + ) + } + + public func extensionProcessIdentifier() -> Int32? { + ready?.extensionPID ?? lastExtensionPID + } + + func capabilities(for handleID: UUID) -> OliphauntCapabilities { + guard handles.contains(handleID) else { + return IOSBrokerCapabilityMapping.initial + } + return ready.map { IOSBrokerCapabilityMapping.map($0.actualCapabilities) } + ?? IOSBrokerCapabilityMapping.initial + } + + func workerDiagnostics(handleID: UUID) async throws -> IOSBrokerWorkerDiagnostics { + try requireHandle(handleID) + if let connection = currentConnection(), + state == .ready(connection.ready.epoch) + || state == .quiescing(connection.ready.epoch) + { + // Diagnostics is deliberately an out-of-band control read. It does + // no database work and may sample the extension while the framed + // data channel is executing or backpressured. + let reply = try await sendControl( + IOSBrokerControlEnvelope( + kind: .diagnostics, + epoch: connection.ready.epoch + ), + expected: .diagnostics + ) + guard currentLaunchID == connection.launchID, + ready?.epoch == connection.ready.epoch + else { + throw BrokerError.workerInterrupted(epoch: connection.ready.epoch) + } + let diagnostics = IOSBrokerWorkerDiagnostics( + wire: try IOSBrokerXPC.decodeWorkerDiagnostics(reply) + ) + guard diagnostics.epoch == connection.ready.epoch else { + throw BrokerError.workerInterrupted(epoch: connection.ready.epoch) + } + return diagnostics + } + let operation = try await enqueue( + kind: .diagnostics, + handleID: handleID, + queuedBytes: 0, + deadline: brokerConfiguration?.requestDeadline + ) + guard let diagnostics = operation.workerDiagnostics else { + throw BrokerError.protocolViolation("worker omitted diagnostics result") + } + return diagnostics + } + + func execute( + handleID: UUID, + bytes: Data, + onChunk: @escaping @Sendable (Data) throws -> Void + ) async throws { + try requireHandle(handleID) + guard !admissionsPaused else { + throw BrokerError.rejected(.queueClosed) + } + guard let configuration = brokerConfiguration else { + throw BrokerError.brokerUnavailable + } + guard bytes.count <= configuration.maximumRequestBytes else { + throw BrokerProtocolError.payloadTooLarge( + actual: UInt64(bytes.count), + maximum: configuration.maximumRequestBytes + ) + } + _ = try await enqueue( + kind: .data(bytes, onChunk), + handleID: handleID, + queuedBytes: bytes.count, + deadline: configuration.requestDeadline + ) + } + + func checkpoint(handleID: UUID) async throws { + try requireHandle(handleID) + guard !admissionsPaused else { + throw BrokerError.rejected(.queueClosed) + } + _ = try await enqueue( + kind: .checkpoint, + handleID: handleID, + queuedBytes: 0, + deadline: brokerConfiguration?.requestDeadline + ) + } + + func prepareForBackground( + handleID: UUID, + deadline: Date + ) async throws -> OliphauntBackgroundPreparationResult { + try requireHandle(handleID) + guard deadline > Date() else { + throw BrokerError.deadlineExceeded + } + admissionsPaused = true + if case .ready(let epoch) = state { + state = .quiescing(epoch) + } + + let queuedData = queue.filter { $0.isData } + for operation in queuedData { + finish(operation, throwing: BrokerError.canceled) + } + + let cancelledActiveWork = active?.isData == true + if cancelledActiveWork { + let operation = active + Task { [weak self, weak operation] in + guard let self, let operation else { return } + try? await self.sendCancellation(for: operation) + } + } + + let duration = Duration.seconds(max(0, deadline.timeIntervalSinceNow)) + let operation = try await enqueue( + kind: .prepareForBackground(deadline), + handleID: handleID, + queuedBytes: 0, + deadline: duration + ) + let workerResult = + operation.backgroundResult + ?? OliphauntBackgroundPreparationResult( + cancelledActiveWork: false, + checkpointed: true + ) + return Self.mergeBackgroundPreparationResult( + hostCancelledActiveWork: cancelledActiveWork, + workerResult: workerResult + ) + } + + static func mergeBackgroundPreparationResult( + hostCancelledActiveWork: Bool, + workerResult: OliphauntBackgroundPreparationResult + ) -> OliphauntBackgroundPreparationResult { + OliphauntBackgroundPreparationResult( + cancelledActiveWork: + hostCancelledActiveWork || workerResult.cancelledActiveWork, + checkpointed: workerResult.checkpointed, + skippedCheckpointReason: workerResult.skippedCheckpointReason + ) + } + + func prepareForBackground( + handleID: UUID, + timeout: Duration + ) async throws -> OliphauntBackgroundPreparationResult { + try await prepareForBackground( + handleID: handleID, + deadline: Date().addingTimeInterval(timeout.timeInterval) + ) + } + + func resumeFromBackground(handleID: UUID) async throws { + try requireHandle(handleID) + let recoveryExpectation = IOSBrokerResumeRecoveryExpectation( + staleEpoch: ready?.epoch ?? state.epoch, + rootManifestDigest: residentRootManifestDigest + ) + let deadline = brokerConfiguration?.requestDeadline + + if currentConnection() == nil, state.epoch != nil { + _ = try await enqueue( + kind: .resumeFromBackground(recovering: recoveryExpectation), + handleID: handleID, + queuedBytes: 0, + deadline: deadline + ) + } else { + var retryPolicy = IOSBrokerResumeRetryPolicy() + do { + _ = try await enqueue( + kind: .resumeFromBackground(recovering: nil), + handleID: handleID, + queuedBytes: 0, + deadline: deadline + ) + } catch { + guard retryPolicy.consumeRetry(for: error) else { + throw error + } + _ = try await enqueue( + kind: .resumeFromBackground(recovering: recoveryExpectation), + handleID: handleID, + queuedBytes: 0, + deadline: deadline + ) + } + } + admissionsPaused = false + if let epoch = ready?.epoch { + state = .ready(epoch) + } + } + + func cancel(handleID: UUID) async throws { + try requireHandle(handleID) + + let queuedForHandle = queue.filter { $0.handleID == handleID && $0.isData } + for operation in queuedForHandle { + finish(operation, throwing: BrokerError.canceled) + } + + guard let active, active.handleID == handleID, active.isData else { + return + } + do { + try await sendCancellation(for: active) + } catch { + interruptCurrentLaunch(reason: "cancellation control failed: \(error)") + throw error + } + } + + #if DEBUG + func injectFault(handleID: UUID, fault: BrokerWorkerFault) async throws { + try requireHandle(handleID) + _ = try await enqueue( + kind: .injectFault(fault), + handleID: handleID, + queuedBytes: 0, + deadline: brokerConfiguration?.requestDeadline + ) + } + #endif + + func close(handleID: UUID) async throws { + guard handles.remove(handleID) != nil else { + return + } + + let queuedForHandle = queue.filter { $0.handleID == handleID } + for operation in queuedForHandle { + finish(operation, throwing: BrokerError.canceled) + } + if let active, active.handleID == handleID, active.isData { + try? await sendCancellation(for: active) + } + if transactionOwner == handleID { + transactionOwner = nil + interruptCurrentLaunch( + reason: "logical handle detached while it owned a physical transaction" + ) + } + + guard handles.isEmpty else { + return + } + detachWhenIdle = true + admissionsPaused = false + await detachIfUnused() + } + + private func enqueue( + kind: PendingOperation.Kind, + handleID: UUID, + queuedBytes: Int, + deadline: Duration? + ) async throws -> PendingOperation { + try requireHandle(handleID) + let requestID = try requestIDs.next() + guard inputBudget.reserve(queuedBytes, for: requestID) else { + throw BrokerError.rejected(.queueClosed) + } + let operation = try await withCheckedThrowingContinuation { + (continuation: CheckedContinuation) in + let operation = PendingOperation( + requestID: requestID, + handleID: handleID, + kind: kind, + queuedBytes: queuedBytes, + continuation: continuation + ) + queue.append(operation) + inFlight[requestID] = operation + if let deadline { + operation.deadlineTask = Task { [weak self, weak operation] in + do { + try await Task.sleep(for: deadline) + } catch { + return + } + guard let self, let operation else { + return + } + await self.deadlineExpired(operation) + } + } + scheduleDrain() + } + return operation + } + + private func scheduleDrain() { + guard !drainRunning else { + return + } + drainRunning = true + Task { await drain() } + } + + private func drain() async { + while true { + guard active == nil else { + drainRunning = false + return + } + while let first = queue.first, first.terminal { + removeFromQueue(first) + } + guard let operation = nextQueuedOperation() else { + drainRunning = false + await detachIfUnused() + return + } + + removeFromQueue(operation) + inputBudget.activate(operation.requestID) + active = operation + + do { + let connection = try await ensureReady() + guard !operation.terminal else { + active = nil + continue + } + operation.epoch = connection.ready.epoch + try await perform(operation, connection: connection) + finish(operation) + } catch { + if !operation.terminal { + handleOperationFailure(operation, error: error) + } + } + if active === operation { + active = nil + } + } + } + + private func perform( + _ operation: PendingOperation, + connection: ActiveConnection + ) async throws { + switch operation.kind { + case .ensureReady: + return + case .data(let bytes, let onChunk): + operation.bytesMayHaveReachedWorker = true + let transactionStatus = try await connection.channel.execute( + requestID: operation.requestID, + epoch: connection.ready.epoch, + protocolVersion: connection.ready.selectedProtocolVersion, + bytes: bytes, + maximumRequestBytes: brokerConfiguration?.maximumRequestBytes + ?? OliphauntBrokerProtocol.defaultMaximumRequestBytes, + onChunk: onChunk + ) + switch transactionStatus { + case .idle: + if transactionOwner == operation.handleID { + transactionOwner = nil + } + case .inTransaction, .failedTransaction: + transactionOwner = operation.handleID + } + case .checkpoint: + _ = try await sendControl( + IOSBrokerControlEnvelope( + kind: .checkpoint, + epoch: connection.ready.epoch, + requestID: operation.requestID + ), + expected: .checkpoint + ) + case .diagnostics: + let reply = try await sendControl( + IOSBrokerControlEnvelope( + kind: .diagnostics, + epoch: connection.ready.epoch, + requestID: operation.requestID + ), + expected: .diagnostics + ) + operation.workerDiagnostics = IOSBrokerWorkerDiagnostics( + wire: try IOSBrokerXPC.decodeWorkerDiagnostics(reply) + ) + #if DEBUG + case .injectFault(let fault): + _ = try await sendControl( + IOSBrokerControlEnvelope( + kind: .injectFault, + epoch: connection.ready.epoch, + requestID: operation.requestID, + fault: fault + ), + expected: .injectFault + ) + #endif + case .prepareForBackground(let deadline): + let reply = try await sendControl( + IOSBrokerControlEnvelope( + kind: .prepareForBackground, + epoch: connection.ready.epoch, + requestID: operation.requestID, + deadlineUnixNanoseconds: deadline.unixNanoseconds + ), + expected: .prepareForBackground, + timeout: Duration.seconds(max(0.001, deadline.timeIntervalSinceNow)) + ) + let cancelled: Bool = reply[IOSBrokerXPC.cancelledActiveWorkKey] ?? false + let checkpointed: Bool = reply[IOSBrokerXPC.checkpointedKey] ?? true + operation.backgroundResult = OliphauntBackgroundPreparationResult( + cancelledActiveWork: cancelled, + checkpointed: checkpointed, + skippedCheckpointReason: checkpointed ? nil : .activeWork + ) + transactionOwner = nil + state = .quiescing(connection.ready.epoch) + case .resumeFromBackground(let recoveryExpectation): + if let recoveryExpectation { + try recoveryExpectation.validate( + recoveredEpoch: connection.ready.epoch, + recoveredRootManifestDigest: connection.ready.rootManifestDigest + ) + } else { + _ = try await sendControl( + IOSBrokerControlEnvelope( + kind: .resumeFromBackground, + epoch: connection.ready.epoch, + requestID: operation.requestID + ), + expected: .resumeFromBackground + ) + } + try await connection.channel.healthCheck( + epoch: connection.ready.epoch, + protocolVersion: connection.ready.selectedProtocolVersion + ) + state = .ready(connection.ready.epoch) + } + } + + private func handleOperationFailure(_ operation: PendingOperation, error: any Error) { + if error is IOSBrokerTransportFailure { + let interruptedEpoch = operation.epoch ?? ready?.epoch ?? state.epoch + interruptCurrentLaunch(reason: String(describing: error)) + if !operation.terminal { + finish( + operation, + throwing: BrokerError.workerInterrupted(epoch: interruptedEpoch) + ) + } + return + } + if let failure = error as? IOSBrokerDataPlaneFailure { + switch failure { + case .rejected(let reason): + if reason == .canceled { + finish(operation, throwing: BrokerError.canceled) + } else { + finish(operation, throwing: BrokerError.rejected(reason)) + } + case .outcomeUnknown, .protocolViolation: + let epoch = operation.epoch ?? ready?.epoch + interruptCurrentLaunch(reason: String(describing: failure)) + if !operation.terminal, let epoch { + finish( + operation, + throwing: BrokerError.outcomeUnknown( + epoch: epoch, + requestID: operation.requestID + ) + ) + } + } + return + } + if let brokerError = error as? BrokerError { + switch brokerError { + case .workerInterrupted, .protocolViolation: + // A timed-out or malformed control exchange leaves the epoch's + // synchronization unprovable. Atomically tear it down so the + // next demand must establish a fresh process/session/epoch. + interruptCurrentLaunch(reason: String(describing: brokerError)) + if !operation.terminal { + finish(operation, throwing: brokerError) + } + return + default: + break + } + } + finish(operation, throwing: error) + } + + private func ensureReady() async throws -> ActiveConnection { + try await ensureReady( + launchAcquisitionRetryPolicy: IOSBrokerLaunchAcquisitionRetryPolicy(), + knownDeadProcessIdentifier: nil + ) + } + + private func ensureReady( + launchAcquisitionRetryPolicy: IOSBrokerLaunchAcquisitionRetryPolicy, + knownDeadProcessIdentifier: Int32? + ) async throws -> ActiveConnection { + if let connection = currentConnection(), + state == .ready(connection.ready.epoch) || state == .quiescing(connection.ready.epoch) + { + return connection + } + guard let configuration = brokerConfiguration, + let residentIdentity + else { + throw BrokerError.brokerUnavailable + } + if state == .unavailable { + throw BrokerError.extensionMissing + } + + let oldEpoch = state.epoch + state = oldEpoch == nil ? .launching : .recovering + let launchID = UUID() + currentLaunchID = launchID + var processAcquired = false + var retryPolicy = launchAcquisitionRetryPolicy + + do { + let identity = try await IOSBrokerExtensionDiscovery.discover( + bundleIdentifier: configuration.extensionBundleIdentifier + ) + guard currentLaunchID == launchID else { + throw BrokerError.workerInterrupted(epoch: oldEpoch) + } + + let interruptionRelay = IOSBrokerInterruptionRelay() + interruptionRelay.install { [weak self] in + Task { await self?.extensionInterrupted(launchID: launchID) } + } + launchMetrics.recordProcessInitializationAttempt() + let processConfiguration = AppExtensionProcess.Configuration( + appExtensionIdentity: identity, + onInterruption: { interruptionRelay.signal() } + ) + let launchedProcess = try await AppExtensionProcess( + configuration: processConfiguration + ) + processAcquired = true + guard currentLaunchID == launchID else { + launchedProcess.invalidate() + throw BrokerError.workerInterrupted(epoch: oldEpoch) + } + + let session = try launchedProcess.makeXPCSession() + session.setTargetQueue(controlQueue) + session.setIncomingMessageHandler { _ in + try? IOSBrokerXPC.makeRejected( + .invalidRequest("host does not accept unsolicited control requests") + ) + } + session.setCancellationHandler { _ in + interruptionRelay.signal() + } + try session.activate() + + let sockets = try IOSBrokerSocketPair.make() + process = IOSBrokerProcessHandle(launchedProcess) + controlSession = session + dataChannel = sockets.host + state = .binding + + let helloMessage = try IOSBrokerXPC.makeHello( + configuration.hello, + dataChannel: sockets.extensionEndpoint + ) + let reply: XPCDictionary + do { + reply = try await request( + session: session, + message: helloMessage, + timeout: configuration.controlReplyTimeout, + epoch: oldEpoch + ).dictionary + } catch { + sockets.extensionEndpoint.close() + throw error + } + // xpc_fd_create owns its duplicate. The sender's original is closed + // only once a worker reply proves the transfer completed. + sockets.extensionEndpoint.close() + + let workerReady = try IOSBrokerXPC.decodeReady(reply) + try validateReady( + workerReady, + hello: configuration.hello, + residentIdentity: residentIdentity + ) + if let oldEpoch { + try IOSBrokerInterruptedLaunchExpectation( + staleEpoch: oldEpoch, + knownDeadProcessIdentifier: knownDeadProcessIdentifier + ).validate( + recoveredEpoch: workerReady.epoch, + recoveredProcessIdentifier: workerReady.extensionPID + ) + } + if let residentRootManifestDigest, + workerReady.rootManifestDigest != residentRootManifestDigest + { + throw BrokerError.rootMismatch( + expected: residentRootManifestDigest, + actual: workerReady.rootManifestDigest + ) + } + guard currentLaunchID == launchID else { + throw BrokerError.workerInterrupted(epoch: workerReady.epoch) + } + ready = workerReady + transactionOwner = nil + lastExtensionPID = workerReady.extensionPID + launchMetrics.recordReadyValidatedLaunch() + + try await sockets.host.healthCheck( + epoch: workerReady.epoch, + protocolVersion: workerReady.selectedProtocolVersion + ) + guard currentLaunchID == launchID else { + throw BrokerError.workerInterrupted(epoch: workerReady.epoch) + } + if residentRootManifestDigest == nil { + residentRootManifestDigest = workerReady.rootManifestDigest + } + state = .ready(workerReady.epoch) + return ActiveConnection( + session: session, + channel: sockets.host, + ready: workerReady, + launchID: launchID + ) + } catch let launchError { + if !processAcquired, + let oldEpoch, + currentLaunchID == launchID, + retryPolicy.consumeRetry( + for: launchError, + recoveringInterruptedEpoch: true + ) + { + let deadProcessIdentifier = lastExtensionPID + do { + try await Task.sleep(for: .milliseconds(500)) + } catch { + if currentLaunchID == launchID { + tearDownConnection(reason: "launch retry cancelled") + state = .interrupted(oldEpoch) + } + throw launchError + } + guard currentLaunchID == launchID else { + throw BrokerError.workerInterrupted(epoch: oldEpoch) + } + tearDownConnection( + reason: "retrying stale ExtensionKit process assertion" + ) + state = .interrupted(oldEpoch) + return try await ensureReady( + launchAcquisitionRetryPolicy: retryPolicy, + knownDeadProcessIdentifier: deadProcessIdentifier + ) + } + if currentLaunchID == launchID { + let missing = (launchError as? BrokerError) == .extensionMissing + tearDownConnection(reason: "launch failed") + if missing { + state = .unavailable + } else if let oldEpoch { + state = .interrupted(oldEpoch) + } else { + state = .idle + } + } + throw launchError + } + } + + private func currentConnection() -> ActiveConnection? { + guard let session = controlSession, + let channel = dataChannel, + let ready, + let launchID = currentLaunchID + else { + return nil + } + return ActiveConnection( + session: session, + channel: channel, + ready: ready, + launchID: launchID + ) + } + + private func validateReady( + _ ready: BrokerReady, + hello: BrokerHello, + residentIdentity: ResidentIdentity + ) throws { + guard ready.selectedProtocolVersion >= hello.minimumProtocolVersion, + ready.selectedProtocolVersion <= hello.maximumProtocolVersion, + OliphauntBrokerProtocol.supports(version: ready.selectedProtocolVersion) + else { + throw BrokerError.incompatibleProtocol( + minimum: hello.minimumProtocolVersion, + maximum: hello.maximumProtocolVersion + ) + } + guard ready.abiVersion == hello.expectedABI else { + throw BrokerError.incompatibleABI( + expected: hello.expectedABI, + actual: ready.abiVersion + ) + } + if let expected = hello.expectedRuntimeVersion, expected != ready.runtimeVersion { + throw BrokerError.runtimeMismatch(expected: expected, actual: ready.runtimeVersion) + } + guard ready.actualRuntimeConfiguration.rootID == hello.rootID else { + throw BrokerError.rootMismatch( + expected: hello.rootID, + actual: ready.actualRuntimeConfiguration.rootID + ) + } + guard + ready.actualRuntimeConfiguration.startupConfigurationDigest + == hello.startupConfigurationDigest + else { + throw BrokerError.invalidConfiguration( + "worker startup-configuration digest does not match the host" + ) + } + guard + ready.actualRuntimeConfiguration.selectedExtensions.sorted() + == residentIdentity.extensions + else { + throw BrokerError.invalidConfiguration("worker extension set does not match the host") + } + let missing = hello.requestedCapabilities.subtracting( + ready.actualCapabilities.enabled + ) + if let capability = missing.sorted(by: { $0.rawValue < $1.rawValue }).first { + throw BrokerError.rejected(.unsupportedCapability(capability)) + } + guard !ready.actualCapabilities.rootSwitchable, + !ready.actualCapabilities.multiRoot, + !ready.actualCapabilities.independentSessions, + ready.actualCapabilities.maxClientSessions == 1, + !ready.actualCapabilities.backupRestore, + !ready.actualCapabilities.serverMode, + ready.actualCapabilities.connectionString == nil + else { + throw BrokerError.protocolViolation( + "worker advertised capabilities that violate iOS broker v1 invariants" + ) + } + } + + private func sendCancellation(for operation: PendingOperation?) async throws { + guard let operation, + let epoch = operation.epoch, + !operation.terminal + else { + return + } + let reply = try await sendControl( + IOSBrokerControlEnvelope( + kind: .cancel, + epoch: epoch, + requestID: operation.requestID + ), + expected: .cancelObserved + ) + let kind = try IOSBrokerXPC.messageKind(in: reply) + guard kind == .cancelObserved || kind == .cancel else { + throw BrokerError.protocolViolation("invalid cancellation acknowledgement") + } + } + + private func sendControl( + _ envelope: IOSBrokerControlEnvelope, + expected: BrokerControlMessageKind, + timeout: Duration? = nil + ) async throws -> XPCDictionary { + guard let session = controlSession else { + throw BrokerError.workerInterrupted(epoch: envelope.epoch) + } + let reply = try await request( + session: session, + message: IOSBrokerXPC.makeControl(envelope), + timeout: timeout ?? brokerConfiguration?.controlReplyTimeout ?? .seconds(15), + epoch: envelope.epoch + ).dictionary + let kind = try IOSBrokerXPC.messageKind(in: reply) + if kind == .rejected { + throw try IOSBrokerXPC.decodeError(reply) + } + if expected == .cancelObserved { + guard kind == .cancelObserved || kind == .cancel else { + throw BrokerError.protocolViolation( + "expected CancelObserved, received \(kind.rawValue)" + ) + } + } else if kind != expected { + throw BrokerError.protocolViolation( + "expected \(expected.rawValue), received \(kind.rawValue)" + ) + } + let succeeded: Bool = reply[IOSBrokerXPC.successKey] ?? true + guard succeeded else { + let reason: String = reply[BrokerControlKey.reason] ?? "control request failed" + throw BrokerError.rejected(.invalidRequest(reason)) + } + return reply + } + + private func request( + session: XPCSession, + message: XPCDictionary, + timeout: Duration, + epoch: BrokerEpoch? + ) async throws -> IOSBrokerXPCReply { + try await withCheckedThrowingContinuation { + (continuation: CheckedContinuation) in + let gate = XPCReplyGate(continuation: continuation) + controlQueue.asyncAfter(deadline: .now() + timeout.timeInterval) { + gate.fail(BrokerError.workerInterrupted(epoch: epoch)) + } + session.send(message: message) { result in + switch result { + case .success(let reply): + gate.succeed(reply) + case .failure: + gate.fail(BrokerError.workerInterrupted(epoch: epoch)) + } + } + } + } + + private func deadlineExpired(_ operation: PendingOperation) async { + guard inFlight[operation.requestID] === operation, + !operation.terminal + else { + return + } + guard active === operation else { + if operation.isBackgroundPreparation { + finish(operation, throwing: BrokerError.deadlineExceeded) + interruptCurrentLaunch( + reason: "background preparation could not quiesce before its deadline" + ) + return + } + finish(operation, throwing: BrokerError.deadlineExceeded) + return + } + guard operation.isData else { + if !operation.terminal { + finish(operation, throwing: BrokerError.deadlineExceeded) + } + interruptCurrentLaunch(reason: "lifecycle control exceeded its deadline") + return + } + + operation.deadlineCancellationRequested = true + Task { [weak self, weak operation] in + guard let self, let operation else { return } + try? await self.sendCancellation(for: operation) + } + let grace = brokerConfiguration?.cancellationGracePeriod ?? .seconds(2) + operation.cancellationGraceTask = Task { [weak self, weak operation] in + do { + try await Task.sleep(for: grace) + } catch { + return + } + guard let self, let operation else { return } + await self.cancellationGraceExpired(operation) + } + } + + private func cancellationGraceExpired(_ operation: PendingOperation) { + guard active === operation, !operation.terminal else { + return + } + interruptCurrentLaunch(reason: "request did not terminate during cancellation grace") + } + + private func extensionInterrupted(launchID: UUID) { + guard currentLaunchID == launchID else { + return + } + interruptCurrentLaunch(reason: "ExtensionFoundation interruption") + } + + private func interruptCurrentLaunch(reason: String) { + guard currentLaunchID != nil || ready != nil || process != nil else { + return + } + interruptionCount &+= 1 + let interruptedEpoch = ready?.epoch ?? state.epoch + transactionOwner = nil + tearDownConnection(reason: reason) + if let interruptedEpoch { + state = .interrupted(interruptedEpoch) + } else { + state = .idle + } + + let operations = Array(inFlight.values) + for operation in operations where !operation.terminal { + if operation.isData, + operation.bytesMayHaveReachedWorker, + let epoch = operation.epoch ?? interruptedEpoch + { + finish( + operation, + throwing: BrokerError.outcomeUnknown( + epoch: epoch, + requestID: operation.requestID + ) + ) + } else if operation.isData { + finish(operation, throwing: BrokerError.notStarted) + } else { + finish( + operation, + throwing: BrokerError.workerInterrupted(epoch: interruptedEpoch) + ) + } + } + queue.removeAll() + inputBudget.reset() + // Recovery is intentionally lazy. A later open, query, checkpoint, or + // resume operation drives interrupted -> recovering -> ready(new epoch). + } + + private func tearDownConnection(reason: String) { + let oldSession = controlSession + let oldProcess = process + currentLaunchID = nil + ready = nil + transactionOwner = nil + dataChannel?.close() + dataChannel = nil + controlSession = nil + process = nil + oldSession?.cancel(reason: reason) + oldProcess?.invalidate() + } + + private func detachIfUnused() async { + guard detachWhenIdle, + handles.isEmpty, + active == nil, + queue.isEmpty + else { + return + } + detachWhenIdle = false + guard let epoch = ready?.epoch, + let launchID = currentLaunchID + else { + if state == .unavailable { + state = .idle + } + return + } + state = .closing + _ = try? await sendControl( + IOSBrokerControlEnvelope(kind: .detach, epoch: epoch), + expected: .detach, + timeout: min( + brokerConfiguration?.cancellationGracePeriod ?? .seconds(2), + .seconds(2) + ) + ) + guard currentLaunchID == launchID else { + return + } + tearDownConnection(reason: "last logical broker handle detached") + state = .idle + } + + private func finish(_ operation: PendingOperation, throwing error: (any Error)? = nil) { + guard !operation.terminal else { + return + } + operation.terminal = true + operation.deadlineTask?.cancel() + operation.cancellationGraceTask?.cancel() + removeFromQueue(operation) + inputBudget.release(operation.requestID) + inFlight.removeValue(forKey: operation.requestID) + if let error { + operation.continuation.resume(throwing: error) + } else { + operation.continuation.resume(returning: operation) + } + } + + private func removeFromQueue(_ operation: PendingOperation) { + queue.removeAll { $0 === operation } + } + + private func nextQueuedOperation() -> PendingOperation? { + guard let transactionOwner else { + return queue.first + } + // A physical PostgreSQL transaction belongs to the logical handle that + // observed ReadyForQuery(T/E). Other callers retain FIFO position until + // that handle observes ReadyForQuery(I). Lifecycle operations may pass + // the pin because they do not execute caller SQL and background prepare + // is responsible for rolling an abandoned transaction back. + return queue.first { + $0.handleID == transactionOwner || $0.mayBypassTransactionPin + } + } + + private func requireHandle(_ handleID: UUID) throws { + guard handles.contains(handleID) else { + throw BrokerError.databaseClosed + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +extension IOSBrokerManager { + fileprivate struct ActiveConnection { + let session: XPCSession + let channel: IOSBrokerDataChannel + let ready: BrokerReady + let launchID: UUID + } + + struct ResidentIdentity: Equatable, Sendable { + let expectedABI: UInt32 + let expectedRuntimeVersion: String? + let startupConfigurationDigest: String + let extensionBundleIdentifier: String? + let durability: OliphauntDurability + let runtimeFootprint: OliphauntRuntimeFootprintProfile + let startupGUCs: [OliphauntStartupGUC] + let username: String? + let database: String? + let extensions: [String] + + init( + broker: IOSBrokerConfiguration, + database configuration: OliphauntConfiguration + ) throws { + guard configuration.mode == .nativeBroker else { + throw OliphauntError.runtimeUnavailable(configuration.mode) + } + if let root = configuration.root { + throw BrokerError.rootMismatch( + expected: OliphauntBrokerProtocol.canonicalRootID, + actual: root.absoluteString + ) + } + guard configuration.runtimeFootprint == .smallMobile else { + throw BrokerError.invalidConfiguration( + "iOS broker v1 requires the smallMobile runtime-footprint profile" + ) + } + guard configuration.durability == .safe else { + throw BrokerError.invalidConfiguration( + "iOS broker v1 requires safe durability" + ) + } + guard configuration.startupGUCs.isEmpty else { + throw BrokerError.invalidConfiguration( + "iOS broker v1 does not support custom startup GUCs" + ) + } + guard configuration.username == nil else { + throw BrokerError.invalidConfiguration( + "iOS broker v1 does not accept a caller-provided PostgreSQL username" + ) + } + guard configuration.database == nil || configuration.database == "postgres" else { + throw BrokerError.invalidConfiguration( + "iOS broker v1 requires PostgreSQL database postgres" + ) + } + expectedABI = broker.expectedABI + expectedRuntimeVersion = broker.expectedRuntimeVersion + startupConfigurationDigest = broker.startupConfigurationDigest + extensionBundleIdentifier = broker.extensionBundleIdentifier + durability = configuration.durability + runtimeFootprint = configuration.runtimeFootprint + startupGUCs = configuration.startupGUCs + username = configuration.username + database = configuration.database + extensions = configuration.extensions.sorted() + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +private final class PendingOperation: @unchecked Sendable { + enum Kind: Sendable { + case ensureReady + case data(Data, @Sendable (Data) throws -> Void) + case checkpoint + case diagnostics + #if DEBUG + case injectFault(BrokerWorkerFault) + #endif + case prepareForBackground(Date) + case resumeFromBackground(recovering: IOSBrokerResumeRecoveryExpectation?) + } + + let requestID: BrokerRequestID + let handleID: UUID + let kind: Kind + let queuedBytes: Int + let continuation: CheckedContinuation + var epoch: BrokerEpoch? + var bytesMayHaveReachedWorker = false + var deadlineCancellationRequested = false + var terminal = false + var deadlineTask: Task? + var cancellationGraceTask: Task? + var backgroundResult: OliphauntBackgroundPreparationResult? + var workerDiagnostics: IOSBrokerWorkerDiagnostics? + + init( + requestID: BrokerRequestID, + handleID: UUID, + kind: Kind, + queuedBytes: Int, + continuation: CheckedContinuation + ) { + self.requestID = requestID + self.handleID = handleID + self.kind = kind + self.queuedBytes = queuedBytes + self.continuation = continuation + } + + var isData: Bool { + if case .data = kind { return true } + return false + } + + var isBackgroundPreparation: Bool { + if case .prepareForBackground = kind { return true } + return false + } + + var mayBypassTransactionPin: Bool { + switch kind { + case .ensureReady, .diagnostics, .prepareForBackground, .resumeFromBackground: + true + #if DEBUG + case .injectFault: + true + #endif + case .data, .checkpoint: + false + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +private final class IOSBrokerInterruptionRelay: @unchecked Sendable { + private let lock = NSLock() + private var handler: (@Sendable () -> Void)? + private var signalled = false + + func install(_ handler: @escaping @Sendable () -> Void) { + lock.lock() + self.handler = handler + let shouldSignal = signalled + lock.unlock() + if shouldSignal { + handler() + } + } + + func signal() { + lock.lock() + signalled = true + let handler = handler + lock.unlock() + handler?() + } +} + +@available(iOS 26.0, macOS 26.0, *) +private final class XPCReplyGate: @unchecked Sendable { + private let lock = NSLock() + private var finished = false + private let continuation: CheckedContinuation + + init(continuation: CheckedContinuation) { + self.continuation = continuation + } + + func succeed(_ dictionary: XPCDictionary) { + guard claim() else { return } + continuation.resume(returning: IOSBrokerXPCReply(dictionary)) + } + + func fail(_ error: BrokerError) { + guard claim() else { return } + continuation.resume(throwing: error) + } + + private func claim() -> Bool { + lock.lock() + guard !finished else { + lock.unlock() + return false + } + finished = true + lock.unlock() + return true + } +} + +@available(iOS 26.0, macOS 26.0, *) +private final class IOSBrokerXPCReply: @unchecked Sendable { + let dictionary: XPCDictionary + + init(_ dictionary: XPCDictionary) { + self.dictionary = dictionary + } +} + +extension Duration { + fileprivate var timeInterval: TimeInterval { + let components = self.components + return TimeInterval(components.seconds) + TimeInterval(components.attoseconds) + / 1_000_000_000_000_000_000 + } +} + +extension Date { + fileprivate var unixNanoseconds: UInt64 { + let value = timeIntervalSince1970 * 1_000_000_000 + guard value.isFinite, value > 0 else { + return 0 + } + return UInt64(min(value, Double(UInt64.max))) + } +} diff --git a/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerPublicAPI.swift b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerPublicAPI.swift new file mode 100644 index 00000000..fd535f04 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerPublicAPI.swift @@ -0,0 +1,414 @@ +import ExtensionFoundation +import Foundation +import Oliphaunt +import OliphauntBrokerProtocol +import OliphauntBrokerXPC + +/// Host-side settings that must agree with the app-extension worker. +/// +/// No filesystem URL is part of this value. Broker v1 always opens the logical +/// root named `default`; the extension alone resolves that name to PGDATA. +public struct IOSBrokerConfiguration: Equatable, Sendable { + public var expectedABI: UInt32 + public var expectedRuntimeVersion: String? + public var startupConfigurationDigest: String + public var requestedCapabilities: Set + public var maximumRequestBytes: Int + public var maximumRawResponseBytes: Int + public var requestDeadline: Duration? + public var extensionBundleIdentifier: String? + public var controlReplyTimeout: Duration + public var cancellationGracePeriod: Duration + + public init( + expectedABI: UInt32, + expectedRuntimeVersion: String? = nil, + startupConfigurationDigest: String, + requestedCapabilities: Set = [ + .processIsolated, + .crashRestartable, + .sameRootLogicalReopen, + .protocolRaw, + .protocolStream, + .queryCancel, + ], + maximumRequestBytes: Int = OliphauntBrokerProtocol.defaultMaximumRequestBytes, + maximumRawResponseBytes: Int = + OliphauntBrokerProtocol.maximumQueuedBytesPerDirection, + requestDeadline: Duration? = .seconds(30), + extensionBundleIdentifier: String? = nil, + controlReplyTimeout: Duration = .seconds(15), + cancellationGracePeriod: Duration = .seconds(2) + ) { + self.expectedABI = expectedABI + self.expectedRuntimeVersion = expectedRuntimeVersion + self.startupConfigurationDigest = startupConfigurationDigest + self.requestedCapabilities = requestedCapabilities + self.maximumRequestBytes = maximumRequestBytes + self.maximumRawResponseBytes = maximumRawResponseBytes + self.requestDeadline = requestDeadline + self.extensionBundleIdentifier = extensionBundleIdentifier + self.controlReplyTimeout = controlReplyTimeout + self.cancellationGracePeriod = cancellationGracePeriod + } + + func validated() throws -> IOSBrokerConfiguration { + guard !startupConfigurationDigest.isEmpty else { + throw BrokerError.invalidConfiguration("startup-configuration digest is empty") + } + guard maximumRequestBytes >= 5 else { + throw BrokerError.invalidConfiguration("maximum request size must be at least 5 bytes") + } + guard maximumRequestBytes <= OliphauntBrokerProtocol.maximumQueuedBytesPerDirection else { + throw BrokerError.invalidConfiguration( + "maximum request size exceeds the bounded host queue" + ) + } + guard maximumRawResponseBytes > 0 else { + throw BrokerError.invalidConfiguration("maximum raw response size must be positive") + } + guard + maximumRawResponseBytes + <= OliphauntBrokerProtocol.maximumQueuedBytesPerDirection + else { + throw BrokerError.invalidConfiguration( + "maximum raw response size exceeds the bounded host response collector" + ) + } + if let requestDeadline, requestDeadline <= .zero { + throw BrokerError.invalidConfiguration("request deadline must be positive") + } + guard controlReplyTimeout > .zero else { + throw BrokerError.invalidConfiguration("control reply timeout must be positive") + } + guard cancellationGracePeriod > .zero else { + throw BrokerError.invalidConfiguration("cancellation grace period must be positive") + } + if let extensionBundleIdentifier, extensionBundleIdentifier.isEmpty { + throw BrokerError.invalidConfiguration("extension bundle identifier is empty") + } + return self + } + + var hello: BrokerHello { + BrokerHello( + expectedABI: expectedABI, + expectedRuntimeVersion: expectedRuntimeVersion, + rootID: OliphauntBrokerProtocol.canonicalRootID, + startupConfigurationDigest: startupConfigurationDigest, + requestedCapabilities: requestedCapabilities + ) + } +} + +/// Runtime values useful to the simulator feasibility harness. The PID is +/// reported by the worker in `Ready`; ExtensionFoundation does not expose one. +public struct IOSBrokerDiagnostics: Equatable, Sendable { + public var state: IOSBrokerManagerState + public var epoch: BrokerEpoch? + public var extensionProcessIdentifier: Int32? + public var logicalHandleCount: Int + public var queuedOperationCount: Int + public var activeRequestID: BrokerRequestID? + /// Number of `AppExtensionProcess` initializer attempts, including failures. + public var launchAttemptCount: UInt64 + /// Number of attempts that reached and passed the worker `Ready` handshake. + public var launchCount: UInt64 + public var interruptionCount: UInt64 + public var admissionsPaused: Bool + + public init( + state: IOSBrokerManagerState, + epoch: BrokerEpoch?, + extensionProcessIdentifier: Int32?, + logicalHandleCount: Int, + queuedOperationCount: Int, + activeRequestID: BrokerRequestID?, + launchAttemptCount: UInt64, + launchCount: UInt64, + interruptionCount: UInt64, + admissionsPaused: Bool = false + ) { + self.state = state + self.epoch = epoch + self.extensionProcessIdentifier = extensionProcessIdentifier + self.logicalHandleCount = logicalHandleCount + self.queuedOperationCount = queuedOperationCount + self.activeRequestID = activeRequestID + self.launchAttemptCount = launchAttemptCount + self.launchCount = launchCount + self.interruptionCount = interruptionCount + self.admissionsPaused = admissionsPaused + } +} + +/// A sanitized worker snapshot returned over the diagnostics control message. +/// It intentionally contains no extension-private filesystem paths. +public struct IOSBrokerCheckpointMemorySample: Equatable, Sendable { + public var sequence: UInt64 + public var startedAtUptimeNanoseconds: UInt64 + public var sampledAtUptimeNanoseconds: UInt64 + public var completedAtUptimeNanoseconds: UInt64 + public var physFootprintBytes: UInt64 + public var residentBytes: UInt64 + public var availableMemoryBytes: UInt64 + + public init( + sequence: UInt64, + startedAtUptimeNanoseconds: UInt64, + sampledAtUptimeNanoseconds: UInt64, + completedAtUptimeNanoseconds: UInt64, + physFootprintBytes: UInt64, + residentBytes: UInt64, + availableMemoryBytes: UInt64 + ) { + self.sequence = sequence + self.startedAtUptimeNanoseconds = startedAtUptimeNanoseconds + self.sampledAtUptimeNanoseconds = sampledAtUptimeNanoseconds + self.completedAtUptimeNanoseconds = completedAtUptimeNanoseconds + self.physFootprintBytes = physFootprintBytes + self.residentBytes = residentBytes + self.availableMemoryBytes = availableMemoryBytes + } + + init(wire: IOSBrokerWireCheckpointMemorySample) { + self.init( + sequence: wire.sequence, + startedAtUptimeNanoseconds: wire.startedAtUptimeNanoseconds, + sampledAtUptimeNanoseconds: wire.sampledAtUptimeNanoseconds, + completedAtUptimeNanoseconds: wire.completedAtUptimeNanoseconds, + physFootprintBytes: wire.physFootprintBytes, + residentBytes: wire.residentBytes, + availableMemoryBytes: wire.availableMemoryBytes + ) + } +} + +public struct IOSBrokerWorkerDiagnostics: Equatable, Sendable { + public var state: String + public var epoch: BrokerEpoch + public var extensionProcessIdentifier: Int32 + public var manifestDigest: String? + public var activeRequestID: BrokerRequestID? + public var nativeDispatchStarted: Bool + public var transactionStatus: String + public var capabilities: BrokerCapabilities + public var currentPhysFootprintBytes: UInt64? + public var currentResidentBytes: UInt64? + public var availableMemoryBytes: UInt64? + public var checkpointInProgress: Bool + public var checkpointMemorySample: IOSBrokerCheckpointMemorySample? + public var storageProtectionEvidenceJSON: String? + public var extensionEntryPreOpenPhysFootprintBytes: UInt64? + public var extensionEntryPreOpenResidentBytes: UInt64? + public var openedIdlePhysFootprintBytes: UInt64? + public var openedIdleResidentBytes: UInt64? + + public init( + state: String, + epoch: BrokerEpoch, + extensionProcessIdentifier: Int32, + manifestDigest: String?, + activeRequestID: BrokerRequestID?, + nativeDispatchStarted: Bool, + transactionStatus: String, + capabilities: BrokerCapabilities, + currentPhysFootprintBytes: UInt64?, + currentResidentBytes: UInt64?, + availableMemoryBytes: UInt64? = nil, + checkpointInProgress: Bool = false, + checkpointMemorySample: IOSBrokerCheckpointMemorySample? = nil, + storageProtectionEvidenceJSON: String? = nil, + extensionEntryPreOpenPhysFootprintBytes: UInt64?, + extensionEntryPreOpenResidentBytes: UInt64?, + openedIdlePhysFootprintBytes: UInt64?, + openedIdleResidentBytes: UInt64? + ) { + self.state = state + self.epoch = epoch + self.extensionProcessIdentifier = extensionProcessIdentifier + self.manifestDigest = manifestDigest + self.activeRequestID = activeRequestID + self.nativeDispatchStarted = nativeDispatchStarted + self.transactionStatus = transactionStatus + self.capabilities = capabilities + self.currentPhysFootprintBytes = currentPhysFootprintBytes + self.currentResidentBytes = currentResidentBytes + self.availableMemoryBytes = availableMemoryBytes + self.checkpointInProgress = checkpointInProgress + self.checkpointMemorySample = checkpointMemorySample + self.storageProtectionEvidenceJSON = storageProtectionEvidenceJSON + self.extensionEntryPreOpenPhysFootprintBytes = extensionEntryPreOpenPhysFootprintBytes + self.extensionEntryPreOpenResidentBytes = extensionEntryPreOpenResidentBytes + self.openedIdlePhysFootprintBytes = openedIdlePhysFootprintBytes + self.openedIdleResidentBytes = openedIdleResidentBytes + } + + init(wire: IOSBrokerWireDiagnostics) { + self.init( + state: wire.state, + epoch: wire.epoch, + extensionProcessIdentifier: wire.extensionProcessIdentifier, + manifestDigest: wire.manifestDigest, + activeRequestID: wire.activeRequestID, + nativeDispatchStarted: wire.nativeDispatchStarted, + transactionStatus: wire.transactionStatus, + capabilities: wire.capabilities, + currentPhysFootprintBytes: wire.currentPhysFootprintBytes, + currentResidentBytes: wire.currentResidentBytes, + availableMemoryBytes: wire.availableMemoryBytes, + checkpointInProgress: wire.checkpointInProgress, + checkpointMemorySample: wire.checkpointMemorySample.map( + IOSBrokerCheckpointMemorySample.init(wire:) + ), + storageProtectionEvidenceJSON: wire.storageProtectionEvidenceJSON, + extensionEntryPreOpenPhysFootprintBytes: + wire.extensionEntryPreOpenPhysFootprintBytes, + extensionEntryPreOpenResidentBytes: + wire.extensionEntryPreOpenResidentBytes, + openedIdlePhysFootprintBytes: wire.openedIdlePhysFootprintBytes, + openedIdleResidentBytes: wire.openedIdleResidentBytes + ) + } +} + +@available(iOS 26.0, macOS 26.0, *) +extension AppExtensionPoint { + /// Bundle-only, non-UI extension point emitted into the containing app by + /// Xcode when `EX_ENABLE_EXTENSION_POINT_GENERATION=YES` is enabled. + @Definition public static var oliphauntBroker: AppExtensionPoint { + Name("OliphauntBroker") + UserInterface(false) + } +} + +@available(iOS 26.0, macOS 26.0, *) +enum IOSBrokerExtensionDiscovery { + static func discover(bundleIdentifier: String?) async throws -> AppExtensionIdentity { + let monitor: AppExtensionPoint.Monitor + do { + monitor = try await AppExtensionPoint.Monitor(appExtensionPoint: .oliphauntBroker) + } catch { + throw BrokerError.extensionMissing + } + + let identities = monitor.identities.filter { identity in + bundleIdentifier.map { identity.bundleIdentifier == $0 } ?? true + } + guard !identities.isEmpty else { + throw BrokerError.extensionMissing + } + guard identities.count == 1 else { + let choices = identities.map(\.bundleIdentifier).sorted().joined(separator: ", ") + throw BrokerError.invalidConfiguration( + "multiple broker extensions were discovered (\(choices)); select a bundle identifier" + ) + } + return identities[0] + } +} + +@available(iOS 26.0, macOS 26.0, *) +public struct IOSBrokerEngine: OliphauntEngine, OliphauntEngineSupportProvider { + public static let nativeServerUnavailableReason = + "NativeServer is unavailable on iOS; the broker exposes no listener or connection string" + + public let configuration: IOSBrokerConfiguration + public let manager: IOSBrokerManager + + public init( + configuration: IOSBrokerConfiguration, + manager: IOSBrokerManager = .shared + ) { + self.configuration = configuration + self.manager = manager + } + + public var supportedModes: [OliphauntEngineModeSupport] { + [ + OliphauntEngineModeSupport( + mode: .nativeDirect, + available: false, + capabilities: OliphauntSDKSupport.capabilities(for: .nativeDirect), + unavailableReason: "this engine is the iOS out-of-process broker adapter" + ), + OliphauntEngineModeSupport( + mode: .nativeBroker, + available: true, + capabilities: IOSBrokerCapabilityMapping.initial + ), + OliphauntEngineModeSupport( + mode: .nativeServer, + available: false, + capabilities: OliphauntSDKSupport.capabilities(for: .nativeServer), + unavailableReason: Self.nativeServerUnavailableReason + ), + ] + } + + public func open( + configuration databaseConfiguration: OliphauntConfiguration + ) async throws + -> any OliphauntSession + { + guard databaseConfiguration.mode == .nativeBroker else { + throw OliphauntError.runtimeUnavailable(databaseConfiguration.mode) + } + return try await manager.open( + configuration: configuration, + databaseConfiguration: databaseConfiguration + ) + } + + public func restore(_ request: OliphauntRestoreRequest) async throws -> URL { + throw OliphauntError.engine( + "iOS NativeBroker backup/restore is unavailable until a bounded streaming archive API exists" + ) + } +} + +enum IOSBrokerCapabilityMapping { + static let initial = OliphauntCapabilities( + mode: .nativeBroker, + processIsolated: true, + multiRoot: false, + reopenable: true, + sameRootLogicalReopen: true, + rootSwitchable: false, + crashRestartable: true, + independentSessions: false, + maxClientSessions: 1, + protocolRaw: true, + protocolStream: true, + queryCancel: true, + backupRestore: false, + backupFormats: [], + restoreFormats: [], + simpleQuery: true, + extensions: true, + connectionString: nil + ) + + static func map(_ broker: BrokerCapabilities) -> OliphauntCapabilities { + OliphauntCapabilities( + mode: .nativeBroker, + processIsolated: broker.processIsolated, + multiRoot: false, + reopenable: broker.sameRootLogicalReopen, + sameRootLogicalReopen: broker.sameRootLogicalReopen, + rootSwitchable: false, + crashRestartable: broker.crashRestartable, + independentSessions: false, + maxClientSessions: 1, + protocolRaw: broker.protocolRaw, + protocolStream: broker.protocolStream, + queryCancel: broker.queryCancel, + backupRestore: false, + backupFormats: [], + restoreFormats: [], + simpleQuery: true, + extensions: true, + connectionString: nil + ) + } +} diff --git a/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerSession.swift b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerSession.swift new file mode 100644 index 00000000..7e3df549 --- /dev/null +++ b/src/sdks/swift/Sources/OliphauntIOSBroker/IOSBrokerSession.swift @@ -0,0 +1,157 @@ +import Foundation +import Oliphaunt +import OliphauntBrokerProtocol + +/// A logical attachment to the one physical broker session. +/// +/// Multiple instances share the application-scoped manager and its FIFO. +/// Closing this value only releases its logical reference; it does not claim +/// that `AppExtensionProcess.invalidate()` terminated the worker process. +@available(iOS 26.0, macOS 26.0, *) +public actor IOSBrokerSession: OliphauntSession { + private let manager: IOSBrokerManager + private let handleID: UUID + private let maximumRawResponseBytes: Int + private var closed = false + + init( + manager: IOSBrokerManager, + handleID: UUID, + maximumRawResponseBytes: Int + ) { + self.manager = manager + self.handleID = handleID + self.maximumRawResponseBytes = maximumRawResponseBytes + } + + public func capabilities() async -> OliphauntCapabilities { + await manager.capabilities(for: handleID) + } + + public func workerDiagnostics() async throws -> IOSBrokerWorkerDiagnostics { + try requireOpen() + return try await manager.workerDiagnostics(handleID: handleID) + } + + public func execProtocolRaw(_ bytes: Data) async throws -> Data { + try requireOpen() + let collector = IOSBrokerResponseCollector( + maximumBytes: maximumRawResponseBytes + ) + try await manager.execute(handleID: handleID, bytes: bytes) { chunk in + try collector.append(chunk) + } + return collector.value + } + + public func execProtocolStream( + _ bytes: Data, + onChunk: @escaping @Sendable (Data) throws -> Void + ) async throws { + try requireOpen() + try await manager.execute( + handleID: handleID, + bytes: bytes, + onChunk: onChunk + ) + } + + public func backup( + _ request: OliphauntBackupRequest + ) async throws -> OliphauntBackupArtifact { + try requireOpen() + throw OliphauntError.engine( + "iOS NativeBroker backup is unavailable: whole-archive Data transfer is not memory bounded" + ) + } + + public func checkpoint() async throws { + try requireOpen() + try await manager.checkpoint(handleID: handleID) + } + + public func prepareForBackground( + deadline: Date + ) async throws -> OliphauntBackgroundPreparationResult { + try requireOpen() + return try await manager.prepareForBackground( + handleID: handleID, + deadline: deadline + ) + } + + public func prepareForBackground( + timeout: Duration = .seconds(5) + ) async throws -> OliphauntBackgroundPreparationResult { + try requireOpen() + return try await manager.prepareForBackground( + handleID: handleID, + timeout: timeout + ) + } + + public func resumeFromBackground() async throws { + try requireOpen() + try await manager.resumeFromBackground(handleID: handleID) + } + + public func cancel() async throws { + try requireOpen() + try await manager.cancel(handleID: handleID) + } + + #if DEBUG + /// Arms a one-shot extension fault for the simulator qualification fixture. + /// This API is absent from distribution builds. + public func injectFault(_ fault: BrokerWorkerFault) async throws { + try requireOpen() + try await manager.injectFault(handleID: handleID, fault: fault) + } + #endif + + public func close() async throws { + guard !closed else { + return + } + closed = true + try await manager.close(handleID: handleID) + } + + private func requireOpen() throws { + guard !closed else { + throw OliphauntError.databaseClosed + } + } +} + +enum IOSBrokerRawResponseLimitError: Error, Equatable, Sendable { + case exceeded(maximumBytes: Int) +} + +final class IOSBrokerResponseCollector: @unchecked Sendable { + private let lock = NSLock() + private let maximumBytes: Int + private var bytes = Data() + + init(maximumBytes: Int) { + precondition(maximumBytes > 0) + self.maximumBytes = maximumBytes + } + + func append(_ chunk: Data) throws { + lock.lock() + defer { lock.unlock() } + guard chunk.count <= maximumBytes - bytes.count else { + throw IOSBrokerRawResponseLimitError.exceeded( + maximumBytes: maximumBytes + ) + } + bytes.append(chunk) + } + + var value: Data { + lock.lock() + defer { lock.unlock() } + return bytes + } +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/BrokerBackendPrivacyFilterTests.swift b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/BrokerBackendPrivacyFilterTests.swift new file mode 100644 index 00000000..fbfc2236 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/BrokerBackendPrivacyFilterTests.swift @@ -0,0 +1,346 @@ +import Foundation +import Testing + +@testable import OliphauntBrokerExtension + +@Test +func backendPrivacyFilterRedactsEveryTextFieldAcrossOneByteSplits() throws { + let sensitivePrefix = "/private/var/mobile/Containers/Data/Application/secret/PGDATA" + let textFieldTags: [UInt8] = [ + 0x4d, 0x44, 0x48, 0x57, 0x50, 0x70, 0x71, 0x73, 0x74, 0x63, 0x64, 0x6e, + 0x46, 0x4c, 0x52, + ] + + for responseTag: UInt8 in [0x45, 0x4e] { + var fields: [(UInt8, String)] = [ + (0x53, responseTag == 0x45 ? "ERROR" : "NOTICE"), + (0x56, responseTag == 0x45 ? "ERROR" : "NOTICE"), + (0x43, "58P01"), + ] + fields.append( + contentsOf: textFieldTags.map { tag in + (tag, "context before \(sensitivePrefix)/base/123 after") + }) + let original = backendFieldMessage(tag: responseTag, fields: fields) + let result = try runPrivacyFilter( + original, + sensitivePrefixes: [sensitivePrefix], + chunkSizes: Array(repeating: 1, count: original.count) + ) + + #expect(!result.output.containsBytes(Data(sensitivePrefix.utf8))) + #expect(result.outputChunks.count == 1) + let decoded = try decodeBackendFieldMessage(result.output) + #expect(decoded.tag == responseTag) + #expect( + decoded.fields.first(where: { $0.0 == 0x53 })?.1 == "ERROR" + || decoded.fields.first(where: { $0.0 == 0x53 })?.1 == "NOTICE") + #expect( + decoded.fields.first(where: { $0.0 == 0x56 })?.1 == "ERROR" + || decoded.fields.first(where: { $0.0 == 0x56 })?.1 == "NOTICE") + #expect(decoded.fields.first(where: { $0.0 == 0x43 })?.1 == "58P01") + for tag in textFieldTags { + #expect(decoded.fields.first(where: { $0.0 == tag })?.1 == "[redacted]") + } + } +} + +@Test +func backendPrivacyFilterPreservesSafeFieldsAndErrorReadyAdjacency() throws { + let prefix = "/private/extension/root" + let safeError = backendFieldMessage( + tag: 0x45, + fields: [ + (0x53, "ERROR"), + (0x56, "ERROR"), + (0x43, "22012"), + (0x4d, "division by zero"), + ] + ) + let ready = backendMessage(tag: 0x5a, body: Data([0x49])) + let input = safeError + ready + let result = try runPrivacyFilter( + input, + sensitivePrefixes: [prefix], + chunkSizes: [2, safeError.count - 1, input.count] + ) + + #expect(result.output == input) + let messages = try splitBackendMessages(result.output) + #expect(messages == [safeError, ready]) +} + +@Test +func backendPrivacyFilterStreamsLargeNonSensitiveFramesByteForByte() throws { + let payload = Data(repeating: 0x61, count: 2 * 1024 * 1024) + var dataRowBody = Data([0, 1]) + appendNetworkUInt32(UInt32(payload.count), to: &dataRowBody) + dataRowBody.append(payload) + let dataRow = backendMessage(tag: 0x44, body: dataRowBody) + let filter = try BrokerBackendPrivacyFilter( + sensitiveAbsolutePrefixes: ["/private/extension/root"] + ) + var outputChunks: [Data] = [] + + let firstCount = 97 + try filter.process(dataRow.prefix(firstCount)) { outputChunks.append($0) } + #expect(!outputChunks.isEmpty) + var offset = firstCount + while offset < dataRow.count { + let end = min(offset + 4093, dataRow.count) + try filter.process(dataRow[offset.. 500) +} + +@Test +func backendPrivacyFilterReplacesOversizedErrorsAndNoticesWithBoundedMessages() throws { + let sensitivePrefix = "/private/extension/runtime" + let filler = String( + repeating: "x", count: BrokerBackendPrivacyFilter.maximumBufferedMessageBytes) + + for responseTag: UInt8 in [0x45, 0x4e] { + let oversized = backendFieldMessage( + tag: responseTag, + fields: [ + (0x53, responseTag == 0x45 ? "ERROR" : "NOTICE"), + (0x43, "58P01"), + (0x4d, "could not open \(sensitivePrefix)/share/stopwords"), + (0x44, filler), + ] + ) + let result = try runPrivacyFilter( + oversized, + sensitivePrefixes: [sensitivePrefix], + chunkSizes: Array(repeating: 137, count: oversized.count / 137 + 1) + ) + + #expect(result.output.count < 256) + #expect(!result.output.containsBytes(Data(sensitivePrefix.utf8))) + let decoded = try decodeBackendFieldMessage(result.output) + #expect(decoded.tag == responseTag) + #expect(decoded.fields.first(where: { $0.0 == 0x43 })?.1 == "58P01") + let message = try #require(decoded.fields.first(where: { $0.0 == 0x4d })?.1) + #expect(message.contains("redacted")) + #expect(!message.contains("/")) + } +} + +@Test +func backendPrivacyFilterFailsClosedForMalformedAndTruncatedMessages() throws { + let prefix = "/private/extension/root" + + do { + let filter = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: [prefix]) + var output = Data() + var malformed = Data([0x45, 0, 0, 0, 3]) + malformed.append(Data(prefix.utf8)) + try expectPrivacyFilterError(.malformedBackendMessage, sensitive: prefix) { + try filter.process(malformed) { output.append($0) } + } + #expect(output.isEmpty) + } + + do { + let filter = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: [prefix]) + var body = Data([0x4d]) + body.append(Data("failure at \(prefix)/base".utf8)) + let malformed = backendMessage(tag: 0x45, body: body) + var output = Data() + try expectPrivacyFilterError(.malformedBackendMessage, sensitive: prefix) { + try filter.process(malformed) { output.append($0) } + } + #expect(output.isEmpty) + } + + do { + let filter = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: [prefix]) + let complete = backendFieldMessage( + tag: 0x4e, + fields: [(0x53, "NOTICE"), (0x4d, "failure at \(prefix)/base")] + ) + let truncated = complete.dropLast(3) + var output = Data() + try filter.process(truncated) { output.append($0) } + #expect(output.isEmpty) + try expectPrivacyFilterError(.incompleteBackendMessage, sensitive: prefix) { + try filter.finish() + } + #expect(output.isEmpty) + } + + do { + let filter = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: [prefix]) + let dataRow = backendMessage(tag: 0x44, body: Data(repeating: 0x61, count: 40)) + var output = Data() + try filter.process(dataRow.dropLast()) { output.append($0) } + #expect(!output.isEmpty) + try expectPrivacyFilterError(.incompleteBackendMessage, sensitive: prefix) { + try filter.finish() + } + } +} + +@Test +func backendPrivacyFilterExpandsResolvedPathAliasesAndRejectsRoot() throws { + let temporary = FileManager.default.temporaryDirectory + .appendingPathComponent("broker-privacy-filter-\(UUID().uuidString)", isDirectory: true) + let real = temporary.appendingPathComponent("real", isDirectory: true) + let alias = temporary.appendingPathComponent("alias", isDirectory: true) + try FileManager.default.createDirectory(at: real, withIntermediateDirectories: true) + try FileManager.default.createSymbolicLink(at: alias, withDestinationURL: real) + defer { try? FileManager.default.removeItem(at: temporary) } + + let filter = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: [alias.path]) + let message = backendFieldMessage( + tag: 0x45, + fields: [(0x53, "ERROR"), (0x43, "58P01"), (0x4d, "missing \(real.path)/file")] + ) + var output = Data() + try filter.process(message) { output.append($0) } + try filter.finish() + #expect(!output.containsBytes(Data(real.path.utf8))) + #expect(try decodeBackendFieldMessage(output).fields.last?.1 == "[redacted]") + + try expectPrivacyFilterError(.invalidConfiguration, sensitive: "ignored") { + _ = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: ["/"]) + } + try expectPrivacyFilterError(.invalidConfiguration, sensitive: "ignored") { + _ = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: []) + } +} + +private struct PrivacyFilterRunResult { + var output: Data + var outputChunks: [Data] +} + +private func runPrivacyFilter( + _ input: Data, + sensitivePrefixes: [String], + chunkSizes: [Int] +) throws -> PrivacyFilterRunResult { + let filter = try BrokerBackendPrivacyFilter(sensitiveAbsolutePrefixes: sensitivePrefixes) + var outputChunks: [Data] = [] + var offset = 0 + var chunkIndex = 0 + while offset < input.count { + let requested = chunkIndex < chunkSizes.count ? chunkSizes[chunkIndex] : input.count + let end = min(offset + max(1, requested), input.count) + try filter.process(input[offset.. Data { + var body = Data() + for (fieldTag, value) in fields { + body.append(fieldTag) + body.append(Data(value.utf8)) + body.append(0) + } + body.append(0) + return backendMessage(tag: tag, body: body) +} + +private func backendMessage(tag: UInt8, body: Data) -> Data { + var message = Data([tag]) + appendNetworkUInt32(UInt32(body.count + 4), to: &message) + message.append(body) + return message +} + +private func appendNetworkUInt32(_ value: UInt32, to data: inout Data) { + data.append(UInt8(truncatingIfNeeded: value >> 24)) + data.append(UInt8(truncatingIfNeeded: value >> 16)) + data.append(UInt8(truncatingIfNeeded: value >> 8)) + data.append(UInt8(truncatingIfNeeded: value)) +} + +private func splitBackendMessages(_ data: Data) throws -> [Data] { + var messages: [Data] = [] + var offset = 0 + while offset < data.count { + guard data.count - offset >= 5 else { throw PrivacyFilterTestError.malformed } + let length = readNetworkUInt32(data[(offset + 1)..<(offset + 5)]) + guard length >= 4, let wireLength = Int(exactly: length + 1), + wireLength <= data.count - offset + else { + throw PrivacyFilterTestError.malformed + } + messages.append(data[offset..<(offset + wireLength)]) + offset += wireLength + } + return messages +} + +private func decodeBackendFieldMessage( + _ data: Data +) throws -> (tag: UInt8, fields: [(UInt8, String)]) { + guard data.count >= 6 else { throw PrivacyFilterTestError.malformed } + let tag = data[data.startIndex] + let length = readNetworkUInt32(data[(data.startIndex + 1)..<(data.startIndex + 5)]) + guard Int(length) + 1 == data.count else { throw PrivacyFilterTestError.malformed } + let body = [UInt8](data.dropFirst(5)) + var fields: [(UInt8, String)] = [] + var offset = 0 + while offset < body.count { + let fieldTag = body[offset] + offset += 1 + if fieldTag == 0 { + guard offset == body.count else { throw PrivacyFilterTestError.malformed } + return (tag, fields) + } + guard let end = body[offset...].firstIndex(of: 0), + let value = String(bytes: body[offset.. UInt32 { + bytes.reduce(UInt32(0)) { ($0 << 8) | UInt32($1) } +} + +private func expectPrivacyFilterError( + _ expected: BrokerBackendPrivacyFilterError, + sensitive: String, + operation: () throws -> Void +) throws { + do { + try operation() + Issue.record("expected backend privacy filter error \(expected)") + } catch let error as BrokerBackendPrivacyFilterError { + #expect(error == expected) + #expect(!error.localizedDescription.contains(sensitive)) + #expect(!error.localizedDescription.contains("/")) + } catch { + Issue.record("unexpected backend privacy filter error: \(error)") + } +} + +private enum PrivacyFilterTestError: Error { + case malformed +} + +extension Data { + fileprivate func containsBytes(_ bytes: Data) -> Bool { + range(of: bytes) != nil + } +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/BrokerExtensionStorageTests.swift b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/BrokerExtensionStorageTests.swift new file mode 100644 index 00000000..d304a7f1 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/BrokerExtensionStorageTests.swift @@ -0,0 +1,195 @@ +import Foundation +import Testing + +@testable import OliphauntBrokerExtension + +@Test +func recursiveDataProtectionAppliesToEveryDirectoryAndRegularFileRootFirst() throws { + let fileManager = FileManager.default + let root = fileManager.temporaryDirectory.appendingPathComponent( + "oliphaunt-protection-apply-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? fileManager.removeItem(at: root) } + + let runtime = root.appendingPathComponent("runtime-cache/runtime/files", isDirectory: true) + let runtimeFile = runtime.appendingPathComponent("share/icu/icudt.dat", isDirectory: false) + let relationDirectory = root.appendingPathComponent("pgdata/base/16384", isDirectory: true) + let relationFile = relationDirectory.appendingPathComponent("32768", isDirectory: false) + let walDirectory = root.appendingPathComponent("pgdata/pg_wal", isDirectory: true) + let walFile = walDirectory.appendingPathComponent( + "000000010000000000000001", + isDirectory: false + ) + let staging = root.appendingPathComponent("staging", isDirectory: true) + let manifest = root.appendingPathComponent("manifest.json", isDirectory: false) + + try fileManager.createDirectory( + at: runtimeFile.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try fileManager.createDirectory(at: relationDirectory, withIntermediateDirectories: true) + try fileManager.createDirectory(at: walDirectory, withIntermediateDirectories: true) + try fileManager.createDirectory(at: staging, withIntermediateDirectories: true) + try Data("runtime".utf8).write(to: runtimeFile) + try Data("relation".utf8).write(to: relationFile) + try Data("wal".utf8).write(to: walFile) + try Data("{}".utf8).write(to: manifest) + + let storage = try BrokerExtensionStorage(location: .extensionPrivate, rootURL: root) + var appliedPaths: [String] = [] + try storage.enforceDataProtectionRecursively(fileManager: fileManager) { url in + appliedPaths.append(url.standardizedFileURL.path) + } + + var expectedPaths = Set([root.standardizedFileURL.path]) + let enumerator = try #require(fileManager.enumerator(at: root, includingPropertiesForKeys: nil)) + for case let url as URL in enumerator { + expectedPaths.insert(url.standardizedFileURL.path) + } + #expect(Set(appliedPaths) == expectedPaths) + #expect(appliedPaths.count == expectedPaths.count) + + let indexes = Dictionary( + uniqueKeysWithValues: appliedPaths.enumerated().map { ($0.element, $0.offset) }) + for path in appliedPaths where path != root.path { + let parent = URL(fileURLWithPath: path).deletingLastPathComponent().standardizedFileURL.path + #expect(try #require(indexes[parent]) < (try #require(indexes[path]))) + } +} + +@Test +func recursiveProtectionEvidenceFailsClosedOnTraversalError() throws { + let fileManager = FileManager.default + let root = fileManager.temporaryDirectory.appendingPathComponent( + "oliphaunt-protection-audit-\(UUID().uuidString)", + isDirectory: true + ) + let denied = root.appendingPathComponent("denied", isDirectory: true) + try fileManager.createDirectory(at: denied, withIntermediateDirectories: true) + try Data("unreadable".utf8).write( + to: denied.appendingPathComponent("entry", isDirectory: false) + ) + try fileManager.setAttributes([.posixPermissions: 0], ofItemAtPath: denied.path) + defer { + try? fileManager.setAttributes([.posixPermissions: 0o700], ofItemAtPath: denied.path) + try? fileManager.removeItem(at: root) + } + + let storage = try BrokerExtensionStorage( + location: .extensionPrivate, + rootURL: root + ) + let evidence = storage.recursiveProtectionEvidence(fileManager: fileManager) + + #expect(evidence.enumerationFailed) + #expect(evidence.unreadableEntryCount > 0) + #expect(!evidence.allEntriesMatchExpectedProtection) +} + +@Test +func recursiveDataProtectionFailsClosedOnTraversalErrorWithoutLeakingPaths() throws { + let fileManager = FileManager.default + let root = fileManager.temporaryDirectory.appendingPathComponent( + "oliphaunt-protection-enforcement-\(UUID().uuidString)", + isDirectory: true + ) + let denied = root.appendingPathComponent("private-entry-name", isDirectory: true) + try fileManager.createDirectory(at: denied, withIntermediateDirectories: true) + try Data("unreadable".utf8).write( + to: denied.appendingPathComponent("secret-file-name", isDirectory: false) + ) + try fileManager.setAttributes([.posixPermissions: 0], ofItemAtPath: denied.path) + defer { + try? fileManager.setAttributes([.posixPermissions: 0o700], ofItemAtPath: denied.path) + try? fileManager.removeItem(at: root) + } + + let storage = try BrokerExtensionStorage(location: .extensionPrivate, rootURL: root) + var appliedPaths: [String] = [] + do { + try storage.enforceDataProtectionRecursively(fileManager: fileManager) { url in + appliedPaths.append(url.path) + } + Issue.record("recursive protection unexpectedly accepted an unreadable subtree") + } catch { + let description = String(describing: error) + #expect(appliedPaths.isEmpty) + #expect(description.contains("cannot enforce broker storage data protection")) + #expect(!description.contains(root.path)) + #expect(!description.contains("private-entry-name")) + #expect(!description.contains("secret-file-name")) + } +} + +@Test +func recursiveDataProtectionRejectsSymbolicLinksWithoutLeakingPaths() throws { + let fileManager = FileManager.default + let root = fileManager.temporaryDirectory.appendingPathComponent( + "oliphaunt-protection-symlink-\(UUID().uuidString)", + isDirectory: true + ) + let outside = fileManager.temporaryDirectory.appendingPathComponent( + "oliphaunt-protection-outside-\(UUID().uuidString)", + isDirectory: true + ) + try fileManager.createDirectory(at: root, withIntermediateDirectories: true) + try fileManager.createDirectory(at: outside, withIntermediateDirectories: true) + let link = root.appendingPathComponent("private-link-name", isDirectory: false) + try fileManager.createSymbolicLink(at: link, withDestinationURL: outside) + defer { + try? fileManager.removeItem(at: root) + try? fileManager.removeItem(at: outside) + } + + let storage = try BrokerExtensionStorage(location: .extensionPrivate, rootURL: root) + var appliedPaths: [String] = [] + do { + try storage.enforceDataProtectionRecursively(fileManager: fileManager) { url in + appliedPaths.append(url.path) + } + Issue.record("recursive protection unexpectedly accepted a symbolic link") + } catch { + let description = String(describing: error) + #expect(appliedPaths.isEmpty) + #expect(description.contains("cannot enforce broker storage data protection")) + #expect(!description.contains(root.path)) + #expect(!description.contains("private-link-name")) + } +} + +@Test +func recursiveDataProtectionSanitizesApplicationFailureAndStopsMutation() throws { + let fileManager = FileManager.default + let root = fileManager.temporaryDirectory.appendingPathComponent( + "oliphaunt-protection-application-\(UUID().uuidString)", + isDirectory: true + ) + let nested = root.appendingPathComponent("nested", isDirectory: true) + let sensitiveName = "private-application-failure" + let failure = nested.appendingPathComponent(sensitiveName, isDirectory: false) + try fileManager.createDirectory(at: nested, withIntermediateDirectories: true) + try Data("failure".utf8).write(to: failure) + defer { try? fileManager.removeItem(at: root) } + + let storage = try BrokerExtensionStorage(location: .extensionPrivate, rootURL: root) + var appliedPaths: [String] = [] + do { + try storage.enforceDataProtectionRecursively(fileManager: fileManager) { url in + appliedPaths.append(url.standardizedFileURL.path) + if url.standardizedFileURL.path == failure.standardizedFileURL.path { + throw NSError( + domain: "failed to protect \(failure.path)", + code: 1 + ) + } + } + Issue.record("recursive protection unexpectedly ignored an application failure") + } catch { + let description = String(describing: error) + #expect(appliedPaths.last == failure.standardizedFileURL.path) + #expect(description.contains("cannot enforce broker storage data protection")) + #expect(!description.contains(root.path)) + #expect(!description.contains(sensitiveName)) + } +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/CapabilityBoundaryTests.swift b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/CapabilityBoundaryTests.swift new file mode 100644 index 00000000..c7052626 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/CapabilityBoundaryTests.swift @@ -0,0 +1,54 @@ +import Foundation +import Oliphaunt +import OliphauntBrokerExtension +import OliphauntBrokerProtocol +import Testing + +@Test +func workerReportsTheConservativeArchiveAndServerCapabilities() throws { + let configuration = try archiveBoundaryConfiguration() + #expect(!configuration.capabilities.backupRestore) + #expect(configuration.capabilities.connectionString == nil) + #expect(!configuration.capabilities.serverMode) +} + +@Test +func backupIsRejectedBeforeNativeDispatch() async throws { + try await expectUnsupportedArchiveOperation("backup") +} + +@Test +func restoreIsRejectedBeforeNativeDispatch() async throws { + try await expectUnsupportedArchiveOperation("restore") +} + +private func expectUnsupportedArchiveOperation(_ operation: String) async throws { + let configuration = try archiveBoundaryConfiguration() + let core = WorkerCore(configuration: configuration) + do { + try await core.rejectBackupOrRestore() + Issue.record("\(operation) unexpectedly crossed the broker boundary") + } catch let error as BrokerError { + guard case .rejected(.unsupportedCapability(let capability)) = error else { + Issue.record("expected unsupported backupRestore rejection, got \(error)") + return + } + #expect(capability == .backupRestore) + } +} + +private func archiveBoundaryConfiguration() throws -> BrokerWorkerConfiguration { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("oliphaunt-broker-archive-boundary-\(UUID().uuidString)") + + let storage = try BrokerExtensionStorage( + location: .extensionPrivate, + rootURL: root + ) + return try BrokerWorkerConfiguration( + storage: storage, + engine: RuntimeUnavailableEngine(), + liboliphauntVersion: "archive-boundary-test", + startupConfigurationDigest: "archive-boundary-test" + ) +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/RestrictedRoleBootstrapTests.swift b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/RestrictedRoleBootstrapTests.swift new file mode 100644 index 00000000..8bc58f13 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/RestrictedRoleBootstrapTests.swift @@ -0,0 +1,390 @@ +import Foundation +import Oliphaunt +import OliphauntBrokerExtension +import OliphauntBrokerProtocol +import Testing + +@Test +func workerConfigurationRequiresTheRestrictedDatabaseIdentity() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("oliphaunt-worker-role-config-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let storage = try BrokerExtensionStorage(location: .extensionPrivate, rootURL: root) + let engine = RestrictedRoleTestEngine() + + let configuration = try BrokerWorkerConfiguration( + storage: storage, + engine: engine, + liboliphauntVersion: "restricted-role-test-runtime", + startupConfigurationDigest: "restricted-role-test-configuration" + ) + #expect(configuration.username == BrokerWorkerConfiguration.restrictedRoleUsername) + #expect(configuration.database == BrokerWorkerConfiguration.restrictedDatabase) + #expect( + configuration.startupGUCs == [ + OliphauntStartupGUC( + "search_path", + BrokerWorkerConfiguration.restrictedSearchPath + ) + ] + ) + + #expect(throws: BrokerError.self) { + _ = try BrokerWorkerConfiguration( + storage: storage, + engine: engine, + liboliphauntVersion: "restricted-role-test-runtime", + startupConfigurationDigest: "restricted-role-test-configuration", + startupGUCs: [OliphauntStartupGUC("SEARCH_PATH", "public")] + ) + } + + for identity in [("postgres", "postgres"), ("oliphaunt_broker", "template1")] { + do { + _ = try BrokerWorkerConfiguration( + storage: storage, + engine: engine, + liboliphauntVersion: "restricted-role-test-runtime", + startupConfigurationDigest: "restricted-role-test-configuration", + username: identity.0, + database: identity.1 + ) + Issue.record("insecure worker identity unexpectedly passed validation: \(identity)") + } catch let error as BrokerError { + guard case .invalidConfiguration = error else { + Issue.record("insecure worker identity returned the wrong error: \(error)") + continue + } + } + } +} + +@Test +func workerEstablishesRestrictedRoleBoundaryBeforeReady() async throws { + let harness = try RestrictedRoleHarness(bootstrapFails: false) + defer { harness.removeStorage() } + + _ = try await harness.core.start(hello: harness.hello) + + let opened = try #require(await harness.engine.openedConfiguration()) + #expect(opened.username == BrokerWorkerConfiguration.restrictedRoleUsername) + #expect(opened.database == BrokerWorkerConfiguration.restrictedDatabase) + #expect( + opened.startupGUCs.last + == OliphauntStartupGUC( + "search_path", + BrokerWorkerConfiguration.restrictedSearchPath + ) + ) + + let session = harness.engine.session + let queries = await session.queries() + #expect(queries.count == 2) + let bootstrap = try #require(queries.first) + #expect(bootstrap.hasPrefix("BEGIN;")) + #expect(bootstrap.contains("CREATE EXTENSION IF NOT EXISTS %I")) + #expect(bootstrap.contains("ARRAY['pg_trgm', 'vector']::text[]")) + #expect(bootstrap.contains("REASSIGN OWNED BY \"oliphaunt_broker\"")) + #expect(bootstrap.contains("TO \"postgres\"")) + #expect(bootstrap.contains("CREATE SCHEMA IF NOT EXISTS \"oliphaunt_broker\"")) + #expect(bootstrap.contains("AUTHORIZATION \"oliphaunt_broker\"")) + #expect(bootstrap.contains("GRANT CONNECT, TEMPORARY")) + #expect(bootstrap.contains("ON DATABASE \"postgres\"")) + #expect(bootstrap.contains("REVOKE CREATE")) + #expect(bootstrap.contains("GRANT USAGE, CREATE")) + #expect(bootstrap.contains("ON SCHEMA \"oliphaunt_broker\"")) + #expect(bootstrap.contains("GRANT USAGE ON SCHEMA public")) + #expect(bootstrap.contains("GRANT pg_checkpoint TO \"oliphaunt_broker\"")) + #expect(bootstrap.contains("pg_relation_filepath(regclass)")) + #expect(bootstrap.contains("pg_tablespace_location(oid)")) + #expect(bootstrap.contains("FROM PUBLIC, \"oliphaunt_broker\"")) + #expect(bootstrap.contains("parent.rolname <> 'pg_checkpoint'")) + #expect(bootstrap.contains("direct_memberships <> ARRAY['pg_checkpoint']::text[]")) + #expect(bootstrap.contains("WITH RECURSIVE effective_role_oids(oid)")) + #expect(bootstrap.contains("ARRAY['oliphaunt_broker', 'pg_checkpoint']::text[]")) + #expect(bootstrap.contains("NOSUPERUSER NOCREATEDB NOCREATEROLE")) + #expect(bootstrap.contains("INHERIT LOGIN NOREPLICATION NOBYPASSRLS")) + #expect( + bootstrap.components(separatedBy: "SET search_path TO \"$user\", public").count + == 3 + ) + #expect(bootstrap.contains("rolconfig @> ARRAY['search_path=\"$user\", public']")) + #expect(bootstrap.contains("SET SESSION AUTHORIZATION \"oliphaunt_broker\"")) + #expect(bootstrap.contains("SET search_path TO \"$user\", public")) + #expect(bootstrap.contains("current_schemas(false)")) + #expect(bootstrap.contains("ARRAY['oliphaunt_broker', 'public']::name[]")) + #expect(bootstrap.contains("session_user <> 'oliphaunt_broker'")) + #expect(bootstrap.contains("current_user <> 'oliphaunt_broker'")) + #expect(bootstrap.contains("current_setting('is_superuser') <> 'off'")) + #expect(bootstrap.contains("owner.rolname = 'postgres'")) + #expect(bootstrap.contains("owner.rolname <> 'postgres'")) + #expect(bootstrap.contains("'pg_database_owner', 'SET'")) + #expect(bootstrap.contains("current_database(), 'CREATE'")) + #expect(bootstrap.contains("'public', 'CREATE'")) + #expect(bootstrap.contains("spcname NOT IN ('pg_default', 'pg_global')")) + #expect(!bootstrap.contains("ALTER DATABASE")) + #expect(bootstrap.hasSuffix("COMMIT;")) + #expect(queries[1] == "SELECT 1 AS broker_health") +} + +@Test +func failedRestrictedRoleValidationRollsBackAndNeverPublishesReady() async throws { + let harness = try RestrictedRoleHarness(bootstrapFails: true) + defer { harness.removeStorage() } + let originalEpoch = await harness.core.epoch + + do { + _ = try await harness.core.start(hello: harness.hello) + Issue.record("worker unexpectedly published Ready after restricted-role failure") + } catch let error as BrokerError { + #expect(error == .brokerUnavailable) + } + + let session = harness.engine.session + #expect(await session.queries().last == "ROLLBACK") + #expect(await session.closeCallCount() == 1) + let diagnostics = try await harness.core.diagnostics(expectedEpoch: originalEpoch) + #expect(diagnostics.state == .created) +} + +@Test +func postOpenStorageProtectionFailureClosesSessionAndNeverPublishesReady() async throws { + let harness = try RestrictedRoleHarness( + bootstrapFails: false, + createsUnprotectablePostOpenArtifact: true + ) + defer { harness.removeStorage() } + let originalEpoch = await harness.core.epoch + + do { + _ = try await harness.core.start(hello: harness.hello) + Issue.record("worker unexpectedly published Ready after storage-protection failure") + } catch let error as BrokerError { + #expect(error == .rejected(.rootOpen)) + #expect(!error.description.contains(harness.rootURL.path)) + #expect(!error.description.contains("post-open-private-artifact")) + } + + #expect(await harness.engine.session.closeCallCount() == 1) + let diagnostics = try await harness.core.diagnostics(expectedEpoch: originalEpoch) + #expect(diagnostics.state == .created) +} + +private struct RestrictedRoleHarness { + let rootURL: URL + let engine: RestrictedRoleTestEngine + let core: WorkerCore + let hello: BrokerHello + + init( + bootstrapFails: Bool, + createsUnprotectablePostOpenArtifact: Bool = false + ) throws { + rootURL = FileManager.default.temporaryDirectory + .appendingPathComponent("oliphaunt-worker-role-\(UUID().uuidString)") + engine = RestrictedRoleTestEngine( + bootstrapFails: bootstrapFails, + createsUnprotectablePostOpenArtifact: createsUnprotectablePostOpenArtifact + ) + let configuration = try BrokerWorkerConfiguration( + storage: BrokerExtensionStorage(location: .extensionPrivate, rootURL: rootURL), + engine: engine, + liboliphauntVersion: "restricted-role-test-runtime", + cABIVersion: 42, + postgresMajorVersion: 18, + startupConfigurationDigest: "restricted-role-test-configuration", + selectedPostgresExtensions: ["vector", "pg_trgm"], + runtimeVersionProvider: { "restricted-role-test-runtime" } + ) + core = WorkerCore(configuration: configuration) + hello = BrokerHello( + expectedABI: 42, + expectedRuntimeVersion: "restricted-role-test-runtime", + startupConfigurationDigest: "restricted-role-test-configuration", + requestedCapabilities: [.protocolRaw, .protocolStream, .queryCancel] + ) + } + + func removeStorage() { + try? FileManager.default.removeItem(at: rootURL) + } +} + +private actor RestrictedRoleTestEngine: OliphauntEngine { + let session: RestrictedRoleTestSession + private let createsUnprotectablePostOpenArtifact: Bool + private var configuration: OliphauntConfiguration? + + init( + bootstrapFails: Bool = false, + createsUnprotectablePostOpenArtifact: Bool = false + ) { + session = RestrictedRoleTestSession(bootstrapFails: bootstrapFails) + self.createsUnprotectablePostOpenArtifact = createsUnprotectablePostOpenArtifact + } + + func open(configuration: OliphauntConfiguration) async throws -> any OliphauntSession { + self.configuration = configuration + let root = try #require(configuration.root) + let pgdata = root.appendingPathComponent("pgdata", isDirectory: true) + try FileManager.default.createDirectory(at: pgdata, withIntermediateDirectories: true) + try Data("18\n".utf8).write( + to: pgdata.appendingPathComponent("PG_VERSION"), + options: .atomic + ) + if createsUnprotectablePostOpenArtifact { + let artifact = root.appendingPathComponent( + "runtime-cache/post-open-private-artifact", + isDirectory: false + ) + try FileManager.default.createSymbolicLink( + at: artifact, + withDestinationURL: pgdata + ) + } + return session + } + + func restore(_ request: OliphauntRestoreRequest) async throws -> URL { + request.root + } + + func openedConfiguration() -> OliphauntConfiguration? { + configuration + } +} + +private actor RestrictedRoleTestSession: OliphauntSession { + private let bootstrapFails: Bool + private var recordedQueries: [String] = [] + private var closeCalls = 0 + + init(bootstrapFails: Bool) { + self.bootstrapFails = bootstrapFails + } + + func capabilities() async -> OliphauntCapabilities { + OliphauntCapabilities( + mode: .nativeDirect, + processIsolated: false, + independentSessions: false, + maxClientSessions: 1 + ) + } + + func execProtocolRaw(_ bytes: Data) async throws -> Data { + let query = try #require(simpleQueryText(bytes)) + recordedQueries.append(query) + if bootstrapFails, recordedQueries.count == 1 { + return restrictedRoleErrorResponse() + } + if query == "SELECT 1 AS broker_health" { + return restrictedRoleHealthResponse() + } + return restrictedRoleCommandResponse() + } + + func execProtocolStream( + _ bytes: Data, + onChunk: @escaping @Sendable (Data) throws -> Void + ) async throws { + try onChunk(restrictedRoleHealthResponse()) + } + + func backup(_ request: OliphauntBackupRequest) async throws -> OliphauntBackupArtifact { + OliphauntBackupArtifact(format: request.format, bytes: Data()) + } + + func cancel() async throws {} + + func close() async throws { + closeCalls += 1 + } + + func queries() -> [String] { + recordedQueries + } + + func closeCallCount() -> Int { + closeCalls + } +} + +private func simpleQueryText(_ request: Data) -> String? { + guard request.count >= 6, + request.first == 0x51, + request.last == 0 + else { + return nil + } + return String(decoding: request.dropFirst(5).dropLast(), as: UTF8.self) +} + +private func restrictedRoleCommandResponse() -> Data { + var response = Data() + appendRestrictedRoleBackendMessage(0x43, body: Data("DO\0".utf8), to: &response) + appendRestrictedRoleBackendMessage(0x5a, body: Data([0x49]), to: &response) + return response +} + +private func restrictedRoleErrorResponse() -> Data { + var response = Data() + var error = Data() + error.append(0x53) + error.append(Data("ERROR\0".utf8)) + error.append(0x43) + error.append(Data("42501\0".utf8)) + error.append(0x4d) + error.append(Data("broker restricted role validation failed\0".utf8)) + error.append(0) + appendRestrictedRoleBackendMessage(0x45, body: error, to: &response) + appendRestrictedRoleBackendMessage(0x5a, body: Data([0x45]), to: &response) + return response +} + +private func restrictedRoleHealthResponse() -> Data { + var response = Data() + var rowDescription = Data() + appendRestrictedRoleInt16(1, to: &rowDescription) + rowDescription.append(Data("broker_health".utf8)) + rowDescription.append(0) + appendRestrictedRoleUInt32(0, to: &rowDescription) + appendRestrictedRoleInt16(0, to: &rowDescription) + appendRestrictedRoleUInt32(23, to: &rowDescription) + appendRestrictedRoleInt16(4, to: &rowDescription) + appendRestrictedRoleUInt32(UInt32.max, to: &rowDescription) + appendRestrictedRoleInt16(0, to: &rowDescription) + appendRestrictedRoleBackendMessage(0x54, body: rowDescription, to: &response) + + var row = Data() + appendRestrictedRoleInt16(1, to: &row) + appendRestrictedRoleUInt32(1, to: &row) + row.append(Data("1".utf8)) + appendRestrictedRoleBackendMessage(0x44, body: row, to: &response) + appendRestrictedRoleBackendMessage(0x43, body: Data("SELECT 1\0".utf8), to: &response) + appendRestrictedRoleBackendMessage(0x5a, body: Data([0x49]), to: &response) + return response +} + +private func appendRestrictedRoleBackendMessage( + _ tag: UInt8, + body: Data, + to response: inout Data +) { + response.append(tag) + appendRestrictedRoleUInt32(UInt32(body.count + 4), to: &response) + response.append(body) +} + +private func appendRestrictedRoleInt16(_ value: Int16, to data: inout Data) { + let bits = UInt16(bitPattern: value) + data.append(UInt8((bits >> 8) & 0xff)) + data.append(UInt8(bits & 0xff)) +} + +private func appendRestrictedRoleUInt32(_ value: UInt32, to data: inout Data) { + data.append(UInt8((value >> 24) & 0xff)) + data.append(UInt8((value >> 16) & 0xff)) + data.append(UInt8((value >> 8) & 0xff)) + data.append(UInt8(value & 0xff)) +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/WorkerCoreHandshakeRecoveryTests.swift b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/WorkerCoreHandshakeRecoveryTests.swift new file mode 100644 index 00000000..bf97b1f7 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/WorkerCoreHandshakeRecoveryTests.swift @@ -0,0 +1,140 @@ +import Foundation +import Oliphaunt +import OliphauntBrokerExtension +import OliphauntBrokerProtocol +import Testing + +@Test +func rejectedHelloDoesNotPoisonTheResidentWorkerProcess() async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("oliphaunt-worker-handshake-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + + let engine = HandshakeFailingOpenEngine() + let configuration = try BrokerWorkerConfiguration( + storage: BrokerExtensionStorage(location: .extensionPrivate, rootURL: root), + engine: engine, + liboliphauntVersion: "handshake-test-runtime", + cABIVersion: 42, + postgresMajorVersion: 18, + startupConfigurationDigest: "handshake-test-configuration", + runtimeVersionProvider: { "handshake-test-runtime" } + ) + let core = WorkerCore(configuration: configuration) + let originalEpoch = await core.epoch + let valid = BrokerHello( + expectedABI: 42, + expectedRuntimeVersion: "handshake-test-runtime", + startupConfigurationDigest: "handshake-test-configuration", + requestedCapabilities: [.protocolRaw, .protocolStream, .queryCancel] + ) + + var incompatibleProtocol = valid + incompatibleProtocol.minimumProtocolVersion = 99 + incompatibleProtocol.maximumProtocolVersion = 99 + await expectHandshakeRejection(.incompatibleProtocol) { + try await core.start(hello: incompatibleProtocol) + } + + var incompatibleABI = valid + incompatibleABI.expectedABI = 43 + await expectHandshakeRejection(.incompatibleABI) { + try await core.start(hello: incompatibleABI) + } + + var runtimeMismatch = valid + runtimeMismatch.expectedRuntimeVersion = "wrong-runtime" + await expectHandshakeRejection(.runtimeMismatch) { + try await core.start(hello: runtimeMismatch) + } + + var rootMismatch = valid + rootMismatch.rootID = "wrong-root" + await expectHandshakeRejection(.rootMismatch) { + try await core.start(hello: rootMismatch) + } + + var configurationMismatch = valid + configurationMismatch.startupConfigurationDigest = "wrong-configuration" + await expectHandshakeRejection(.invalidConfiguration) { + try await core.start(hello: configurationMismatch) + } + + let diagnostics = try await core.diagnostics(expectedEpoch: originalEpoch) + #expect(diagnostics.state == .created) + #expect(diagnostics.epoch == originalEpoch) + #expect(await engine.openCount() == 0) + + do { + _ = try await core.start(hello: valid) + Issue.record("the deliberately failing engine unexpectedly opened") + } catch let error as BrokerError { + #expect(error == .rejected(.rootOpen)) + #expect(!error.description.contains(extensionPrivatePathSentinel)) + #expect(!error.description.lowercased().contains("pgdata")) + } catch { + Issue.record("valid Hello did not reach the engine: \(error)") + } + #expect(await engine.openCount() == 1) +} + +private enum ExpectedHandshakeError { + case incompatibleProtocol + case incompatibleABI + case runtimeMismatch + case rootMismatch + case invalidConfiguration + + func matches(_ error: BrokerError) -> Bool { + switch (self, error) { + case (.incompatibleProtocol, .incompatibleProtocol), + (.incompatibleABI, .incompatibleABI), + (.runtimeMismatch, .runtimeMismatch), + (.rootMismatch, .rootMismatch), + (.invalidConfiguration, .invalidConfiguration): + true + default: + false + } + } +} + +private func expectHandshakeRejection( + _ expected: ExpectedHandshakeError, + operation: () async throws -> BrokerReady +) async { + do { + _ = try await operation() + Issue.record("mismatched Hello unexpectedly succeeded") + } catch let error as BrokerError { + #expect(expected.matches(error)) + } catch { + Issue.record("mismatched Hello returned an unstructured error: \(error)") + } +} + +private let extensionPrivatePathSentinel = + "/private/var/mobile/Containers/Data/PluginKitPlugin/worker-open-sentinel/pgdata" + +private struct HandshakeOpenError: Error, CustomStringConvertible { + var description: String { + "native direct open failed at \(extensionPrivatePathSentinel)" + } +} + +private actor HandshakeFailingOpenEngine: OliphauntEngine { + private var count = 0 + + func open(configuration: OliphauntConfiguration) async throws -> any OliphauntSession { + count += 1 + throw HandshakeOpenError() + } + + func restore(_ request: OliphauntRestoreRequest) async throws -> URL { + throw HandshakeOpenError() + } + + func openCount() -> Int { + count + } +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/WorkerCoreRecoveryTests.swift b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/WorkerCoreRecoveryTests.swift new file mode 100644 index 00000000..cc55c1b7 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerExtensionTests/WorkerCoreRecoveryTests.swift @@ -0,0 +1,707 @@ +import Foundation +import Oliphaunt +import OliphauntBrokerExtension +import OliphauntBrokerProtocol +import Testing + +@Test +func uploadInterruptionClosesSessionAndAllowsFreshEpoch() async throws { + let harness = try RecoveryHarness(behaviors: [.succeed, .succeed]) + defer { harness.removeStorage() } + + let firstReady = try await harness.core.start(hello: harness.hello) + let firstSession = try #require(await harness.engine.session(at: 0)) + let sink = RecordingFrameSink() + + _ = try await harness.core.handle( + try brokerFrame(.requestBegin, ready: firstReady, requestID: 1), + sink: sink + ) + _ = try await harness.core.handle( + try brokerFrame( + .requestBytes, + ready: firstReady, + requestID: 1, + payload: try OliphauntProtocol.simpleQuery("SELECT 1") + ), + sink: sink + ) + + await harness.core.interruptCurrentEpoch() + + let interrupted = try await harness.core.diagnostics(expectedEpoch: firstReady.epoch) + #expect(interrupted.state == .interrupted) + #expect(interrupted.activeRequestID == nil) + #expect(!interrupted.nativeDispatchStarted) + #expect(await firstSession.closeCallCount() == 1) + + let recovered = try await harness.core.start(hello: harness.hello) + #expect(recovered.epoch != firstReady.epoch) + #expect(await harness.engine.openCallCount() == 2) + #expect(try await harness.core.diagnostics(expectedEpoch: recovered.epoch).state == .ready) +} + +@Test +func nativeStreamFailureIsRecoverableWithoutReplay() async throws { + let harness = try RecoveryHarness(behaviors: [.fail, .succeed]) + defer { harness.removeStorage() } + + let firstReady = try await harness.core.start(hello: harness.hello) + let firstSession = try #require(await harness.engine.session(at: 0)) + let sink = RecordingFrameSink() + + _ = try await harness.core.handle( + try brokerFrame(.requestBegin, ready: firstReady, requestID: 7), + sink: sink + ) + _ = try await harness.core.handle( + try brokerFrame( + .requestBytes, + ready: firstReady, + requestID: 7, + payload: try OliphauntProtocol.simpleQuery("INSERT INTO t VALUES (1)") + ), + sink: sink + ) + do { + _ = try await harness.core.handle( + try brokerFrame(.requestEnd, ready: firstReady, requestID: 7), + sink: sink + ) + Issue.record("expected the native stream to fail") + } catch let error as RecoveryTestError { + #expect(error == .nativeStreamFailed) + } + + let interrupted = try await harness.core.diagnostics(expectedEpoch: firstReady.epoch) + #expect(interrupted.state == .interrupted) + #expect(interrupted.activeRequestID == nil) + #expect(await firstSession.streamCallCount() == 1) + #expect(await firstSession.closeCallCount() == 1) + #expect(sink.frameTypes() == [.outcomeUnknown]) + + let recovered = try await harness.core.start(hello: harness.hello) + #expect(recovered.epoch != firstReady.epoch) + #expect(await harness.engine.openCallCount() == 2) + #expect(await firstSession.streamCallCount() == 1) + #expect(try await harness.core.diagnostics(expectedEpoch: recovered.epoch).state == .ready) +} + +@Test +func runningCancellationSignalsNativeBeforeWorkerBookkeepingAndCompletesOnce() async throws { + let harness = try RecoveryHarness(behaviors: [.blockUntilReleasedAfterCancellation]) + defer { harness.removeStorage() } + + let ready = try await harness.core.start(hello: harness.hello) + let session = try #require(await harness.engine.session(at: 0)) + let sink = RecordingFrameSink() + let requestID = try BrokerRequestID(validating: 11) + + _ = try await harness.core.handle( + try brokerFrame(.requestBegin, ready: ready, requestID: requestID.rawValue), + sink: sink + ) + _ = try await harness.core.handle( + try brokerFrame( + .requestBytes, + ready: ready, + requestID: requestID.rawValue, + payload: try OliphauntProtocol.simpleQuery("SELECT pg_sleep(10)") + ), + sink: sink + ) + let execution = Task { + try await harness.core.handle( + try brokerFrame(.requestEnd, ready: ready, requestID: requestID.rawValue), + sink: sink + ) + } + await session.waitUntilStreamIsBlocked() + + let direct = try await harness.core.cancellationController.requestCancellation( + epoch: ready.epoch, + requestID: requestID + ) + #expect(direct == .signalSent) + #expect(await session.cancelCallCount() == 1) + + let bookkeeping = Task { + try await harness.core.cancelRequest( + epoch: ready.epoch, + requestID: requestID + ) + } + #expect(try await bookkeeping.value == .nativeSignal(.alreadyRequested)) + #expect(await session.cancelCallCount() == 1) + + await session.releaseBlockedStream() + _ = try await execution.value + + let frameTypes = sink.frameTypes() + #expect(frameTypes.dropLast(2).allSatisfy { $0 == .responseBytes }) + #expect(Array(frameTypes.suffix(2)) == [.cancelObserved, .completed]) + #expect(frameTypes.count { $0 == .cancelObserved } == 1) + #expect(frameTypes.count { $0 == .completed } == 1) + #expect( + try await harness.core.diagnostics(expectedEpoch: ready.epoch).activeRequestID + == nil + ) +} + +@Test +func workerSanitizesBackendErrorsBeforeTheyReachTheFrameSink() async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("oliphaunt-private-path-\(UUID().uuidString)", isDirectory: true) + let response = backendPathErrorResponse(path: root.path) + let harness = try RecoveryHarness( + behaviors: [.chunks(response.map { Data([$0]) })], + rootURL: root + ) + defer { harness.removeStorage() } + + let ready = try await harness.core.start(hello: harness.hello) + let sink = RecordingFrameSink() + _ = try await harness.core.handle( + try brokerFrame(.requestBegin, ready: ready, requestID: 19), + sink: sink + ) + _ = try await harness.core.handle( + try brokerFrame( + .requestBytes, + ready: ready, + requestID: 19, + payload: try OliphauntProtocol.simpleQuery("SELECT path_error_probe()") + ), + sink: sink + ) + _ = try await harness.core.handle( + try brokerFrame(.requestEnd, ready: ready, requestID: 19), + sink: sink + ) + + let visibleResponse = sink.responsePayload() + #expect(visibleResponse.range(of: Data(root.path.utf8)) == nil) + #expect(visibleResponse.range(of: Data("[redacted]".utf8)) != nil) + #expect(sink.frameTypes().last == .completed) + #expect( + try await harness.core.diagnostics(expectedEpoch: ready.epoch).state + == .ready + ) +} + +@Test +func interruptedStartAttemptCannotPublishOrOverlapAReplacementSession() async throws { + let harness = try RecoveryHarness( + behaviors: [.succeed, .succeed], + blockFirstOpen: true + ) + defer { harness.removeStorage() } + + let initialEpoch = await harness.core.epoch + let opening = Task { + try await harness.core.start(hello: harness.hello) + } + await harness.engine.waitUntilFirstOpenIsBlocked() + let invalidatedSession = try #require(await harness.engine.session(at: 0)) + + await harness.core.interruptCurrentEpoch() + await expectWorkerInterrupted(epoch: initialEpoch) { + try await harness.core.start(hello: harness.hello) + } + + await harness.engine.releaseFirstOpen() + await expectWorkerInterrupted(epoch: initialEpoch) { + try await opening.value + } + + #expect(await invalidatedSession.closeCallCount() == 1) + #expect(await harness.engine.openCallCount() == 1) + let interrupted = try await harness.core.diagnostics(expectedEpoch: initialEpoch) + #expect(interrupted.state == .interrupted) + + let recovered = try await harness.core.start(hello: harness.hello) + #expect(recovered.epoch != initialEpoch) + #expect(await harness.engine.openCallCount() == 2) + #expect(try await harness.core.diagnostics(expectedEpoch: recovered.epoch).state == .ready) +} + +@Test +func staleControlGenerationCannotActOnRecoveredWorker() async throws { + let harness = try RecoveryHarness(behaviors: [.succeed, .succeed]) + defer { harness.removeStorage() } + + let firstReady = try await harness.core.start(hello: harness.hello) + await harness.core.interruptCurrentEpoch() + let recovered = try await harness.core.start(hello: harness.hello) + let recoveredSession = try #require(await harness.engine.session(at: 1)) + + await expectWorkerInterrupted(epoch: firstReady.epoch) { + try await harness.core.checkpoint(expectedEpoch: firstReady.epoch) + } + await expectWorkerInterrupted(epoch: firstReady.epoch) { + try await harness.core.prepareForBackground( + expectedEpoch: firstReady.epoch, + deadline: Date().addingTimeInterval(1) + ) + } + await expectWorkerInterrupted(epoch: firstReady.epoch) { + try await harness.core.resumeFromBackground(expectedEpoch: firstReady.epoch) + } + await expectWorkerInterrupted(epoch: firstReady.epoch) { + try await harness.core.detach(expectedEpoch: firstReady.epoch) + } + await expectWorkerInterrupted(epoch: firstReady.epoch) { + try await harness.core.diagnostics(expectedEpoch: firstReady.epoch) + } + #if DEBUG + await expectWorkerInterrupted(epoch: firstReady.epoch) { + try await harness.core.injectFault( + .duringBackup, + expectedEpoch: firstReady.epoch + ) + } + #endif + + // Startup performs the restricted-role bootstrap and then its health check. + #expect(await recoveredSession.rawCallCount() == 2) + #expect(await recoveredSession.closeCallCount() == 0) + let diagnostics = try await harness.core.diagnostics(expectedEpoch: recovered.epoch) + #expect(diagnostics.state == .ready) + #expect(diagnostics.epoch == recovered.epoch) +} + +@Test +func backgroundCheckpointDeadlineInterruptsAndGatesFreshEpochUntilClose() async throws { + let harness = try RecoveryHarness( + behaviors: [.succeed, .succeed], + blockFirstControlRaw: true + ) + defer { harness.removeStorage() } + + let firstReady = try await harness.core.start(hello: harness.hello) + let firstSession = try #require(await harness.engine.session(at: 0)) + let started = ContinuousClock.now + do { + _ = try await harness.core.prepareForBackground( + expectedEpoch: firstReady.epoch, + deadline: Date().addingTimeInterval(0.75) + ) + Issue.record("expected the blocked checkpoint to invalidate its epoch") + } catch let error as BrokerError { + #expect(error == .workerInterrupted(epoch: firstReady.epoch)) + } + #expect(started.duration(to: .now) < .seconds(0.75)) + #expect( + try await harness.core.diagnostics(expectedEpoch: firstReady.epoch).state + == .interrupted + ) + + for _ in 0..<100 where await firstSession.closeCallCount() == 0 { + try await Task.sleep(for: .milliseconds(10)) + } + #expect(await firstSession.cancelCallCount() == 1) + #expect(await firstSession.closeCallCount() == 1) + + let recovered = try await harness.core.start(hello: harness.hello) + #expect(recovered.epoch != firstReady.epoch) + #expect(await harness.engine.openCallCount() == 2) +} + +private struct RecoveryHarness { + let rootURL: URL + let engine: RecoveryTestEngine + let core: WorkerCore + let hello: BrokerHello + + init( + behaviors: [RecoveryStreamBehavior], + rootURL: URL? = nil, + blockFirstOpen: Bool = false, + blockFirstControlRaw: Bool = false + ) throws { + self.rootURL = + rootURL + ?? FileManager.default.temporaryDirectory + .appendingPathComponent( + "oliphaunt-worker-recovery-\(UUID().uuidString)", + isDirectory: true + ) + engine = RecoveryTestEngine( + behaviors: behaviors, + blockFirstOpen: blockFirstOpen, + blockFirstControlRaw: blockFirstControlRaw + ) + let storage = try BrokerExtensionStorage( + location: .extensionPrivate, + rootURL: self.rootURL + ) + let configuration = try BrokerWorkerConfiguration( + storage: storage, + engine: engine, + liboliphauntVersion: "recovery-test-runtime", + cABIVersion: 42, + postgresMajorVersion: 18, + startupConfigurationDigest: "recovery-test-configuration", + runtimeVersionProvider: { "recovery-test-runtime" } + ) + core = WorkerCore(configuration: configuration) + hello = BrokerHello( + expectedABI: 42, + expectedRuntimeVersion: "recovery-test-runtime", + startupConfigurationDigest: "recovery-test-configuration", + requestedCapabilities: [.protocolRaw, .protocolStream, .queryCancel] + ) + } + + func removeStorage() { + try? FileManager.default.removeItem(at: rootURL) + } +} + +private enum RecoveryStreamBehavior: Sendable { + case succeed + case fail + case chunks([Data]) + case blockUntilReleasedAfterCancellation +} + +private enum RecoveryTestError: Error, Equatable { + case nativeStreamFailed +} + +private actor RecoveryTestEngine: OliphauntEngine { + private let behaviors: [RecoveryStreamBehavior] + private let blockFirstOpen: Bool + private let blockFirstControlRaw: Bool + private var sessions: [RecoveryTestSession] = [] + private var firstOpenIsBlocked = false + private var firstOpenWasReleased = false + private var firstOpenStartedWaiters: [CheckedContinuation] = [] + private var firstOpenReleaseContinuation: CheckedContinuation? + + init( + behaviors: [RecoveryStreamBehavior], + blockFirstOpen: Bool, + blockFirstControlRaw: Bool + ) { + self.behaviors = behaviors + self.blockFirstOpen = blockFirstOpen + self.blockFirstControlRaw = blockFirstControlRaw + } + + func open(configuration: OliphauntConfiguration) async throws -> any OliphauntSession { + let root = try #require(configuration.root) + let pgVersion = + root + .appendingPathComponent("pgdata", isDirectory: true) + .appendingPathComponent("PG_VERSION", isDirectory: false) + try Data("18\n".utf8).write(to: pgVersion, options: .atomic) + + let behavior = sessions.count < behaviors.count ? behaviors[sessions.count] : .succeed + let session = RecoveryTestSession( + streamBehavior: behavior, + blockControlRaw: blockFirstControlRaw && sessions.isEmpty + ) + sessions.append(session) + if blockFirstOpen, sessions.count == 1 { + firstOpenIsBlocked = true + let waiters = firstOpenStartedWaiters + firstOpenStartedWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + await withCheckedContinuation { continuation in + if firstOpenWasReleased { + continuation.resume() + } else { + firstOpenReleaseContinuation = continuation + } + } + } + return session + } + + func restore(_ request: OliphauntRestoreRequest) async throws -> URL { + request.root + } + + func openCallCount() -> Int { + sessions.count + } + + func session(at index: Int) -> RecoveryTestSession? { + sessions.indices.contains(index) ? sessions[index] : nil + } + + func waitUntilFirstOpenIsBlocked() async { + if firstOpenIsBlocked { + return + } + await withCheckedContinuation { continuation in + firstOpenStartedWaiters.append(continuation) + } + } + + func releaseFirstOpen() { + firstOpenWasReleased = true + firstOpenReleaseContinuation?.resume() + firstOpenReleaseContinuation = nil + } +} + +private actor RecoveryTestSession: OliphauntSession { + private let streamBehavior: RecoveryStreamBehavior + private var rawCalls = 0 + private var streamCalls = 0 + private var closeCalls = 0 + private var cancelCalls = 0 + private let blockControlRaw: Bool + private var controlRawWasReleased = false + private var controlRawContinuation: CheckedContinuation? + private var streamIsBlocked = false + private var blockedStreamWasReleased = false + private var streamStartedWaiters: [CheckedContinuation] = [] + private var blockedStreamContinuation: CheckedContinuation? + + init(streamBehavior: RecoveryStreamBehavior, blockControlRaw: Bool) { + self.streamBehavior = streamBehavior + self.blockControlRaw = blockControlRaw + } + + func capabilities() async -> OliphauntCapabilities { + OliphauntCapabilities( + mode: .nativeDirect, + processIsolated: false, + independentSessions: false, + maxClientSessions: 1 + ) + } + + func execProtocolRaw(_ bytes: Data) async throws -> Data { + rawCalls += 1 + // Startup call 1 establishes the role boundary and call 2 performs the + // health check. Block the first later lifecycle control statement. + if blockControlRaw, rawCalls > 2, !controlRawWasReleased { + await withCheckedContinuation { continuation in + if controlRawWasReleased { + continuation.resume() + } else { + controlRawContinuation = continuation + } + } + } + return backendSelectOneResponse() + } + + func execProtocolStream( + _ bytes: Data, + onChunk: @escaping @Sendable (Data) throws -> Void + ) async throws { + streamCalls += 1 + switch streamBehavior { + case .succeed: + try onChunk(backendSelectOneResponse()) + case .fail: + throw RecoveryTestError.nativeStreamFailed + case .chunks(let chunks): + for chunk in chunks { + try onChunk(chunk) + } + case .blockUntilReleasedAfterCancellation: + streamIsBlocked = true + let waiters = streamStartedWaiters + streamStartedWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + await withCheckedContinuation { continuation in + if blockedStreamWasReleased { + continuation.resume() + } else { + blockedStreamContinuation = continuation + } + } + try onChunk(backendQueryCanceledResponse()) + } + } + + func backup(_ request: OliphauntBackupRequest) async throws -> OliphauntBackupArtifact { + OliphauntBackupArtifact(format: request.format, bytes: Data()) + } + + func cancel() async throws { + cancelCalls += 1 + controlRawWasReleased = true + controlRawContinuation?.resume() + controlRawContinuation = nil + } + + func close() async throws { + closeCalls += 1 + } + + func streamCallCount() -> Int { + streamCalls + } + + func rawCallCount() -> Int { + rawCalls + } + + func closeCallCount() -> Int { + closeCalls + } + + func cancelCallCount() -> Int { + cancelCalls + } + + func waitUntilStreamIsBlocked() async { + if streamIsBlocked { + return + } + await withCheckedContinuation { continuation in + streamStartedWaiters.append(continuation) + } + } + + func releaseBlockedStream() { + blockedStreamWasReleased = true + blockedStreamContinuation?.resume() + blockedStreamContinuation = nil + } +} + +private func expectWorkerInterrupted( + epoch: BrokerEpoch, + performing operation: () async throws -> Value +) async { + do { + _ = try await operation() + Issue.record("expected worker interruption for stale epoch \(epoch)") + } catch let error as BrokerError { + #expect(error == .workerInterrupted(epoch: epoch)) + } catch { + Issue.record("expected BrokerError.workerInterrupted, got \(error)") + } +} + +private final class RecordingFrameSink: BrokerFrameSink, @unchecked Sendable { + private let lock = NSLock() + private var frames: [BrokerFrame] = [] + + func send(_ frame: BrokerFrame) throws { + lock.lock() + defer { lock.unlock() } + frames.append(frame) + } + + func frameTypes() -> [BrokerFrameType] { + lock.lock() + defer { lock.unlock() } + return frames.map(\.header.frameType) + } + + func responsePayload() -> Data { + lock.lock() + defer { lock.unlock() } + return + frames + .filter { $0.header.frameType == .responseBytes } + .reduce(into: Data()) { $0.append($1.payload) } + } +} + +private func brokerFrame( + _ type: BrokerFrameType, + ready: BrokerReady, + requestID: UInt64, + payload: Data = Data() +) throws -> BrokerFrame { + try BrokerFrame( + protocolVersion: ready.selectedProtocolVersion, + frameType: type, + epoch: ready.epoch, + requestID: requestID, + payload: payload + ) +} + +private func backendSelectOneResponse() -> Data { + var response = Data() + + var rowDescription = Data() + appendInt16(1, to: &rowDescription) + rowDescription.append(Data("broker_health".utf8)) + rowDescription.append(0) + appendUInt32(0, to: &rowDescription) + appendInt16(0, to: &rowDescription) + appendUInt32(23, to: &rowDescription) + appendInt16(4, to: &rowDescription) + appendUInt32(UInt32.max, to: &rowDescription) + appendInt16(0, to: &rowDescription) + appendBackendMessage(0x54, body: rowDescription, to: &response) + + var row = Data() + appendInt16(1, to: &row) + appendUInt32(1, to: &row) + row.append(Data("1".utf8)) + appendBackendMessage(0x44, body: row, to: &response) + + appendBackendMessage(0x43, body: Data("SELECT 1\0".utf8), to: &response) + appendBackendMessage(0x5a, body: Data([0x49]), to: &response) + return response +} + +private func backendPathErrorResponse(path: String) -> Data { + var response = Data() + var error = Data() + error.append(0x53) + error.append(Data("ERROR\0".utf8)) + error.append(0x56) + error.append(Data("ERROR\0".utf8)) + error.append(0x43) + error.append(Data("F0000\0".utf8)) + error.append(0x4d) + error.append(Data("could not open \(path)/runtime-cache/private.stop\0".utf8)) + error.append(0) + appendBackendMessage(0x45, body: error, to: &response) + appendBackendMessage(0x5a, body: Data([0x49]), to: &response) + return response +} + +private func backendQueryCanceledResponse() -> Data { + var response = Data() + var error = Data() + error.append(0x53) + error.append(Data("ERROR\0".utf8)) + error.append(0x56) + error.append(Data("ERROR\0".utf8)) + error.append(0x43) + error.append(Data("57014\0".utf8)) + error.append(0x4d) + error.append(Data("canceling statement due to user request\0".utf8)) + error.append(0) + appendBackendMessage(0x45, body: error, to: &response) + appendBackendMessage(0x5a, body: Data([0x49]), to: &response) + return response +} + +private func appendBackendMessage(_ tag: UInt8, body: Data, to response: inout Data) { + response.append(tag) + appendUInt32(UInt32(body.count + 4), to: &response) + response.append(body) +} + +private func appendInt16(_ value: Int16, to data: inout Data) { + let bits = UInt16(bitPattern: value) + data.append(UInt8((bits >> 8) & 0xff)) + data.append(UInt8(bits & 0xff)) +} + +private func appendUInt32(_ value: UInt32, to data: inout Data) { + data.append(UInt8((value >> 24) & 0xff)) + data.append(UInt8((value >> 16) & 0xff)) + data.append(UInt8((value >> 8) & 0xff)) + data.append(UInt8(value & 0xff)) +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/BrokerFrameTests.swift b/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/BrokerFrameTests.swift new file mode 100644 index 00000000..1a8b99f3 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/BrokerFrameTests.swift @@ -0,0 +1,384 @@ +import Foundation +import OliphauntBrokerProtocol +import Testing + +@Test +func frameHeaderHasStableFortyByteNetworkOrderEncoding() throws { + let payload = Data([0xaa, 0xbb, 0xcc]) + let frame = try BrokerFrame( + frameType: .requestBytes, + epoch: brokerTestEpoch, + requestID: 0x0102_0304_0506_0708, + payload: payload + ) + + let expectedHeader = Data([ + 0x4f, 0x4c, 0x50, 0x42, + 0x00, 0x01, + 0x00, 0x28, + 0x02, + 0x00, + 0x00, 0x00, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x00, 0x00, 0x00, 0x03, + ]) + + let encoded = try frame.encoded() + #expect(OliphauntBrokerProtocol.headerLength == 40) + #expect(encoded.count == 43) + #expect(Data(encoded.prefix(40)) == expectedHeader) + #expect(Data(encoded.suffix(3)) == payload) + + let decoded = try BrokerFrameHeader.decode(expectedHeader, expectedEpoch: brokerTestEpoch) + #expect(decoded == frame.header) +} + +@Test +func everyFrameTypeRoundTripsAndEnforcesItsRequestIDDomain() throws { + for frameType in BrokerFrameType.allCases { + let validRequestID: UInt64 = frameType.requiresRequestID ? 91 : 0 + let frame = try BrokerFrame( + frameType: frameType, + epoch: brokerTestEpoch, + requestID: validRequestID, + payload: Data([frameType.rawValue]) + ) + var decoder = BrokerFrameDecoder(expectedEpoch: brokerTestEpoch) + #expect(try decoder.append(frame.encoded()) == [frame]) + #expect(try decoder.finish().isEmpty) + + let invalidRequestID: UInt64 = frameType.requiresRequestID ? 0 : 91 + expectProtocolError( + .invalidRequestIDForFrame(frameType: frameType, requestID: invalidRequestID) + ) { + try BrokerFrame( + frameType: frameType, + epoch: brokerTestEpoch, + requestID: invalidRequestID + ) + } + } +} + +@Test +func decoderAcceptsEveryFragmentBoundary() throws { + let frame = try BrokerFrame( + frameType: .responseBytes, + epoch: brokerTestEpoch, + requestID: 7, + payload: Data((0..<73).map(UInt8.init)) + ) + let encoded = try frame.encoded() + + for split in 0...encoded.count { + var decoder = BrokerFrameDecoder(expectedEpoch: brokerTestEpoch) + var decoded: [BrokerFrame] = [] + decoded += try decoder.append(Data(encoded[.. BrokerHello { + BrokerHello( + minimumProtocolVersion: minimumVersion, + maximumProtocolVersion: maximumVersion, + expectedABI: expectedABI, + expectedRuntimeVersion: expectedRuntimeVersion, + rootID: rootID, + startupConfigurationDigest: digest, + requestedCapabilities: [.processIsolated, .protocolRaw, .protocolStream] + ) +} + +@Test +func handshakeNegotiatesHighestMutuallySupportedVersion() throws { + #expect(OliphauntBrokerProtocol.minimumVersion == 1) + #expect(OliphauntBrokerProtocol.maximumVersion == 1) + #expect(try BrokerHandshake.negotiateVersion(hello()) == 1) + #expect( + try BrokerHandshake.negotiateVersion( + hello(minimumVersion: 0, maximumVersion: 2) + ) == 1 + ) + + expectBrokerError(.incompatibleProtocol(minimum: 0, maximum: 0)) { + try BrokerHandshake.negotiateVersion(hello(minimumVersion: 0, maximumVersion: 0)) + } + expectBrokerError(.incompatibleProtocol(minimum: 2, maximum: 3)) { + try BrokerHandshake.negotiateVersion(hello(minimumVersion: 2, maximumVersion: 3)) + } + expectBrokerError( + .invalidConfiguration("minimum protocol version exceeds maximum") + ) { + try BrokerHandshake.negotiateVersion(hello(minimumVersion: 2, maximumVersion: 1)) + } +} + +@Test +func handshakeValidatesABIIdentityRootAndStartupConfiguration() throws { + #expect( + try BrokerHandshake.validate( + hello(), + actualABI: 6, + actualRuntimeVersion: "0.4.0", + residentRootID: nil, + startupConfigurationDigest: "configuration-sha256" + ) == 1 + ) + #expect( + try BrokerHandshake.validate( + hello(expectedRuntimeVersion: nil), + actualABI: 6, + actualRuntimeVersion: "newer-runtime", + residentRootID: OliphauntBrokerProtocol.canonicalRootID, + startupConfigurationDigest: "configuration-sha256" + ) == 1 + ) + + expectBrokerError(.incompatibleABI(expected: 7, actual: 6)) { + try BrokerHandshake.validate( + hello(expectedABI: 7), + actualABI: 6, + actualRuntimeVersion: "0.4.0", + residentRootID: nil, + startupConfigurationDigest: "configuration-sha256" + ) + } + expectBrokerError(.runtimeMismatch(expected: "0.4.0", actual: "0.5.0")) { + try BrokerHandshake.validate( + hello(), + actualABI: 6, + actualRuntimeVersion: "0.5.0", + residentRootID: nil, + startupConfigurationDigest: "configuration-sha256" + ) + } + expectBrokerError(.rootMismatch(expected: "default", actual: "another-root")) { + try BrokerHandshake.validate( + hello(rootID: "another-root"), + actualABI: 6, + actualRuntimeVersion: "0.4.0", + residentRootID: nil, + startupConfigurationDigest: "configuration-sha256" + ) + } + expectBrokerError(.rootMismatch(expected: "resident-root", actual: "default")) { + try BrokerHandshake.validate( + hello(), + actualABI: 6, + actualRuntimeVersion: "0.4.0", + residentRootID: "resident-root", + startupConfigurationDigest: "configuration-sha256" + ) + } + expectBrokerError(.invalidConfiguration("startup-configuration digest mismatch")) { + try BrokerHandshake.validate( + hello(digest: "wrong"), + actualABI: 6, + actualRuntimeVersion: "0.4.0", + residentRootID: nil, + startupConfigurationDigest: "configuration-sha256" + ) + } +} + +@Test +func defaultCapabilitiesEncodeOnlyConservativeIOSBrokerClaims() throws { + let capabilities = BrokerCapabilities() + let expectedEnabled: Set = [ + .processIsolated, + .crashRestartable, + .sameRootLogicalReopen, + .protocolRaw, + .protocolStream, + .queryCancel, + ] + #expect(capabilities.enabled == expectedEnabled) + + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let encoded = try encoder.encode(capabilities) + let json = try #require(String(data: encoded, encoding: .utf8)) + #expect( + json + == "{\"backgroundContinuable\":false,\"backupRestore\":false,\"crashRestartable\":true,\"hangRestartable\":false,\"implementation\":\"iosExtensionBroker\",\"independentSessions\":false,\"maxClientSessions\":1,\"minimumOS\":\"iOS 26\",\"mode\":\"nativeBroker\",\"multiRoot\":false,\"processIsolated\":true,\"protocolRaw\":true,\"protocolStream\":true,\"queryCancel\":true,\"requiresAppGroup\":false,\"rootSwitchable\":false,\"sameRootLogicalReopen\":true,\"serverMode\":false,\"streamingRequestInput\":false}" + ) + + let decoded = try JSONDecoder().decode(BrokerCapabilities.self, from: encoded) + #expect(decoded == capabilities) + #expect(decoded.mode == "nativeBroker") + #expect(decoded.implementation == "iosExtensionBroker") + #expect(decoded.minimumOS == "iOS 26") + #expect(decoded.processIsolated) + #expect(decoded.crashRestartable) + #expect(!decoded.hangRestartable) + #expect(decoded.sameRootLogicalReopen) + #expect(!decoded.rootSwitchable) + #expect(!decoded.multiRoot) + #expect(!decoded.independentSessions) + #expect(decoded.maxClientSessions == 1) + #expect(!decoded.backgroundContinuable) + #expect(!decoded.requiresAppGroup) + #expect(decoded.protocolRaw) + #expect(decoded.protocolStream) + #expect(!decoded.streamingRequestInput) + #expect(decoded.queryCancel) + #expect(!decoded.backupRestore) + #expect(decoded.connectionString == nil) + #expect(!decoded.serverMode) +} + +@Test +func readyAndStructuredOutcomeUnknownSurviveCodableRoundTrips() throws { + let capabilities = BrokerCapabilities() + let ready = BrokerReady( + selectedProtocolVersion: 1, + epoch: brokerTestEpoch, + extensionPID: 4242, + runtimeVersion: "0.4.0", + abiVersion: 6, + postgresMajorVersion: 18, + rootManifestDigest: "manifest-sha256", + actualCapabilities: capabilities, + actualRuntimeConfiguration: BrokerRuntimeConfiguration( + rootID: "default", + startupConfigurationDigest: "configuration-sha256", + selectedExtensions: ["pg_trgm", "vector"] + ) + ) + let encodedReady = try JSONEncoder().encode(ready) + #expect(try JSONDecoder().decode(BrokerReady.self, from: encodedReady) == ready) + + let requestID = try BrokerRequestID(validating: 99) + let error = BrokerError.outcomeUnknown(epoch: brokerTestEpoch, requestID: requestID) + let encodedError = try JSONEncoder().encode(error) + #expect(try JSONDecoder().decode(BrokerError.self, from: encodedError) == error) + #expect(error.description.contains(brokerTestEpoch.description)) + #expect(error.description.contains("99")) +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/BrokerStateMachineTests.swift b/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/BrokerStateMachineTests.swift new file mode 100644 index 00000000..3a5982ca --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/BrokerStateMachineTests.swift @@ -0,0 +1,248 @@ +import Foundation +import OliphauntBrokerProtocol +import Testing + +@Test +func frontendRequestAssemblerAcceptsFragmentedAndCoalescedMessages() throws { + let query = postgresFrontendMessage( + type: Character("Q").asciiValue!, + body: Array("SELECT 1\0".utf8) + ) + let sync = postgresFrontendMessage(type: Character("S").asciiValue!, body: []) + let request = query + sync + + for split in 0...request.count { + var assembler = BrokerFrontendRequestAssembler(maximumRequestBytes: request.count) + try assembler.append(Data(request[.. BrokerRequestLifecycle { + var lifecycle = BrokerRequestLifecycle( + epoch: brokerTestEpoch, + requestID: try BrokerRequestID(validating: id) + ) + try lifecycle.beginReceiving() + try lifecycle.finishReceiving() + try lifecycle.beginNativeDispatch() + return lifecycle + } + + var cancelFirst = try runningLifecycle(id: 1) + let acceptedCancellation = cancelFirst.requestCancellation() + #expect(acceptedCancellation) + #expect(cancelFirst.state == .cancelRequested) + let acceptedSecondCancellation = cancelFirst.requestCancellation() + #expect(!acceptedSecondCancellation) + let establishedCancellation = cancelFirst.establishTerminal(.canceled) + #expect(establishedCancellation) + let establishedCompletionAfterCancellation = cancelFirst.establishTerminal(.completed) + #expect(!establishedCompletionAfterCancellation) + #expect(cancelFirst.state == .terminal(.canceled)) + + var completionFirst = try runningLifecycle(id: 2) + let establishedCompletion = completionFirst.establishTerminal(.completed) + #expect(establishedCompletion) + let acceptedLateCancellation = completionFirst.requestCancellation() + #expect(!acceptedLateCancellation) + let establishedLateCancellation = completionFirst.establishTerminal(.canceled) + #expect(!establishedLateCancellation) + #expect(completionFirst.state == .terminal(.completed)) + + for (index, terminal) in [ + BrokerTerminalResult.completed, + .rejected(.queueClosed), + .outcomeUnknown, + .canceled, + .notStarted, + ].enumerated() { + var lifecycle = try runningLifecycle(id: UInt64(index + 10)) + let establishedFirst = lifecycle.establishTerminal(terminal) + #expect(establishedFirst) + let establishedSecond = lifecycle.establishTerminal(.completed) + #expect(!establishedSecond) + let acceptedCancellationAfterTerminal = lifecycle.requestCancellation() + #expect(!acceptedCancellationAfterTerminal) + #expect(lifecycle.state == .terminal(terminal)) + } +} + +@Test +func requestIDSequenceIsMonotonicAndFailsClosedAtExhaustion() throws { + expectProtocolError(.invalidRequestID(0)) { + try BrokerRequestID(validating: 0) + } + expectProtocolError(.invalidRequestID(0)) { + try BrokerRequestIDSequence(startingAt: 0) + } + + var ordinary = try BrokerRequestIDSequence(startingAt: 41) + let firstOrdinary = try ordinary.next() + let secondOrdinary = try ordinary.next() + #expect(firstOrdinary.rawValue == 41) + #expect(secondOrdinary.rawValue == 42) + + var exhaustion = try BrokerRequestIDSequence(startingAt: UInt64.max) + let finalID = try exhaustion.next() + #expect(finalID.rawValue == UInt64.max) + expectProtocolError(.requestIDSpaceExhausted) { + try exhaustion.next() + } + expectProtocolError(.requestIDSpaceExhausted) { + try exhaustion.next() + } +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/TestSupport.swift b/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/TestSupport.swift new file mode 100644 index 00000000..a1f57c6e --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerProtocolTests/TestSupport.swift @@ -0,0 +1,69 @@ +import Foundation +import OliphauntBrokerProtocol +import Testing + +let brokerTestEpoch = BrokerEpoch( + UUID(uuidString: "00112233-4455-6677-8899-AABBCCDDEEFF")! +) + +let otherBrokerTestEpoch = BrokerEpoch( + UUID(uuidString: "FFEEDDCC-BBAA-9988-7766-554433221100")! +) + +func expectProtocolError( + _ expected: BrokerProtocolError, + performing operation: () throws -> T +) { + do { + _ = try operation() + Issue.record("expected protocol error \(expected), but the operation succeeded") + } catch let actual as BrokerProtocolError { + #expect(actual == expected) + } catch { + Issue.record("expected protocol error \(expected), got \(error)") + } +} + +func expectBrokerError( + _ expected: BrokerError, + performing operation: () throws -> T +) { + do { + _ = try operation() + Issue.record("expected broker error \(expected), but the operation succeeded") + } catch let actual as BrokerError { + #expect(actual == expected) + } catch { + Issue.record("expected broker error \(expected), got \(error)") + } +} + +func postgresFrontendMessage(type: UInt8, body: [UInt8]) -> Data { + let length = UInt32(body.count + 4) + var result = Data([type]) + result.append(UInt8((length >> 24) & 0xff)) + result.append(UInt8((length >> 16) & 0xff)) + result.append(UInt8((length >> 8) & 0xff)) + result.append(UInt8(length & 0xff)) + result.append(contentsOf: body) + return result +} + +func writeNetworkUInt16(_ value: UInt16, to bytes: inout Data, at offset: Int) { + bytes[offset] = UInt8((value >> 8) & 0xff) + bytes[offset + 1] = UInt8(value & 0xff) +} + +func writeNetworkUInt32(_ value: UInt32, to bytes: inout Data, at offset: Int) { + bytes[offset] = UInt8((value >> 24) & 0xff) + bytes[offset + 1] = UInt8((value >> 16) & 0xff) + bytes[offset + 2] = UInt8((value >> 8) & 0xff) + bytes[offset + 3] = UInt8(value & 0xff) +} + +func writeNetworkUInt64(_ value: UInt64, to bytes: inout Data, at offset: Int) { + for byteOffset in 0..<8 { + let shift = UInt64(56 - byteOffset * 8) + bytes[offset + byteOffset] = UInt8((value >> shift) & 0xff) + } +} diff --git a/src/sdks/swift/Tests/OliphauntBrokerXPCTests/IOSBrokerXPCTests.swift b/src/sdks/swift/Tests/OliphauntBrokerXPCTests/IOSBrokerXPCTests.swift new file mode 100644 index 00000000..cfdf48f9 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntBrokerXPCTests/IOSBrokerXPCTests.swift @@ -0,0 +1,168 @@ +import Darwin +import Foundation +import OliphauntBrokerProtocol +import OliphauntBrokerXPC +import Testing +import XPC + +@available(iOS 26.0, macOS 26.0, *) +@Test +func xpcHandshakePreservesStructuredBrokerErrors() throws { + let expectedErrors: [BrokerError] = [ + .incompatibleProtocol(minimum: 2, maximum: 3), + .incompatibleABI(expected: 6, actual: 7), + .runtimeMismatch(expected: "runtime-expected", actual: "runtime-actual"), + .rootMismatch(expected: "default", actual: "other-logical-root"), + .invalidConfiguration("minimum protocol version exceeds maximum"), + .invalidConfiguration("startup-configuration digest mismatch"), + .rejected(.unsupportedCapability(.backupRestore)), + .rejected(.invalidRequest("a broker data channel is already active")), + ] + for expected in expectedErrors { + let reply = try IOSBrokerXPC.makeError(expected) + #expect(try IOSBrokerXPC.decodeError(reply) == expected) + do { + _ = try IOSBrokerXPC.decodeReady(reply) + Issue.record("a rejected handshake must throw its structured broker error") + } catch let error as BrokerError { + #expect(error == expected) + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func xpcExtensionBoundaryRedactsPrivatePathsFromErrorsAndFallback() throws { + let sentinelRoot = "/private/var/mobile/Containers/Data/PluginKitPlugin/path-sentinel-root" + let sentinelPGDATA = "\(sentinelRoot)/pgdata" + let cases: [(error: any Error, expected: BrokerError)] = [ + ( + BrokerError.invalidConfiguration("cannot open \(sentinelPGDATA)"), + .invalidConfiguration("extension configuration was rejected") + ), + ( + BrokerError.protocolViolation("native startup failed at \(sentinelPGDATA)"), + .protocolViolation("extension control message was invalid") + ), + ( + BrokerError.rejected(.invalidRequest("invalid root \(sentinelRoot)")), + .rejected(.invalidRequest("extension rejected the request")) + ), + ( + NSError( + domain: NSCocoaErrorDomain, + code: CocoaError.fileReadNoSuchFile.rawValue, + userInfo: [NSFilePathErrorKey: sentinelPGDATA] + ), + .brokerUnavailable + ), + ] + + for testCase in cases { + let reply = try IOSBrokerXPC.makeError(testCase.error) + #expect(try IOSBrokerXPC.decodeError(reply) == testCase.expected) + let encoded: String? = reply[BrokerControlKey.error] + let reason: String? = reply[BrokerControlKey.reason] + let fallback = IOSBrokerXPC.extensionBoundaryError(testCase.error).description + #expect(reason == testCase.expected.description) + #expect(fallback == testCase.expected.description) + for value in [encoded, reason, fallback].compactMap({ $0 }) { + #expect(!value.contains(sentinelRoot)) + #expect(!value.contains("path-sentinel-root")) + #expect(!value.contains("/private/var/mobile")) + #expect(!value.lowercased().contains("pgdata")) + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func xpcFileDescriptorBoxingDuplicatesOwnership() throws { + var descriptors: [Int32] = [-1, -1] + #expect(pipe(&descriptors) == 0) + defer { Darwin.close(descriptors[1]) } + + let sender = try IOSBrokerOwnedFileDescriptor( + takingOwnershipOf: descriptors[0] + ) + let hello = BrokerHello( + expectedABI: 6, + startupConfigurationDigest: "xpc-fd-test", + requestedCapabilities: [.protocolRaw] + ) + let message = try IOSBrokerXPC.makeHello(hello, dataChannel: sender) + #expect(sender.close()) + + let decoded = try IOSBrokerXPC.decodeHello(message) + #expect(decoded.hello == hello) + let received = try decoded.dataChannel.borrowedDescriptor() + var sent = UInt8(ascii: "X") + #expect(Darwin.write(descriptors[1], &sent, 1) == 1) + var byte: UInt8 = 0 + #expect(Darwin.read(received, &byte, 1) == 1) + #expect(byte == UInt8(ascii: "X")) + #expect(decoded.dataChannel.close()) + #expect(!decoded.dataChannel.close()) +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func xpcDiagnosticsDecodesCompleteCheckpointMemoryEvidence() throws { + var reply = try makeDiagnosticsReply() + reply[IOSBrokerXPC.checkpointMemorySampleSequenceKey] = UInt64(4) + reply[IOSBrokerXPC.checkpointMemorySampleStartedAtUptimeNanosecondsKey] = UInt64(100) + reply[IOSBrokerXPC.checkpointMemorySampledAtUptimeNanosecondsKey] = UInt64(110) + reply[IOSBrokerXPC.checkpointMemorySampleCompletedAtUptimeNanosecondsKey] = UInt64(120) + reply[IOSBrokerXPC.checkpointMemorySamplePhysFootprintBytesKey] = UInt64(1_024) + reply[IOSBrokerXPC.checkpointMemorySampleResidentBytesKey] = UInt64(2_048) + reply[IOSBrokerXPC.checkpointMemorySampleAvailableMemoryBytesKey] = UInt64(4_096) + + let diagnostics = try IOSBrokerXPC.decodeWorkerDiagnostics(reply) + #expect( + diagnostics.checkpointMemorySample + == IOSBrokerWireCheckpointMemorySample( + sequence: 4, + startedAtUptimeNanoseconds: 100, + sampledAtUptimeNanoseconds: 110, + completedAtUptimeNanoseconds: 120, + physFootprintBytes: 1_024, + residentBytes: 2_048, + availableMemoryBytes: 4_096 + ) + ) + #expect(!diagnostics.checkpointInProgress) +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func xpcDiagnosticsRejectsPartialCheckpointMemoryEvidence() throws { + var reply = try makeDiagnosticsReply() + reply[IOSBrokerXPC.checkpointMemorySampleSequenceKey] = UInt64(4) + + do { + _ = try IOSBrokerXPC.decodeWorkerDiagnostics(reply) + Issue.record("partial checkpoint memory evidence must be rejected") + } catch let error as BrokerError { + guard case .protocolViolation(let message) = error else { + Issue.record("expected protocolViolation, got \(error)") + return + } + #expect(message.contains("incomplete checkpoint memory evidence")) + } +} + +@available(iOS 26.0, macOS 26.0, *) +private func makeDiagnosticsReply() throws -> XPCDictionary { + var reply = IOSBrokerXPC.makeAcknowledgement(.diagnostics) + reply[IOSBrokerXPC.stateKey] = "ready" + reply[BrokerControlKey.epoch] = BrokerEpoch.fresh().description + reply[BrokerControlKey.extensionPID] = Int64(42) + reply[IOSBrokerXPC.nativeDispatchStartedKey] = false + reply[IOSBrokerXPC.transactionStatusKey] = "idle" + reply[IOSBrokerXPC.capabilitiesKey] = String( + decoding: try JSONEncoder().encode(BrokerCapabilities()), + as: UTF8.self + ) + reply[IOSBrokerXPC.checkpointInProgressKey] = false + return reply +} diff --git a/src/sdks/swift/Tests/OliphauntIOSBrokerTests/IOSBrokerBackgroundPreparationTests.swift b/src/sdks/swift/Tests/OliphauntIOSBrokerTests/IOSBrokerBackgroundPreparationTests.swift new file mode 100644 index 00000000..28695fab --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntIOSBrokerTests/IOSBrokerBackgroundPreparationTests.swift @@ -0,0 +1,41 @@ +import Oliphaunt +import Testing + +@testable import OliphauntIOSBroker + +@available(iOS 26.0, macOS 26.0, *) +@Test +func backgroundPreparationPreservesHostObservedActiveCancellation() { + let worker = OliphauntBackgroundPreparationResult( + cancelledActiveWork: false, + checkpointed: true + ) + + let merged = IOSBrokerManager.mergeBackgroundPreparationResult( + hostCancelledActiveWork: true, + workerResult: worker + ) + + #expect(merged.cancelledActiveWork) + #expect(merged.checkpointed) + #expect(merged.skippedCheckpointReason == nil) +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func backgroundPreparationPreservesWorkerSkipReason() { + let worker = OliphauntBackgroundPreparationResult( + cancelledActiveWork: true, + checkpointed: false, + skippedCheckpointReason: .activeWork + ) + + let merged = IOSBrokerManager.mergeBackgroundPreparationResult( + hostCancelledActiveWork: false, + workerResult: worker + ) + + #expect(merged.cancelledActiveWork) + #expect(!merged.checkpointed) + #expect(merged.skippedCheckpointReason == .activeWork) +} diff --git a/src/sdks/swift/Tests/OliphauntIOSBrokerTests/IOSBrokerHostInvariantTests.swift b/src/sdks/swift/Tests/OliphauntIOSBrokerTests/IOSBrokerHostInvariantTests.swift new file mode 100644 index 00000000..0c47cbf3 --- /dev/null +++ b/src/sdks/swift/Tests/OliphauntIOSBrokerTests/IOSBrokerHostInvariantTests.swift @@ -0,0 +1,794 @@ +import Darwin +import Foundation +import Oliphaunt +import OliphauntBrokerProtocol +import OliphauntBrokerXPC +import Testing + +@testable import OliphauntIOSBroker + +@Test +func workerDiagnosticsMapsRetainedCheckpointMemoryEvidence() { + let epoch = BrokerEpoch.fresh() + let wire = IOSBrokerWireDiagnostics( + state: "ready", + epoch: epoch, + extensionProcessIdentifier: 42, + manifestDigest: "manifest", + activeRequestID: nil, + nativeDispatchStarted: false, + transactionStatus: "idle", + capabilities: BrokerCapabilities(), + currentPhysFootprintBytes: 10, + currentResidentBytes: 20, + availableMemoryBytes: 30, + checkpointInProgress: false, + checkpointMemorySample: IOSBrokerWireCheckpointMemorySample( + sequence: 7, + startedAtUptimeNanoseconds: 100, + sampledAtUptimeNanoseconds: 110, + completedAtUptimeNanoseconds: 120, + physFootprintBytes: 1_024, + residentBytes: 2_048, + availableMemoryBytes: 4_096 + ), + storageProtectionEvidenceJSON: nil, + extensionEntryPreOpenPhysFootprintBytes: 40, + extensionEntryPreOpenResidentBytes: 50, + openedIdlePhysFootprintBytes: 60, + openedIdleResidentBytes: 70 + ) + + let diagnostics = IOSBrokerWorkerDiagnostics(wire: wire) + #expect(diagnostics.epoch == epoch) + #expect( + diagnostics.checkpointMemorySample + == IOSBrokerCheckpointMemorySample( + sequence: 7, + startedAtUptimeNanoseconds: 100, + sampledAtUptimeNanoseconds: 110, + completedAtUptimeNanoseconds: 120, + physFootprintBytes: 1_024, + residentBytes: 2_048, + availableMemoryBytes: 4_096 + ) + ) + #expect(!diagnostics.checkpointInProgress) +} + +@Test +func completedWithoutTerminalReadyForQueryIsAProtocolViolation() async throws { + let sockets = try IOSBrokerSocketPair.make() + defer { sockets.host.close() } + + let workerDescriptor = try sockets.extensionEndpoint.takeDescriptor() + let epoch = BrokerEpoch.fresh() + let requestID = try BrokerRequestID(validating: 17) + let worker = Task.detached { + defer { Darwin.close(workerDescriptor) } + try makeBlocking(workerDescriptor) + while true { + let header = try readFrameHeader(workerDescriptor, epoch: epoch) + _ = try readExactly(workerDescriptor, count: Int(header.payloadLength)) + if header.frameType == .requestEnd { + break + } + } + + let commandComplete = backendMessage(type: 0x43, body: Data("SELECT 1\0".utf8)) + try writeAll( + try BrokerFrame( + frameType: .responseBytes, + epoch: epoch, + requestID: requestID.rawValue, + payload: commandComplete + ).encoded(), + to: workerDescriptor + ) + try writeAll( + try BrokerFrame( + frameType: .completed, + epoch: epoch, + requestID: requestID.rawValue + ).encoded(), + to: workerDescriptor + ) + } + + do { + _ = try await sockets.host.execute( + requestID: requestID, + epoch: epoch, + protocolVersion: OliphauntBrokerProtocol.maximumVersion, + bytes: simpleQuery("SELECT 1"), + maximumRequestBytes: 1024, + onChunk: { _ in } + ) + Issue.record("Completed without ReadyForQuery must not be accepted") + } catch let failure as IOSBrokerDataPlaneFailure { + switch failure { + case .protocolViolation(let message): + #expect(message.contains("without a terminal ReadyForQuery")) + default: + Issue.record("expected protocolViolation, got \(failure)") + } + } + try await worker.value +} + +@Test +func socketCloseDuringResponseIsOutcomeUnknownAfterPartialDelivery() async throws { + let sockets = try IOSBrokerSocketPair.make() + defer { sockets.host.close() } + + let workerDescriptor = try sockets.extensionEndpoint.takeDescriptor() + let epoch = BrokerEpoch.fresh() + let requestID = try BrokerRequestID(validating: 18) + let worker = Task.detached { + defer { Darwin.close(workerDescriptor) } + try makeBlocking(workerDescriptor) + while true { + let header = try readFrameHeader(workerDescriptor, epoch: epoch) + _ = try readExactly(workerDescriptor, count: Int(header.payloadLength)) + if header.frameType == .requestEnd { + break + } + } + + let partial = backendMessage(type: 0x43, body: Data("SELECT 1\0".utf8)) + try writeAll( + try BrokerFrame( + frameType: .responseBytes, + epoch: epoch, + requestID: requestID.rawValue, + payload: partial + ).encoded(), + to: workerDescriptor + ) + // Closing without Completed makes the already-dispatched result + // ambiguous even though the streaming consumer observed bytes. + } + let observed = LockedChunkAccumulator() + + do { + _ = try await sockets.host.execute( + requestID: requestID, + epoch: epoch, + protocolVersion: OliphauntBrokerProtocol.maximumVersion, + bytes: simpleQuery("SELECT 1"), + maximumRequestBytes: 1024, + onChunk: { observed.append($0) } + ) + Issue.record("socket EOF during a response must be OutcomeUnknown") + } catch let failure as IOSBrokerDataPlaneFailure { + guard case .outcomeUnknown = failure else { + Issue.record("expected outcomeUnknown, got \(failure)") + try await worker.value + return + } + } + #expect(observed.byteCount > 0) + try await worker.value +} + +@Test +func rawResponseLimitAfterDispatchIsBoundedAndOutcomeUnknown() async throws { + let sockets = try IOSBrokerSocketPair.make() + defer { sockets.host.close() } + + let workerDescriptor = try sockets.extensionEndpoint.takeDescriptor() + let epoch = BrokerEpoch.fresh() + let requestID = try BrokerRequestID(validating: 20) + let worker = Task.detached { + defer { Darwin.close(workerDescriptor) } + try makeBlocking(workerDescriptor) + while true { + let header = try readFrameHeader(workerDescriptor, epoch: epoch) + _ = try readExactly(workerDescriptor, count: Int(header.payloadLength)) + if header.frameType == .requestEnd { + break + } + } + + try writeAll( + try BrokerFrame( + frameType: .responseBytes, + epoch: epoch, + requestID: requestID.rawValue, + payload: backendMessage(type: 0x43, body: Data("SELECT 1\0".utf8)) + ).encoded(), + to: workerDescriptor + ) + } + let collector = IOSBrokerResponseCollector(maximumBytes: 8) + + do { + _ = try await sockets.host.execute( + requestID: requestID, + epoch: epoch, + protocolVersion: OliphauntBrokerProtocol.maximumVersion, + bytes: simpleQuery("SELECT 1"), + maximumRequestBytes: 1024, + onChunk: { try collector.append($0) } + ) + Issue.record("an over-limit raw response unexpectedly completed") + } catch let failure as IOSBrokerDataPlaneFailure { + guard case .outcomeUnknown = failure else { + Issue.record("expected outcomeUnknown, got \(failure)") + try await worker.value + return + } + } + #expect(collector.value.isEmpty) + try await worker.value +} + +@Test +func socketCloseDuringUploadIsOutcomeUnknown() async throws { + let sockets = try IOSBrokerSocketPair.make() + defer { sockets.host.close() } + + let workerDescriptor = try sockets.extensionEndpoint.takeDescriptor() + let epoch = BrokerEpoch.fresh() + let requestID = try BrokerRequestID(validating: 19) + let worker = Task.detached { + defer { Darwin.close(workerDescriptor) } + try makeBlocking(workerDescriptor) + let header = try readFrameHeader(workerDescriptor, epoch: epoch) + #expect(header.frameType == .requestBegin) + _ = try readExactly(workerDescriptor, count: Int(header.payloadLength)) + // The peer disappears after admission but before the complete frontend + // request is known to have arrived. The host must never call this safe + // to replay. + } + let largeRequest = simpleQuery( + "SELECT '" + String(repeating: "u", count: 2 * 1024 * 1024) + "'") + + do { + _ = try await sockets.host.execute( + requestID: requestID, + epoch: epoch, + protocolVersion: OliphauntBrokerProtocol.maximumVersion, + bytes: largeRequest, + maximumRequestBytes: 3 * 1024 * 1024, + onChunk: { _ in } + ) + Issue.record("socket loss during request upload must be OutcomeUnknown") + } catch let failure as IOSBrokerDataPlaneFailure { + guard case .outcomeUnknown = failure else { + Issue.record("expected outcomeUnknown, got \(failure)") + try await worker.value + return + } + } + try await worker.value +} + +@Test +func ownedDescriptorTransferAndCloseAreExactlyOnce() throws { + var descriptors: [Int32] = [-1, -1] + #expect(pipe(&descriptors) == 0) + defer { Darwin.close(descriptors[1]) } + + let owned = try IOSBrokerOwnedFileDescriptor(takingOwnershipOf: descriptors[0]) + #expect(owned.isOpen) + let transferred = try owned.takeDescriptor() + #expect(!owned.isOpen) + #expect(!owned.close()) + do { + _ = try owned.borrowedDescriptor() + Issue.record("a transferred descriptor remained borrowable") + } catch let error as POSIXError { + #expect(error.code == .EBADF) + } + #expect(Darwin.close(transferred) == 0) + #expect(!owned.close()) +} + +@Test +func readyForQueryMustBeTheTerminalBackendMessage() throws { + var valid = IOSBrokerBackendResponseObserver() + try valid.append(backendMessage(type: 0x43, body: Data("SELECT 1\0".utf8))) + try valid.append(backendMessage(type: 0x5A, body: Data([0x49]))) + #expect(try valid.finish() == .idle) + + var followedByAnotherMessage = IOSBrokerBackendResponseObserver() + try followedByAnotherMessage.append(backendMessage(type: 0x5A, body: Data([0x49]))) + try followedByAnotherMessage.append(backendMessage(type: 0x4E, body: Data())) + do { + _ = try followedByAnotherMessage.finish() + Issue.record("ReadyForQuery followed by another message must not be terminal") + } catch let failure as IOSBrokerDataPlaneFailure { + switch failure { + case .protocolViolation(let message): + #expect(message.contains("without a terminal ReadyForQuery")) + default: + Issue.record("expected protocolViolation, got \(failure)") + } + } +} + +@Test +func activeAndQueuedRequestsShareTheAggregateInputBudget() throws { + var budget = IOSBrokerInputBudget(maximumBytes: 8) + let active = try BrokerRequestID(validating: 1) + let queued = try BrokerRequestID(validating: 2) + let rejected = try BrokerRequestID(validating: 3) + + let admittedActive = budget.reserve(5, for: active) + #expect(admittedActive) + budget.activate(active) + #expect(budget.state(for: active) == .active) + #expect(budget.accountedBytes == 5) + + let admittedQueued = budget.reserve(3, for: queued) + #expect(admittedQueued) + #expect(budget.state(for: queued) == .queued) + #expect(budget.accountedBytes == 8) + let rejectedWhileFull = budget.reserve(1, for: rejected) + #expect(!rejectedWhileFull) + + budget.release(active) + #expect(budget.accountedBytes == 3) + let admittedAfterTerminal = budget.reserve(1, for: rejected) + #expect(admittedAfterTerminal) + #expect(budget.accountedBytes == 4) +} + +@Test +func resumeRecoveryRetriesWorkerInterruptionExactlyOnce() throws { + let staleEpoch = BrokerEpoch.fresh() + var policy = IOSBrokerResumeRetryPolicy() + + let firstRetry = policy.consumeRetry( + for: BrokerError.workerInterrupted(epoch: staleEpoch) + ) + #expect(firstRetry) + #expect(policy.retryCount == 1) + let secondRetry = policy.consumeRetry( + for: BrokerError.workerInterrupted(epoch: staleEpoch) + ) + #expect(!secondRetry) + + var semanticFailurePolicy = IOSBrokerResumeRetryPolicy() + let protocolRetry = semanticFailurePolicy.consumeRetry( + for: BrokerError.protocolViolation("malformed reply") + ) + #expect(!protocolRetry) + let configurationRetry = semanticFailurePolicy.consumeRetry( + for: BrokerError.invalidConfiguration("mismatch") + ) + #expect(!configurationRetry) + #expect(semanticFailurePolicy.retryCount == 0) +} + +@Test +func resumeRecoveryRequiresFreshEpochAndSameRootIdentity() throws { + let staleEpoch = BrokerEpoch.fresh() + let freshEpoch = BrokerEpoch.fresh() + let expectedDigest = "root-manifest-a" + let expectation = IOSBrokerResumeRecoveryExpectation( + staleEpoch: staleEpoch, + rootManifestDigest: expectedDigest + ) + + try expectation.validate( + recoveredEpoch: freshEpoch, + recoveredRootManifestDigest: expectedDigest + ) + + do { + try expectation.validate( + recoveredEpoch: staleEpoch, + recoveredRootManifestDigest: expectedDigest + ) + Issue.record("resume recovery accepted the interrupted epoch") + } catch let error as BrokerError { + guard case .protocolViolation(let reason) = error else { + Issue.record("expected protocolViolation, got \(error)") + return + } + #expect(reason.contains("reused the interrupted worker epoch")) + } + + do { + try expectation.validate( + recoveredEpoch: freshEpoch, + recoveredRootManifestDigest: "root-manifest-b" + ) + Issue.record("resume recovery accepted a different root identity") + } catch let error as BrokerError { + #expect( + error + == .rootMismatch( + expected: expectedDigest, + actual: "root-manifest-b" + ) + ) + } +} + +@Test +func launchAcquisitionRetriesNestedDeadProcessAssertionExactlyOnce() throws { + let deadProcess = NSError( + domain: "RBSAssertionErrorDomain", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "Specified target process does not exist"] + ) + let failedAssertion = NSError( + domain: "com.apple.extensionKit.errorDomain", + code: 4, + userInfo: [NSUnderlyingErrorKey: deadProcess] + ) + let failedLaunch = NSError( + domain: "com.apple.extensionKit.errorDomain", + code: 2, + userInfo: [NSUnderlyingErrorKey: failedAssertion] + ) + var policy = IOSBrokerLaunchAcquisitionRetryPolicy() + + let firstRetry = policy.consumeRetry( + for: failedLaunch, + recoveringInterruptedEpoch: true + ) + #expect(firstRetry) + #expect(policy.retryCount == 1) + let secondRetry = policy.consumeRetry( + for: failedLaunch, + recoveringInterruptedEpoch: true + ) + #expect(!secondRetry) + + var initialLaunchPolicy = IOSBrokerLaunchAcquisitionRetryPolicy() + let initialLaunchRetry = initialLaunchPolicy.consumeRetry( + for: failedLaunch, + recoveringInterruptedEpoch: false + ) + #expect(!initialLaunchRetry) + #expect(initialLaunchPolicy.retryCount == 0) +} + +@Test +func launchMetricsSeparateProcessAttemptsFromValidatedReadyHandshakes() { + var metrics = IOSBrokerLaunchMetrics() + #expect(metrics.attemptCount == 0) + #expect(metrics.successfulCount == 0) + + // A wedged or otherwise failed process acquisition still proves that the + // host attempted a replacement, but must not inflate successful launches. + metrics.recordProcessInitializationAttempt() + #expect(metrics.attemptCount == 1) + #expect(metrics.successfulCount == 0) + + metrics.recordProcessInitializationAttempt() + metrics.recordReadyValidatedLaunch() + #expect(metrics.attemptCount == 2) + #expect(metrics.successfulCount == 1) +} + +@Test +func launchAcquisitionDoesNotRetrySemanticOrAmbiguousDataFailures() throws { + let failures: [any Error] = [ + BrokerError.protocolViolation("malformed reply"), + BrokerError.invalidConfiguration("mismatch"), + BrokerError.outcomeUnknown( + epoch: BrokerEpoch.fresh(), + requestID: try BrokerRequestID(validating: 91) + ), + NSError(domain: "RBSAssertionErrorDomain", code: 1), + NSError(domain: "com.apple.extensionKit.errorDomain", code: 2), + ] + + for failure in failures { + var policy = IOSBrokerLaunchAcquisitionRetryPolicy() + let retry = policy.consumeRetry( + for: failure, + recoveringInterruptedEpoch: true + ) + #expect(!retry) + #expect(policy.retryCount == 0) + } +} + +@Test +func interruptedLaunchRequiresFreshEpochAndKnownDeadProcess() throws { + let staleEpoch = BrokerEpoch.fresh() + let freshEpoch = BrokerEpoch.fresh() + let expectation = IOSBrokerInterruptedLaunchExpectation( + staleEpoch: staleEpoch, + knownDeadProcessIdentifier: 3931 + ) + + try expectation.validate( + recoveredEpoch: freshEpoch, + recoveredProcessIdentifier: 3932 + ) + + do { + try expectation.validate( + recoveredEpoch: staleEpoch, + recoveredProcessIdentifier: 3932 + ) + Issue.record("launch recovery accepted the interrupted epoch") + } catch let error as BrokerError { + guard case .protocolViolation(let reason) = error else { + Issue.record("expected protocolViolation, got \(error)") + return + } + #expect(reason.contains("reused the interrupted worker epoch")) + } + + do { + try expectation.validate( + recoveredEpoch: freshEpoch, + recoveredProcessIdentifier: 3931 + ) + Issue.record("launch recovery accepted the known-dead process") + } catch let error as BrokerError { + guard case .protocolViolation(let reason) = error else { + Issue.record("expected protocolViolation, got \(error)") + return + } + #expect(reason.contains("reused the known-dead extension process")) + } +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func residentIdentityAcceptsOnlyWorkerSupportedRuntimeFields() throws { + let broker = IOSBrokerConfiguration( + expectedABI: 42, + startupConfigurationDigest: "host-invariant-test" + ) + let supported = OliphauntConfiguration( + mode: .nativeBroker, + durability: .safe, + runtimeFootprint: .smallMobile, + extensions: ["vector", "pg_trgm"] + ) + _ = try IOSBrokerManager.ResidentIdentity(broker: broker, database: supported) + + var explicitDefaults = supported + explicitDefaults.database = "postgres" + _ = try IOSBrokerManager.ResidentIdentity(broker: broker, database: explicitDefaults) + + var explicitUsername = supported + explicitUsername.username = "postgres" + expectInvalidConfiguration( + explicitUsername, + broker: broker, + reason: "iOS broker v1 does not accept a caller-provided PostgreSQL username" + ) + + var differentRoot = supported + differentRoot.root = URL(fileURLWithPath: "/tmp/not-the-extension-private-default") + do { + _ = try IOSBrokerManager.ResidentIdentity( + broker: broker, + database: differentRoot + ) + Issue.record("iOS broker v1 accepted a caller-provided root") + } catch let error as BrokerError { + guard case .rootMismatch(let expected, let actual) = error else { + Issue.record("expected rootMismatch, got \(error)") + return + } + #expect(expected == OliphauntBrokerProtocol.canonicalRootID) + #expect(actual.contains("not-the-extension-private-default")) + } + + var unsafeDurability = supported + unsafeDurability.durability = .balanced + expectInvalidConfiguration( + unsafeDurability, + broker: broker, + reason: "iOS broker v1 requires safe durability" + ) + + var customGUCs = supported + customGUCs.startupGUCs = [OliphauntStartupGUC("statement_timeout", "1000")] + expectInvalidConfiguration( + customGUCs, + broker: broker, + reason: "iOS broker v1 does not support custom startup GUCs" + ) + + var customUsername = supported + customUsername.username = "application" + expectInvalidConfiguration( + customUsername, + broker: broker, + reason: "iOS broker v1 does not accept a caller-provided PostgreSQL username" + ) + + var customDatabase = supported + customDatabase.database = "application" + expectInvalidConfiguration( + customDatabase, + broker: broker, + reason: "iOS broker v1 requires PostgreSQL database postgres" + ) +} + +@available(iOS 26.0, macOS 26.0, *) +@Test +func iosBrokerEngineNeverAdvertisesServerOrConnectionString() throws { + let engine = IOSBrokerEngine( + configuration: IOSBrokerConfiguration( + expectedABI: 6, + startupConfigurationDigest: "server-boundary-test" + ), + manager: IOSBrokerManager() + ) + let broker = try #require( + engine.supportedModes.first(where: { $0.mode == .nativeBroker }) + ) + #expect(broker.available) + #expect(broker.capabilities.connectionString == nil) + #expect(!broker.capabilities.backupRestore) + #expect(!broker.capabilities.independentSessions) + #expect(broker.capabilities.maxClientSessions == 1) + + let server = try #require( + engine.supportedModes.first(where: { $0.mode == .nativeServer }) + ) + #expect(!server.available) + #expect(server.unavailableReason == IOSBrokerEngine.nativeServerUnavailableReason) +} + +@Test +func rawResponseCollectorFailsBeforeExceedingItsMemoryCeiling() throws { + let collector = IOSBrokerResponseCollector(maximumBytes: 8) + try collector.append(Data(repeating: 0x41, count: 5)) + #expect(collector.value.count == 5) + + do { + try collector.append(Data(repeating: 0x42, count: 4)) + Issue.record("raw response collector exceeded its declared ceiling") + } catch let error as IOSBrokerRawResponseLimitError { + #expect(error == .exceeded(maximumBytes: 8)) + } + #expect(collector.value.count == 5) +} + +@Test +func brokerConfigurationRejectsAnUnboundedRawResponseCollector() throws { + var configuration = IOSBrokerConfiguration( + expectedABI: 6, + startupConfigurationDigest: "raw-response-ceiling-test" + ) + configuration.maximumRawResponseBytes = + OliphauntBrokerProtocol.maximumQueuedBytesPerDirection + 1 + + do { + _ = try configuration.validated() + Issue.record("an oversized raw response collector was accepted") + } catch let error as BrokerError { + guard case .invalidConfiguration = error else { + Issue.record("expected invalidConfiguration, got \(error)") + return + } + } +} + +@available(iOS 26.0, macOS 26.0, *) +private func expectInvalidConfiguration( + _ configuration: OliphauntConfiguration, + broker: IOSBrokerConfiguration, + reason expectedReason: String +) { + do { + _ = try IOSBrokerManager.ResidentIdentity( + broker: broker, + database: configuration + ) + Issue.record("expected invalid configuration: \(expectedReason)") + } catch let error as BrokerError { + guard case .invalidConfiguration(let actualReason) = error else { + Issue.record("expected invalidConfiguration, got \(error)") + return + } + #expect(actualReason == expectedReason) + } catch { + Issue.record("expected BrokerError.invalidConfiguration, got \(error)") + } +} + +private func simpleQuery(_ sql: String) -> Data { + var body = Data(sql.utf8) + body.append(0) + var request = Data([0x51]) + appendUInt32(UInt32(body.count + 4), to: &request) + request.append(body) + return request +} + +private func backendMessage(type: UInt8, body: Data) -> Data { + var message = Data([type]) + appendUInt32(UInt32(body.count + 4), to: &message) + message.append(body) + return message +} + +private func appendUInt32(_ value: UInt32, to data: inout Data) { + data.append(UInt8((value >> 24) & 0xFF)) + data.append(UInt8((value >> 16) & 0xFF)) + data.append(UInt8((value >> 8) & 0xFF)) + data.append(UInt8(value & 0xFF)) +} + +private func makeBlocking(_ descriptor: Int32) throws { + let flags = fcntl(descriptor, F_GETFL) + guard flags >= 0, fcntl(descriptor, F_SETFL, flags & ~O_NONBLOCK) >= 0 else { + throw POSIXError(POSIXError.Code(rawValue: errno) ?? .EIO) + } +} + +private func readFrameHeader( + _ descriptor: Int32, epoch: BrokerEpoch +) throws + -> BrokerFrameHeader +{ + let bytes = try readExactly( + descriptor, + count: Int(OliphauntBrokerProtocol.headerLength) + ) + return try BrokerFrameHeader.decode(bytes, expectedEpoch: epoch) +} + +private func readExactly(_ descriptor: Int32, count: Int) throws -> Data { + var result = Data() + result.reserveCapacity(count) + var buffer = [UInt8](repeating: 0, count: min(16 * 1024, max(1, count))) + while result.count < count { + let wanted = min(buffer.count, count - result.count) + let received = buffer.withUnsafeMutableBytes { rawBuffer in + Darwin.read(descriptor, rawBuffer.baseAddress, wanted) + } + if received > 0 { + result.append(contentsOf: buffer[0.. 0 { + offset += written + } else if written < 0, errno == EINTR { + continue + } else { + throw POSIXError(POSIXError.Code(rawValue: errno) ?? .EIO) + } + } + } +} + +private final class LockedChunkAccumulator: @unchecked Sendable { + private let lock = NSLock() + private var bytes = 0 + + var byteCount: Int { + lock.lock() + defer { lock.unlock() } + return bytes + } + + func append(_ chunk: Data) { + lock.lock() + bytes += chunk.count + lock.unlock() + } +} diff --git a/src/sdks/swift/moon.yml b/src/sdks/swift/moon.yml index 99207c49..0bf34160 100644 --- a/src/sdks/swift/moon.yml +++ b/src/sdks/swift/moon.yml @@ -78,6 +78,72 @@ tasks: options: cache: local runFromWorkspaceRoot: true + smoke-ios-broker: + tags: ["runtime", "smoke", "ios", "simulator", "spike"] + command: "bash src/sdks/swift/tools/run-ios-broker-simulator.sh" + inputs: + - "/Package.swift" + - "/spikes/ios-native-broker/**/*" + - "/src/sdks/swift/Package.swift" + - "/src/sdks/swift/Sources/**/*" + - "/src/sdks/swift/tools/prepare-ios-broker-artifacts.sh" + - "/src/sdks/swift/tools/run-ios-broker-simulator.sh" + - "/src/sdks/react-native/tools/expo-runner-common.sh" + - "/src/sdks/react-native/tools/expo-runner-runtime-resources.sh" + - "/src/sdks/react-native/tools/expo-runner-workspace.sh" + - "/src/sdks/react-native/tools/mobile-extension-runtime.sh" + - "/src/runtimes/liboliphaunt/native/**/*" + outputs: + - "/target/ios-native-broker-spike/logs/**/*" + - "/target/ios-native-broker-spike/reports/**/*" + options: + cache: false + runFromWorkspaceRoot: true + runInCI: false + smoke-ios-broker-full-matrix: + tags: ["runtime", "smoke", "ios", "simulator", "spike", "faults"] + command: "bash src/sdks/swift/tools/run-ios-broker-full-simulator-matrix.sh" + inputs: + - "/Package.swift" + - "/spikes/ios-native-broker/**/*" + - "/src/sdks/swift/Package.swift" + - "/src/sdks/swift/Sources/**/*" + - "/src/sdks/swift/tools/prepare-ios-broker-artifacts.sh" + - "/src/sdks/swift/tools/run-ios-broker-simulator.sh" + - "/src/sdks/swift/tools/run-ios-broker-full-simulator-matrix.sh" + - "/src/sdks/react-native/tools/expo-runner-common.sh" + - "/src/sdks/react-native/tools/expo-runner-runtime-resources.sh" + - "/src/sdks/react-native/tools/expo-runner-workspace.sh" + - "/src/sdks/react-native/tools/mobile-extension-runtime.sh" + - "/src/runtimes/liboliphaunt/native/**/*" + outputs: + - "/target/ios-native-broker-full-matrix/**/*" + options: + cache: false + runFromWorkspaceRoot: true + runInCI: false + smoke-ios-broker-device: + tags: ["runtime", "smoke", "ios", "device", "physical", "spike"] + command: "bash src/sdks/swift/tools/run-ios-broker-device.sh" + inputs: + - "/Package.swift" + - "/spikes/ios-native-broker/**/*" + - "/src/sdks/swift/Package.swift" + - "/src/sdks/swift/Sources/**/*" + - "/src/sdks/swift/tools/prepare-ios-broker-artifacts.sh" + - "/src/sdks/swift/tools/run-ios-broker-device.sh" + - "/src/sdks/react-native/tools/expo-runner-common.sh" + - "/src/sdks/react-native/tools/expo-runner-runtime-resources.sh" + - "/src/sdks/react-native/tools/expo-runner-workspace.sh" + - "/src/sdks/react-native/tools/mobile-extension-runtime.sh" + - "/src/runtimes/liboliphaunt/native/**/*" + outputs: + - "/target/ios-native-broker-device-spike/logs/**/*" + - "/target/ios-native-broker-device-spike/reports/**/*" + options: + cache: false + runFromWorkspaceRoot: true + runInCI: false package: tags: ["package"] command: "bash src/sdks/swift/tools/check-sdk.sh package-shape" diff --git a/src/sdks/swift/tools/check-sdk.sh b/src/sdks/swift/tools/check-sdk.sh index 290b090d..5790a0ec 100755 --- a/src/sdks/swift/tools/check-sdk.sh +++ b/src/sdks/swift/tools/check-sdk.sh @@ -98,7 +98,7 @@ check_swiftpm_release_asset_manifest() { if [ -n "${OLIPHAUNT_SWIFT_RELEASE_ASSET_DIR:-}" ]; then asset_dir="$OLIPHAUNT_SWIFT_RELEASE_ASSET_DIR" - asset_base_url="${OLIPHAUNT_SWIFT_RELEASE_ASSET_BASE_URL:-file://$asset_dir}" + asset_base_url="${OLIPHAUNT_SWIFT_RELEASE_ASSET_BASE_URL:-https://github.com/f0rr0/oliphaunt/releases/download/liboliphaunt-native-v$liboliphaunt_version}" [ -d "$asset_dir" ] || { echo "Swift release asset directory does not exist: $asset_dir" >&2 exit 1 @@ -133,6 +133,57 @@ check_swiftpm_release_asset_manifest() { echo "SwiftPM release fixture manifest must not point at a monorepo-local XCFramework path" >&2 exit 1 fi + + release_package="$scratch_root/swiftpm-release-package" + release_dump="$scratch_root/swiftpm-release-package.json" + rm -rf "$release_package" + mkdir -p "$release_package/src/sdks/swift" + cp -R "$archive_package_dir/." "$release_package/src/sdks/swift/" + cp -R "$generated_tree/." "$release_package/" + cp "$release_manifest" "$release_package/Package.swift" + printf '\n==> swift package --package-path %s dump-package\n' "$release_package" + swift package --package-path "$release_package" dump-package >"$release_dump" + run node - "$release_dump" <<'NODE' +const { readFileSync } = require("node:fs"); + +const label = "SwiftPM rendered release broker graph"; +const fail = (message) => { + console.error(`${label}: ${message}`); + process.exit(1); +}; +const document = JSON.parse(readFileSync(process.argv[2], "utf8")); +const products = new Map(document.products.map((product) => [product.name, product])); +const targets = new Map(document.targets.map((target) => [target.name, target])); +const expected = new Map([ + ["OliphauntBrokerProtocol", []], + ["OliphauntBrokerXPC", ["OliphauntBrokerProtocol"]], + ["OliphauntIOSBroker", ["Oliphaunt", "OliphauntBrokerProtocol", "OliphauntBrokerXPC"]], + ["OliphauntBrokerExtension", ["COliphaunt", "Oliphaunt", "OliphauntBrokerProtocol"]], +]); + +for (const [name, expectedDependencies] of expected) { + const product = products.get(name); + if (product === undefined || JSON.stringify(product.targets) !== JSON.stringify([name])) { + fail(`product ${name} must expose exactly target ${name}`); + } + const target = targets.get(name); + if (target === undefined || target.type !== "regular") { + fail(`target ${name} must be a regular source target`); + } + const dependencies = target.dependencies.map((dependency) => { + if (!Array.isArray(dependency.byName) || typeof dependency.byName[0] !== "string") { + fail(`target ${name} has an unsupported dependency declaration`); + } + return dependency.byName[0]; + }).sort(); + if (JSON.stringify(dependencies) !== JSON.stringify(expectedDependencies)) { + fail( + `target ${name} dependencies ${JSON.stringify(dependencies)} ` + + `do not match ${JSON.stringify(expectedDependencies)}`, + ); + } +} +NODE } check_swiftpm_extension_product_generator() { @@ -618,7 +669,15 @@ require swift require unzip require node require cc -for product in COliphaunt Oliphaunt OliphauntExtensionSupport; do +for product in \ + COliphaunt \ + OliphauntBrokerProtocol \ + OliphauntBrokerXPC \ + OliphauntIOSBroker \ + OliphauntBrokerExtension \ + Oliphaunt \ + OliphauntExtensionSupport +do if ! grep -Fq ".library(name: \"$product\"" "$package_dir/Package.swift"; then echo "Swift base package must expose public consumer integration product $product" >&2 exit 1 @@ -716,6 +775,21 @@ for required in \ Sources/Oliphaunt/OliphauntQuery.swift \ Sources/Oliphaunt/OliphauntRuntimeResources.swift \ Sources/Oliphaunt/OliphauntExtensionResources.swift \ + Sources/OliphauntBrokerProtocol/BrokerFrame.swift \ + Sources/OliphauntBrokerProtocol/BrokerProtocol.swift \ + Sources/OliphauntBrokerProtocol/BrokerStateMachines.swift \ + Sources/OliphauntBrokerXPC/IOSBrokerXPC.swift \ + Sources/OliphauntIOSBroker/IOSBrokerDataChannel.swift \ + Sources/OliphauntIOSBroker/IOSBrokerManager.swift \ + Sources/OliphauntIOSBroker/IOSBrokerPublicAPI.swift \ + Sources/OliphauntIOSBroker/IOSBrokerSession.swift \ + Sources/OliphauntBrokerExtension/BackendResponseObservation.swift \ + Sources/OliphauntBrokerExtension/BrokerBackendPrivacyFilter.swift \ + Sources/OliphauntBrokerExtension/BrokerExtensionStorage.swift \ + Sources/OliphauntBrokerExtension/BrokerFaultInjection.swift \ + Sources/OliphauntBrokerExtension/BrokerSocketWorker.swift \ + Sources/OliphauntBrokerExtension/CancellationController.swift \ + Sources/OliphauntBrokerExtension/WorkerCore.swift \ Sources/OliphauntExtensionSupport/OliphauntExtensionSupport.swift \ tools/render-extension-products.mjs \ tools/extension-resource-inventory.mjs \ diff --git a/src/sdks/swift/tools/prepare-ios-broker-artifacts.sh b/src/sdks/swift/tools/prepare-ios-broker-artifacts.sh new file mode 100755 index 00000000..ad7beb02 --- /dev/null +++ b/src/sdks/swift/tools/prepare-ios-broker-artifacts.sh @@ -0,0 +1,575 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" +root="$(git -C "$(dirname "$script_path")" rev-parse --show-toplevel 2>/dev/null)" || { + echo "error: prepare-ios-broker-artifacts.sh must run inside the Oliphaunt checkout" >&2 + exit 1 +} +cd "$root" + +# Reuse the mobile runtime package contract rather than maintaining a second +# copy for the Swift spike. These files define template normalization, exact +# extension assets, static-registry metadata, and resource-tree validation. +. "$root/src/sdks/react-native/tools/expo-runner-common.sh" +. "$root/src/sdks/react-native/tools/expo-runner-workspace.sh" +. "$root/src/sdks/react-native/tools/mobile-extension-runtime.sh" +. "$root/src/sdks/react-native/tools/expo-runner-runtime-resources.sh" + +selected_extensions="$(oliphaunt_dev_normalize_mobile_extensions "vector,pg_trgm" "iOS")" +static_extensions="$(oliphaunt_dev_mobile_static_extensions_for_selection "$selected_extensions")" +broker_database_role="oliphaunt_broker" +broker_database_name="postgres" +[ "$selected_extensions" = "vector,pg_trgm" ] || \ + fail "broker artifact selection drifted: $selected_extensions" +[ "$static_extensions" = "vector,pg_trgm" ] || \ + fail "broker static-extension selection drifted: $static_extensions" + +artifact_platform="${OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM:-simulator}" +case "$artifact_platform" in + simulator) + default_artifact_root="$root/target/ios-native-broker-artifacts" + ;; + device) + default_artifact_root="$root/target/ios-native-broker-device-artifacts" + ;; + *) + fail "OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM must be simulator or device" + ;; +esac + +artifact_root_raw="${OLIPHAUNT_IOS_BROKER_ARTIFACT_ROOT:-$default_artifact_root}" +case "$artifact_root_raw" in + /*) ;; + *) artifact_root_raw="$root/$artifact_root_raw" ;; +esac +artifact_parent="$(dirname "$artifact_root_raw")" +artifact_name="$(basename "$artifact_root_raw")" +case "$artifact_name" in + ''|.|..) fail "unsafe broker artifact root: $artifact_root_raw" ;; +esac +mkdir -p "$artifact_parent" +artifact_parent="$(cd "$artifact_parent" && pwd -P)" +artifact_root="$artifact_parent/$artifact_name" +case "$artifact_root" in + /|"$root"|"$root/target") fail "refusing broad broker artifact root: $artifact_root" ;; +esac +[ ! -L "$artifact_root" ] || fail "broker artifact root must not be a symlink: $artifact_root" + +work_root="$artifact_root/work" +logs_dir="$artifact_root/logs" +scratch_root="$work_root/runtime-package" +xcframework_out="$artifact_root/liboliphaunt.xcframework" +resources_out="$artifact_root/runtime-resources" +environment_out="$artifact_root/broker-artifacts.env" +manifest_out="$artifact_root/manifest.properties" +mkdir -p "$artifact_root" "$work_root" "$logs_dir" "$scratch_root/logs" +[ ! -L "$xcframework_out" ] || fail "XCFramework output must not be a symlink: $xcframework_out" +[ ! -L "$resources_out" ] || fail "runtime-resource output must not be a symlink: $resources_out" + +for command_name in awk basename cp dirname find grep mkdir mktemp mv nm node otool \ + plutil rsync sed shasum sort tr wc xcodebuild xcrun; do + need_cmd "$command_name" +done +if [ "$artifact_platform" = "device" ]; then + need_cmd install_name_tool +fi +[ "$(uname -s)" = "Darwin" ] || fail "iOS broker artifacts require macOS" + +case "$artifact_platform" in + simulator) + native_root="${OLIPHAUNT_IOS_SIMULATOR_ROOT:-$root/target/liboliphaunt-ios-simulator}" + native_build_script="$root/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-simulator.sh" + native_platform_name="iOS simulator" + expected_library_platform="IOSSIMULATOR" + manifest_platform="ios-simulator" + check_log="$logs_dir/check-ios-simulator.log" + build_log="$logs_dir/build-ios-simulator.log" + ;; + device) + native_root="${OLIPHAUNT_IOS_DEVICE_ROOT:-$root/target/liboliphaunt-ios-device}" + native_build_script="$root/src/runtimes/liboliphaunt/native/bin/build-postgres18-ios-device.sh" + native_platform_name="iOS device" + expected_library_platform="IOS" + manifest_platform="ios-device" + check_log="$logs_dir/check-ios-device.log" + build_log="$logs_dir/build-ios-device.log" + ;; +esac +native_dylib="${OLIPHAUNT_IOS_BROKER_DYLIB:-$native_root/out/liboliphaunt.dylib}" +static_registry_source="${OLIPHAUNT_IOS_BROKER_STATIC_REGISTRY_SOURCE:-$native_root/out/liboliphaunt_mobile_static_registry.c}" +minimum_ios="${OLIPHAUNT_IOS_BROKER_MIN_VERSION:-26.0}" +printf '%s\n' "$minimum_ios" | grep -Eq '^[0-9]+([.][0-9]+){0,2}$' || \ + fail "OLIPHAUNT_IOS_BROKER_MIN_VERSION must be a numeric iOS version" +allow_native_builds="${OLIPHAUNT_IOS_BROKER_ALLOW_NATIVE_BUILD:-1}" +case "$allow_native_builds" in + 1|true|TRUE|yes|YES|on|ON) allow_native_builds=1 ;; + 0|false|FALSE|no|NO|off|OFF) allow_native_builds=0 ;; + *) fail "OLIPHAUNT_IOS_BROKER_ALLOW_NATIVE_BUILD must be boolean" ;; +esac + +run_native_builder() { + case "$artifact_platform" in + simulator) + env \ + OLIPHAUNT_IOS_SIMULATOR_ROOT="$native_root" \ + OLIPHAUNT_IOS_SIMULATOR_MIN_VERSION="$minimum_ios" \ + OLIPHAUNT_MOBILE_STATIC_EXTENSIONS="$static_extensions" \ + "$native_build_script" "$@" + ;; + device) + env \ + OLIPHAUNT_IOS_DEVICE_ROOT="$native_root" \ + OLIPHAUNT_IOS_MIN_VERSION="$minimum_ios" \ + OLIPHAUNT_MOBILE_STATIC_EXTENSIONS="$static_extensions" \ + "$native_build_script" "$@" + ;; + esac +} + +if [ "$native_dylib" = "$native_root/out/liboliphaunt.dylib" ]; then + if ! run_native_builder --check-current >"$check_log" 2>&1; then + [ "$allow_native_builds" = "1" ] || { + tail -80 "$check_log" >&2 || true + fail "$native_platform_name liboliphaunt is missing or stale and native builds are disabled" + } + echo "Preparing current $native_platform_name liboliphaunt (extensions=$static_extensions)..." >&2 + run_native_builder >"$build_log" 2>&1 || { + tail -120 "$build_log" >&2 || true + fail "failed to build $native_platform_name liboliphaunt" + } + fi +fi + +[ -f "$native_dylib" ] || fail "missing $native_platform_name dylib: $native_dylib" +[ -f "$static_registry_source" ] || fail "missing mobile static registry: $static_registry_source" +library_platform="$(xcrun vtool -show-build "$native_dylib" 2>/dev/null | awk '/platform / { print $2; exit }')" +[ "$library_platform" = "$expected_library_platform" ] || \ + fail "liboliphaunt platform is $library_platform, expected $expected_library_platform" +case "$(otool -D "$native_dylib" 2>/dev/null)" in + *"@rpath/liboliphaunt.dylib"*) ;; + *) fail "$native_platform_name dylib has an unexpected install name: $native_dylib" ;; +esac +if [ "$artifact_platform" = "device" ]; then + native_install_name="$(otool -D "$native_dylib" 2>/dev/null | sed -n '2{s/^[[:space:]]*//;s/[[:space:]]*$//;p;}')" + [ "$native_install_name" = "@rpath/liboliphaunt.dylib" ] || \ + fail "iOS device dylib has an unexpected install name: $native_install_name" + native_architectures="$(xcrun lipo -archs "$native_dylib" 2>/dev/null)" + [ "$native_architectures" = "arm64" ] || \ + fail "iOS device dylib architectures are $native_architectures, expected arm64" + native_minimum_ios="$(xcrun vtool -show-build "$native_dylib" 2>/dev/null | awk '/minos / { print $2; exit }')" + [ "$native_minimum_ios" = "$minimum_ios" ] || \ + fail "iOS device dylib minimum OS is $native_minimum_ios, expected $minimum_ios" +fi + +library_symbols="$(nm -g "$native_dylib" 2>/dev/null)" +case "$library_symbols" in + *"_liboliphaunt_selected_static_extensions"*) ;; + *) fail "$native_platform_name dylib has no selected-static-extension registry" ;; +esac +for extension in vector pg_trgm; do + module_stem="$(oliphaunt_mobile_static_extension_module_stem "$extension")" + symbol_prefix="$(oliphaunt_static_symbol_prefix "$module_stem")" + case "$library_symbols" in + *"_${symbol_prefix}_Pg_magic_func"*) ;; + *) fail "$native_platform_name dylib is missing static $extension symbols" ;; + esac +done + +native_headers="$root/src/runtimes/liboliphaunt/native/include" +[ -f "$native_headers/oliphaunt.h" ] || fail "missing public native header" +xcframework_stage="$(mktemp -d "$work_root/xcframework.XXXXXX")" +broker_template_source="" +broker_template_socket_dir="" +broker_template_pg_ctl="" +broker_template_server_started=0 +cleanup_stage() { + if [ "$broker_template_server_started" = "1" ] && + [ -n "$broker_template_pg_ctl" ] && + [ -n "$broker_template_source" ]; then + "$broker_template_pg_ctl" -D "$broker_template_source" -m immediate stop >/dev/null 2>&1 || true + fi + rm -rf "$xcframework_stage" + if [ -n "$broker_template_socket_dir" ]; then + rm -rf "$broker_template_socket_dir" + fi +} +trap cleanup_stage EXIT INT TERM + +validate_device_framework() { + local framework="$1" + local binary="$framework/liboliphaunt" + local framework_platform framework_architectures framework_install_name framework_minimum_ios + [ -d "$framework" ] || fail "missing iOS device framework: $framework" + [ -f "$framework/Info.plist" ] || fail "iOS device framework is missing Info.plist" + [ -f "$framework/Headers/oliphaunt.h" ] || fail "iOS device framework is missing oliphaunt.h" + [ -f "$framework/Modules/module.modulemap" ] || fail "iOS device framework is missing its module map" + [ -f "$binary" ] || fail "iOS device framework is missing its executable" + plutil -lint "$framework/Info.plist" >/dev/null || \ + fail "iOS device framework has an invalid Info.plist" + [ "$(plutil -extract CFBundleExecutable raw -o - "$framework/Info.plist")" = "liboliphaunt" ] || \ + fail "iOS device framework has an unexpected CFBundleExecutable" + [ "$(plutil -extract CFBundlePackageType raw -o - "$framework/Info.plist")" = "FMWK" ] || \ + fail "iOS device framework has an unexpected CFBundlePackageType" + [ "$(plutil -extract CFBundleSupportedPlatforms.0 raw -o - "$framework/Info.plist")" = "iPhoneOS" ] || \ + fail "iOS device framework has an unexpected supported platform" + [ "$(plutil -extract MinimumOSVersion raw -o - "$framework/Info.plist")" = "$minimum_ios" ] || \ + fail "iOS device framework has an unexpected minimum OS version" + framework_platform="$(xcrun vtool -show-build "$binary" 2>/dev/null | awk '/platform / { print $2; exit }')" + [ "$framework_platform" = "IOS" ] || \ + fail "iOS device framework binary platform is $framework_platform, expected IOS" + framework_minimum_ios="$(xcrun vtool -show-build "$binary" 2>/dev/null | awk '/minos / { print $2; exit }')" + [ "$framework_minimum_ios" = "$minimum_ios" ] || \ + fail "iOS device framework binary minimum OS is $framework_minimum_ios, expected $minimum_ios" + framework_architectures="$(xcrun lipo -archs "$binary" 2>/dev/null)" + [ "$framework_architectures" = "arm64" ] || \ + fail "iOS device framework binary architectures are $framework_architectures, expected arm64" + framework_install_name="$(otool -D "$binary" 2>/dev/null | sed -n '2{s/^[[:space:]]*//;s/[[:space:]]*$//;p;}')" + [ "$framework_install_name" = "@rpath/liboliphaunt.framework/liboliphaunt" ] || \ + fail "iOS device framework has an unexpected install name: $framework_install_name" +} + +case "$artifact_platform" in + simulator) + echo "Creating simulator-only liboliphaunt XCFramework..." >&2 + xcodebuild -create-xcframework \ + -library "$native_dylib" \ + -headers "$native_headers" \ + -output "$xcframework_stage/liboliphaunt.xcframework" \ + >"$logs_dir/create-xcframework.log" 2>&1 || { + tail -120 "$logs_dir/create-xcframework.log" >&2 || true + fail "failed to create simulator liboliphaunt XCFramework" + } + ;; + device) + runtime_version_file="$root/src/runtimes/liboliphaunt/native/VERSION" + [ -f "$runtime_version_file" ] || fail "missing liboliphaunt version file" + runtime_version="$(tr -d '\r\n' <"$runtime_version_file")" + printf '%s\n' "$runtime_version" | grep -Eq '^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)$' || \ + fail "liboliphaunt VERSION must be stable x.y.z, got: $runtime_version" + device_framework="$xcframework_stage/liboliphaunt.framework" + mkdir -p "$device_framework/Headers" "$device_framework/Modules" + cp "$native_dylib" "$device_framework/liboliphaunt" + install_name_tool -id \ + "@rpath/liboliphaunt.framework/liboliphaunt" \ + "$device_framework/liboliphaunt" + rsync -a --delete "$native_headers/" "$device_framework/Headers/" + cat >"$device_framework/Modules/module.modulemap" <<'MODULEMAP' +framework module liboliphaunt { + umbrella header "oliphaunt.h" + export * + module * { export * } +} +MODULEMAP + cat >"$device_framework/Info.plist" < + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + liboliphaunt + CFBundleIdentifier + dev.oliphaunt.liboliphaunt + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + liboliphaunt + CFBundlePackageType + FMWK + CFBundleShortVersionString + $runtime_version + CFBundleSupportedPlatforms + + iPhoneOS + + CFBundleVersion + 1 + MinimumOSVersion + $minimum_ios + UIDeviceFamily + + 1 + + + +FRAMEWORK_PLIST + validate_device_framework "$device_framework" + echo "Creating device-only liboliphaunt XCFramework..." >&2 + xcodebuild -create-xcframework \ + -framework "$device_framework" \ + -output "$xcframework_stage/liboliphaunt.xcframework" \ + >"$logs_dir/create-xcframework.log" 2>&1 || { + tail -120 "$logs_dir/create-xcframework.log" >&2 || true + fail "failed to create device liboliphaunt XCFramework" + } + ;; +esac +rm -rf "$xcframework_out" +mv "$xcframework_stage/liboliphaunt.xcframework" "$xcframework_out" + +xcframework_info="$xcframework_out/Info.plist" +[ -f "$xcframework_info" ] || fail "XCFramework is missing Info.plist" +slice_identifier="$(plutil -extract AvailableLibraries.0.LibraryIdentifier raw -o - "$xcframework_info")" +slice_platform="$(plutil -extract AvailableLibraries.0.SupportedPlatform raw -o - "$xcframework_info")" +slice_variant="$(plutil -extract AvailableLibraries.0.SupportedPlatformVariant raw -o - "$xcframework_info" 2>/dev/null || true)" +slice_architecture="$(plutil -extract AvailableLibraries.0.SupportedArchitectures.0 raw -o - "$xcframework_info")" +slice_library_path="$(plutil -extract AvailableLibraries.0.LibraryPath raw -o - "$xcframework_info")" +case "$artifact_platform" in + simulator) + [ "$slice_platform:$slice_variant:$slice_architecture" = "ios:simulator:arm64" ] || \ + fail "unexpected XCFramework slice: $slice_platform/$slice_variant/$slice_architecture" + ;; + device) + [ "$slice_platform:$slice_variant:$slice_architecture" = "ios::arm64" ] || \ + fail "unexpected XCFramework slice: $slice_platform/${slice_variant:-none}/$slice_architecture" + if plutil -extract AvailableLibraries.0.SupportedArchitectures.1 raw -o - \ + "$xcframework_info" >/dev/null 2>&1; then + fail "iOS device XCFramework unexpectedly contains multiple architectures" + fi + [ "$slice_library_path" = "liboliphaunt.framework" ] || \ + fail "iOS device XCFramework must contain liboliphaunt.framework, got: $slice_library_path" + ;; +esac +if plutil -extract AvailableLibraries.1.LibraryIdentifier raw -o - "$xcframework_info" >/dev/null 2>&1; then + fail "$artifact_platform broker XCFramework unexpectedly contains multiple slices" +fi +case "$artifact_platform" in + simulator) + packaged_native_library="$xcframework_out/$slice_identifier/$slice_library_path" + [ -f "$packaged_native_library" ] || \ + fail "XCFramework dylib is missing: $packaged_native_library" + [ "$(shasum -a 256 "$packaged_native_library" | awk '{ print $1 }')" = \ + "$(shasum -a 256 "$native_dylib" | awk '{ print $1 }')" ] || \ + fail "XCFramework dylib differs from the validated simulator artifact" + ;; + device) + packaged_framework="$xcframework_out/$slice_identifier/$slice_library_path" + validate_device_framework "$packaged_framework" + packaged_native_library="$packaged_framework/liboliphaunt" + [ "$(shasum -a 256 "$packaged_native_library" | awk '{ print $1 }')" = \ + "$(shasum -a 256 "$device_framework/liboliphaunt" | awk '{ print $1 }')" ] || \ + fail "XCFramework framework binary differs from the validated device artifact" + ;; +esac + +runtime_source="${OLIPHAUNT_IOS_BROKER_RUNTIME_DIR:-}" +if [ -z "$runtime_source" ]; then + export OLIPHAUNT_EXPO_ALLOW_NATIVE_BUILDS="$allow_native_builds" + runtime_source="$(ensure_host_runtime_assets)" +fi +[ -f "$runtime_source/share/postgresql/postgres.bki" ] || \ + fail "runtime source is missing postgres.bki: $runtime_source" + +mobile_postgres_build_dir="${OLIPHAUNT_IOS_BROKER_POSTGRES_SOURCE_DIR:-}" +if [ -z "$mobile_postgres_build_dir" ]; then + for candidate in \ + "$native_root/postgresql-18.4" \ + "$(host_runtime_work_root)/postgresql-18.4"; do + if [ -d "$candidate/contrib/pg_trgm" ]; then + mobile_postgres_build_dir="$candidate" + break + fi + done +fi +[ -d "$mobile_postgres_build_dir/contrib/pg_trgm" ] || \ + fail "missing pinned PostgreSQL 18.4 source tree for pg_trgm resources" +export OLIPHAUNT_MOBILE_POSTGRES_BUILD_DIR="$mobile_postgres_build_dir" + +mobile_template_initdb="${OLIPHAUNT_IOS_BROKER_INITDB:-$runtime_source/bin/initdb}" +[ -x "$mobile_template_initdb" ] || fail "missing PostgreSQL 18 initdb: $mobile_template_initdb" +initdb_version="$($mobile_template_initdb --version 2>/dev/null || true)" +case "$initdb_version" in + *" 18.4"*) ;; + *) fail "broker template requires PostgreSQL 18.4 initdb, got: $initdb_version" ;; +esac + +wal_segsize_mb="${OLIPHAUNT_IOS_BROKER_WAL_SEGSIZE_MB:-16}" +case "$wal_segsize_mb" in + ''|*[!0-9]*) fail "OLIPHAUNT_IOS_BROKER_WAL_SEGSIZE_MB must be an integer" ;; +esac +template_source="$( + find_latest_mobile_pgdata \ + iOS \ + "${OLIPHAUNT_IOS_BROKER_TEMPLATE_PGDATA_DIR:-}" \ + OLIPHAUNT_IOS_BROKER_TEMPLATE_PGDATA_DIR \ + OLIPHAUNT_IOS_BROKER_INITDB +)" +[ "$(tr -d '\r\n' <"$template_source/PG_VERSION")" = "18" ] || \ + fail "broker template PGDATA is not PostgreSQL 18: $template_source" + +# Seed a non-bootstrap login for the extension worker. WorkerCore uses this +# role only while no host data channel exists: it installs the selected static +# extensions, grants its narrow checkpoint capability, and permanently drops +# SUPERUSER before Ready. PostgreSQL refuses to demote initdb's bootstrap role, +# so authenticating the host-visible session as postgres cannot satisfy the +# broker's PGDATA-confidentiality boundary. +for broker_template_tool in pg_ctl psql; do + [ -x "$runtime_source/bin/$broker_template_tool" ] || \ + fail "broker template role provisioning requires $runtime_source/bin/$broker_template_tool" +done +broker_template_source="$work_root/broker-template-pgdata" +case "$broker_template_source" in + "$template_source") fail "broker template staging path overlaps its source" ;; +esac +rm -rf "$broker_template_source" +mkdir -p "$broker_template_source" +rsync -a --delete \ + --exclude postmaster.pid \ + --exclude postmaster.opts \ + --exclude 'pg_stat_tmp/*' \ + "$template_source/" "$broker_template_source/" +rm -f "$broker_template_source/postmaster.pid" "$broker_template_source/postmaster.opts" +normalize_template_pgdata "$broker_template_source" + +broker_template_socket_dir="$(mktemp -d /tmp/oliphaunt-broker-template.XXXXXX)" +chmod 700 "$broker_template_socket_dir" +broker_template_pg_ctl="$runtime_source/bin/pg_ctl" +broker_template_log="$logs_dir/prepare-broker-template.log" +broker_template_control_log="$logs_dir/prepare-broker-template-control.log" +"$broker_template_pg_ctl" \ + -D "$broker_template_source" \ + -l "$broker_template_log" \ + -o "-k $broker_template_socket_dir -h ''" \ + -w start >"$broker_template_control_log" 2>&1 || { + tail -120 "$broker_template_control_log" >&2 || true + tail -120 "$broker_template_log" >&2 || true + fail "failed to start broker template PostgreSQL for role provisioning" + } +broker_template_server_started=1 +broker_template_psql=( + "$runtime_source/bin/psql" + -X + -A + -t + -F '|' + -v ON_ERROR_STOP=1 + -h "$broker_template_socket_dir" + -U postgres + -d "$broker_database_name" +) +broker_role_state="$( + "${broker_template_psql[@]}" -c \ + "SELECT rolsuper, rolcanlogin FROM pg_roles WHERE rolname = '$broker_database_role'" +)" +if [ -z "$broker_role_state" ]; then + "${broker_template_psql[@]}" -c \ + "CREATE ROLE $broker_database_role LOGIN SUPERUSER" >/dev/null + broker_role_state="$( + "${broker_template_psql[@]}" -c \ + "SELECT rolsuper, rolcanlogin FROM pg_roles WHERE rolname = '$broker_database_role'" + )" +fi +[ "$broker_role_state" = "t|t" ] || \ + fail "broker template role must be a login superuser before first WorkerCore open" +"${broker_template_psql[@]}" -c \ + "ALTER ROLE $broker_database_role SET search_path TO \"\$user\", public" >/dev/null +[ "$( + "${broker_template_psql[@]}" -c \ + "SELECT rolconfig @> ARRAY['search_path=\"\$user\", public'] FROM pg_roles WHERE rolname = '$broker_database_role'" +)" = "t" ] || fail "broker template role lost its durable restricted search_path" +[ "$( + "${broker_template_psql[@]}" -c \ + "SELECT rolsuper FROM pg_roles WHERE rolname = 'postgres'" +)" = "t" ] || fail "broker template lost its inaccessible bootstrap superuser" +"$broker_template_pg_ctl" -D "$broker_template_source" -m fast -w stop \ + >>"$broker_template_control_log" 2>&1 || { + tail -120 "$broker_template_control_log" >&2 || true + tail -120 "$broker_template_log" >&2 || true + fail "failed to stop broker template PostgreSQL after role provisioning" + } +broker_template_server_started=0 +rm -rf "$broker_template_socket_dir" +broker_template_socket_dir="" +normalize_template_pgdata "$broker_template_source" +template_source="$broker_template_source" + +echo "Preparing validated mobile runtime resources (extensions=$selected_extensions)..." >&2 +prepare_mobile_runtime_resource_package \ + iOS \ + "$runtime_source" \ + "$template_source" \ + "$static_registry_source" \ + "$selected_extensions" \ + "${OLIPHAUNT_IOS_BROKER_REPACKAGE_RESOURCES:-0}" \ + "$resources_out" \ + >"$logs_dir/prepare-runtime-resources.log" + +runtime_manifest="$resources_out/oliphaunt/runtime/manifest.properties" +template_manifest="$resources_out/oliphaunt/template-pgdata/manifest.properties" +static_manifest="$resources_out/oliphaunt/static-registry/manifest.properties" +runtime_files="$resources_out/oliphaunt/runtime/files" +template_files="$resources_out/oliphaunt/template-pgdata/files" +for required_file in \ + "$runtime_manifest" \ + "$template_manifest" \ + "$static_manifest" \ + "$runtime_files/share/postgresql/postgres.bki" \ + "$runtime_files/share/postgresql/extension/vector.control" \ + "$runtime_files/share/postgresql/extension/pg_trgm.control" \ + "$resources_out/oliphaunt/static-registry/oliphaunt_static_registry.c" \ + "$template_files/PG_VERSION"; do + [ -f "$required_file" ] || fail "prepared broker resource is missing: $required_file" +done +case "$(grep '^brokerDatabaseRole=' "$template_manifest" 2>/dev/null || true)" in + '') printf 'brokerDatabaseRole=%s\n' "$broker_database_role" >>"$template_manifest" ;; + "brokerDatabaseRole=$broker_database_role") ;; + *) fail "prepared broker template manifest has an unexpected database role" ;; +esac +grep -Fqx "brokerDatabaseRole=$broker_database_role" "$template_manifest" || \ + fail "prepared broker template manifest omitted its restricted database role" +for extension in vector pg_trgm; do + find "$runtime_files/share/postgresql/extension" -maxdepth 1 -type f \ + -name "$extension--*.sql" -print -quit | grep -q . || \ + fail "prepared broker resources are missing $extension SQL" +done +[ -f "$runtime_files/share/postgresql/extension/pg_trgm--1.3.sql" ] || \ + fail "prepared broker resources are missing an installable pg_trgm base version" +if grep -Eiq \ + '^[[:space:]]*(CREATE[[:space:]]+EXTENSION[[:space:]]+pg_trgm|\\copy([[:space:]]|$))' \ + "$runtime_files"/share/postgresql/extension/pg_trgm--*.sql; then + fail "prepared broker resources contain the pg_trgm regression script instead of extension install SQL" +fi +grep -Fqx "selectedExtensions=pg_trgm,vector" "$runtime_manifest" || \ + fail "runtime manifest did not preserve exact extension selection" +grep -Fqx "registeredExtensions=$selected_extensions" "$static_manifest" || \ + fail "static registry did not preserve exact extension selection" +grep -Fqx "nativeModuleStems=vector,pg_trgm" "$static_manifest" || \ + fail "static registry did not preserve exact native-module stems" +if find "$resources_out/oliphaunt" -type f \( -name '*.dylib' -o -name '*.so' \) -print -quit | grep -q .; then + fail "broker runtime resources unexpectedly contain dynamic extension modules" +fi + +xcframework_sha256="$(directory_fingerprint "$xcframework_out")" +resources_sha256="$(directory_fingerprint "$resources_out/oliphaunt")" +dylib_sha256="$(shasum -a 256 "$native_dylib" | awk '{ print $1 }')" +initdb_sha256="$(shasum -a 256 "$mobile_template_initdb" | awk '{ print $1 }')" +cat >"$manifest_out" <"$environment_out" +printf 'export OLIPHAUNT_IOS_BROKER_RESOURCES=%q\n' "$resources_out" >>"$environment_out" +if [ "$artifact_platform" = "device" ]; then + printf 'export OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM=device\n' >>"$environment_out" +fi + +trap - EXIT INT TERM +cleanup_stage +printf 'OLIPHAUNT_IOS_BROKER_ARTIFACTS_PASS xcframework=%s resources=%s env=%s manifest=%s\n' \ + "$xcframework_out" "$resources_out" "$environment_out" "$manifest_out" diff --git a/src/sdks/swift/tools/quarantine-ios-broker-simulator-storage.sh b/src/sdks/swift/tools/quarantine-ios-broker-simulator-storage.sh new file mode 100644 index 00000000..30ff2ee5 --- /dev/null +++ b/src/sdks/swift/tools/quarantine-ios-broker-simulator-storage.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +safe_bundle_identifier() { + case "$1" in + ''|*[!A-Za-z0-9.-]*) return 1 ;; + *) return 0 ;; + esac +} + +safe_process_name() { + case "$1" in + ''|*[!A-Za-z0-9._-]*) return 1 ;; + *) return 0 ;; + esac +} + +write_refusal() { + local reason="$1" + local detail="${2:-}" + { + printf 'status=refused\n' + printf 'reason=%s\n' "$reason" + [ -z "$detail" ] || printf 'detail=%s\n' "$detail" + } >"$report_path" +} + +if [ "$#" -ne 8 ]; then + fail "usage: $0 UDID HOST_BUNDLE_ID EXTENSION_BUNDLE_ID APP_PRODUCT HOST_EXECUTABLE EXTENSION_PRODUCT EXTENSION_EXECUTABLE REPORT" +fi + +selected_udid="$1" +app_bundle_id="$2" +extension_bundle_id="$3" +app_product_name="$4" +host_executable="$5" +extension_product_name="$6" +extension_executable="$7" +report_path="${8:-}" + +case "$selected_udid" in + ''|*[!A-Fa-f0-9-]*) fail "unsafe simulator UDID" ;; +esac +safe_bundle_identifier "$app_bundle_id" || fail "unsafe host bundle identifier" +safe_bundle_identifier "$extension_bundle_id" || fail "unsafe extension bundle identifier" +safe_process_name "$app_product_name" || fail "unsafe host product name" +safe_process_name "$host_executable" || fail "unsafe host executable name" +safe_process_name "$extension_product_name" || fail "unsafe extension product name" +safe_process_name "$extension_executable" || fail "unsafe extension executable name" +[ -n "$report_path" ] || fail "storage quarantine report path is empty" +mkdir -p "$(dirname "$report_path")" +: >"$report_path" + +for command_name in ps ruby sleep xcrun; do + command -v "$command_name" >/dev/null 2>&1 || fail "missing required command: $command_name" +done + +process_wait_attempts="${OLIPHAUNT_IOS_BROKER_QUARANTINE_PROCESS_WAIT_ATTEMPTS:-50}" +case "$process_wait_attempts" in + ''|*[!0-9]*) fail "quarantine process wait attempts must be a positive integer" ;; +esac +[ "$process_wait_attempts" -gt 0 ] || \ + fail "quarantine process wait attempts must be positive" +[ "$process_wait_attempts" -ge 3 ] || \ + fail "quarantine process wait attempts must be at least three" + +simulator_data_root="$( + xcrun simctl getenv "$selected_udid" SIMULATOR_SHARED_RESOURCES_DIRECTORY 2>/dev/null || true +)" +if [ -z "$simulator_data_root" ]; then + write_refusal "data-root-unavailable" + fail "selected simulator data root is unavailable" +fi + +target_processes() { + local snapshot matches + if ! snapshot="$(ps -axo pid=,command= 2>/dev/null)"; then + write_refusal "process-list-unavailable" + fail "could not inspect simulator target processes" + fi + matches="$(printf '%s\n' "$snapshot" | \ + OLIPHAUNT_QUARANTINE_DATA_ROOT="$simulator_data_root" \ + OLIPHAUNT_QUARANTINE_APP_PRODUCT="$app_product_name" \ + OLIPHAUNT_QUARANTINE_HOST_EXECUTABLE="$host_executable" \ + OLIPHAUNT_QUARANTINE_EXTENSION_PRODUCT="$extension_product_name" \ + OLIPHAUNT_QUARANTINE_EXTENSION_EXECUTABLE="$extension_executable" \ + ruby -e ' +data_root = ENV.fetch("OLIPHAUNT_QUARANTINE_DATA_ROOT") +app_product = ENV.fetch("OLIPHAUNT_QUARANTINE_APP_PRODUCT") +host_executable = ENV.fetch("OLIPHAUNT_QUARANTINE_HOST_EXECUTABLE") +extension_product = ENV.fetch("OLIPHAUNT_QUARANTINE_EXTENSION_PRODUCT") +extension_executable = ENV.fetch("OLIPHAUNT_QUARANTINE_EXTENSION_EXECUTABLE") + +bundle_root = "#{data_root}/Containers/Bundle/Application/" +host_needle = "/#{app_product}.app/#{host_executable}" +extension_needle = + "/#{app_product}.app/Extensions/#{extension_product}.appex/#{extension_executable}" + +def contains_executable?(command, needle) + offset = command.index(needle) + return false unless offset + + boundary = offset + needle.bytesize + boundary == command.bytesize || command.getbyte(boundary) == 0x20 +end + +STDIN.each_line do |line| + line.chomp! + match = line.match(/\A\s*(\d+)\s+(.+)\z/) + next unless match + + pid = match[1] + command = match[2] + next unless command.start_with?(bundle_root) + + if contains_executable?(command, extension_needle) + puts "extension:#{pid}" + elsif contains_executable?(command, host_needle) + puts "host:#{pid}" + end +end +' + )" + printf '%s' "$matches" +} + +wait_for_target_processes_to_exit() { + local attempt matches quiet_observations=0 + for ((attempt = 1; attempt <= process_wait_attempts; attempt++)); do + matches="$(target_processes)" + if [ -z "$matches" ]; then + quiet_observations=$((quiet_observations + 1)) + if [ "$quiet_observations" -ge 3 ]; then + return 0 + fi + else + quiet_observations=0 + fi + if [ "$attempt" -lt "$process_wait_attempts" ]; then + sleep 0.1 + fi + done + + matches="$(target_processes)" + write_refusal "active-target-processes" "$(printf '%s' "$matches" | tr '\n' ',')" + fail "target host or extension process remained active" +} + +# A missing/not-running target is expected, so command status is not evidence. +# The bounded process snapshots below are the authoritative termination check. +xcrun simctl terminate "$selected_udid" "$app_bundle_id" >/dev/null 2>&1 || true +xcrun simctl terminate "$selected_udid" "$extension_bundle_id" >/dev/null 2>&1 || true +wait_for_target_processes_to_exit +xcrun simctl uninstall "$selected_udid" "$app_bundle_id" >/dev/null 2>&1 || true +xcrun simctl terminate "$selected_udid" "$extension_bundle_id" >/dev/null 2>&1 || true +wait_for_target_processes_to_exit + +if ! ruby -rjson -rdigest -rsecurerandom -rtime \ + - "$selected_udid" "$simulator_data_root" "$report_path" <<'RUBY' +udid, data_root, report_path = ARGV +stale_digest = "ios-native-broker-spike-v1" +current_digest = "ios-native-broker-spike-v2-restricted-role" + +def write_report(path, values) + temporary = "#{path}.tmp-#{Process.pid}-#{SecureRandom.hex(4)}" + File.open(temporary, "wb", 0o600) do |file| + values.each { |key, value| file.write("#{key}=#{value}\n") } + end + File.rename(temporary, path) +ensure + File.unlink(temporary) if defined?(temporary) && File.exist?(temporary) +end + +def refuse(report_path, reason) + write_report(report_path, status: "refused", reason: reason) + warn "error: simulator storage quarantine refused: #{reason}" + exit 1 +end + +def no_symlink_ancestry?(path) + return false unless path.start_with?("/") && File.expand_path(path) == path + + cursor = "/" + path.split("/").reject(&:empty?).each do |component| + cursor = File.join(cursor, component) + begin + stat = File.lstat(cursor) + rescue Errno::ENOENT + break + end + return false if stat.symlink? + end + true +end + +unless no_symlink_ancestry?(data_root) && File.directory?(data_root) + refuse(report_path, "unsafe-data-root") +end + +begin + canonical_data_root = File.realpath(data_root) +rescue SystemCallError + refuse(report_path, "unsafe-data-root") +end +unless canonical_data_root == data_root + refuse(report_path, "unsafe-data-root") +end +components = canonical_data_root.split("/").reject(&:empty?) +unless components.last(4) == ["CoreSimulator", "Devices", udid, "data"] + refuse(report_path, "unexpected-data-root") +end + +root = File.join(canonical_data_root, "Library", "Application Support", "Oliphaunt", "default") +unless no_symlink_ancestry?(root) + refuse(report_path, "unsafe-broker-root") +end +unless File.exist?(root) + write_report(report_path, status: "absent", root: "default") + exit 0 +end +unless File.directory?(root) && File.realpath(root) == root + refuse(report_path, "unsafe-broker-root") +end + +parent = File.dirname(root) +lock_suffix = Digest::SHA256.hexdigest(root)[0, 32] +lock_path = File.join(parent, ".oliphaunt-root-#{lock_suffix}.lock") + +File.open(lock_path, File::RDWR | File::CREAT, 0o600) do |lock| + unless lock.flock(File::LOCK_EX | File::LOCK_NB) + refuse(report_path, "root-lock-busy") + end + + begin + root_stat = File.lstat(root) + manifest = JSON.parse(File.binread(File.join(root, "manifest.json"))) + pg_version = File.binread(File.join(root, "pgdata", "PG_VERSION")).strip + rescue JSON::ParserError, SystemCallError + refuse(report_path, "unrecognized-manifest") + end + + expected_common = { + "formatVersion" => 1, + "cABIVersion" => 6, + "liboliphauntVersion" => "0.1.1", + "postgresMajorVersion" => 18, + "selectedPostgresExtensions" => ["pg_trgm", "vector"], + "dataProtectionPolicy" => "completeUntilFirstUserAuthentication", + } + common_matches = expected_common.all? { |key, value| manifest[key] == value } + root_uuid = manifest["rootUUID"] + common_matches &&= root_uuid.is_a?(String) && root_uuid.match?( + /\A[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}\z/ + ) + common_matches &&= pg_version == "18" + refuse(report_path, "unrecognized-manifest") unless common_matches + + digest = manifest["startupConfigurationDigest"] + if digest == current_digest + write_report( + report_path, + status: "retained-current", + root: "default", + startupConfigurationDigest: current_digest + ) + exit 0 + end + refuse(report_path, "unrecognized-manifest") unless digest == stale_digest + + current_stat = File.lstat(root) + unless current_stat.dev == root_stat.dev && current_stat.ino == root_stat.ino + refuse(report_path, "broker-root-changed") + end + + quarantine_leaf = + ".oliphaunt-quarantine-default-#{Time.now.utc.strftime("%Y%m%dT%H%M%SZ")}-" \ + "#{Process.pid}-#{SecureRandom.hex(4)}" + quarantine = File.join(parent, quarantine_leaf) + refuse(report_path, "quarantine-collision") if File.exist?(quarantine) + File.rename(root, quarantine) + write_report( + report_path, + status: "quarantined", + root: "default", + startupConfigurationDigest: stale_digest, + quarantineLeaf: quarantine_leaf + ) +end +RUBY +then + fail "simulator storage quarantine validation failed" +fi diff --git a/src/sdks/swift/tools/quarantine-ios-broker-simulator-storage.test.sh b/src/sdks/swift/tools/quarantine-ios-broker-simulator-storage.test.sh new file mode 100644 index 00000000..21c49df1 --- /dev/null +++ b/src/sdks/swift/tools/quarantine-ios-broker-simulator-storage.test.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +helper="$script_dir/quarantine-ios-broker-simulator-storage.sh" +test_root="$(mktemp -d /private/tmp/oliphaunt-simulator-quarantine.XXXXXX)" +stub_bin="$test_root/bin" +udid="11111111-2222-3333-4444-555555555555" +data_root="$test_root/CoreSimulator/Devices/$udid/data" +report="$test_root/report.txt" +xcrun_log="$test_root/xcrun.log" +lock_pid="" + +cleanup() { + if [ -n "$lock_pid" ] && kill -0 "$lock_pid" 2>/dev/null; then + kill "$lock_pid" 2>/dev/null || true + wait "$lock_pid" 2>/dev/null || true + fi + rm -rf "$test_root" +} +trap cleanup EXIT INT TERM + +fail() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +mkdir -p "$stub_bin" +cat >"$stub_bin/xcrun" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$TEST_XCRUN_LOG" +if [ "${1:-}" = "simctl" ] && [ "${2:-}" = "getenv" ]; then + printf '%s\n' "$TEST_SIMULATOR_DATA_ROOT" +fi +STUB +cat >"$stub_bin/ps" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +case "${TEST_PS_MODE:-inactive}" in + inactive) ;; + active) + printf '101 %s/Containers/Bundle/Application/ABC/OliphauntBrokerSpike.app/OliphauntBrokerSpike\n' \ + "$TEST_SIMULATOR_DATA_ROOT" + printf '102 %s/Containers/Bundle/Application/ABC/OliphauntBrokerSpike.app/Extensions/BrokerAppExtension.appex/BrokerAppExtension -LaunchArguments value\n' \ + "$TEST_SIMULATOR_DATA_ROOT" + ;; + unrelated) + printf '103 /private/tmp/unrelated/OliphauntBrokerSpike.app/OliphauntBrokerSpike\n' + ;; + *) exit 2 ;; +esac +STUB +cat >"$stub_bin/sleep" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB +chmod +x "$stub_bin/xcrun" "$stub_bin/ps" "$stub_bin/sleep" + +broker_parent() { + printf '%s/Library/Application Support/Oliphaunt\n' "$1" +} + +broker_root() { + printf '%s/default\n' "$(broker_parent "$1")" +} + +write_manifest() { + local target_data_root="$1" + local digest="$2" + local target_root + target_root="$(broker_root "$target_data_root")" + mkdir -p "$target_root/pgdata" + printf '18\n' >"$target_root/pgdata/PG_VERSION" + ruby -rjson -e ' + digest, path = ARGV + manifest = { + "cABIVersion" => 6, + "dataProtectionPolicy" => "completeUntilFirstUserAuthentication", + "formatVersion" => 1, + "liboliphauntVersion" => "0.1.1", + "postgresMajorVersion" => 18, + "rootUUID" => "A6237639-3166-43A8-91D5-5ABCE2A04187", + "selectedPostgresExtensions" => ["pg_trgm", "vector"], + "startupConfigurationDigest" => digest, + } + File.binwrite(path, JSON.generate(manifest)) + ' "$digest" "$target_root/manifest.json" +} + +reset_case() { + rm -rf "$data_root" + mkdir -p "$data_root" + : >"$report" + : >"$xcrun_log" +} + +run_helper() { + local process_mode="${1:-inactive}" + PATH="$stub_bin:$PATH" \ + TEST_XCRUN_LOG="$xcrun_log" \ + TEST_SIMULATOR_DATA_ROOT="$data_root" \ + TEST_PS_MODE="$process_mode" \ + OLIPHAUNT_IOS_BROKER_QUARANTINE_PROCESS_WAIT_ATTEMPTS=3 \ + bash "$helper" \ + "$udid" \ + dev.oliphaunt.brokerspike \ + dev.oliphaunt.brokerspike.extension \ + OliphauntBrokerSpike \ + OliphauntBrokerSpike \ + BrokerAppExtension \ + BrokerAppExtension \ + "$report" +} + +reset_case +write_manifest "$data_root" "ios-native-broker-spike-v1" +run_helper unrelated +stale_root="$(broker_root "$data_root")" +stale_parent="$(broker_parent "$data_root")" +[ ! -e "$stale_root" ] || fail "recognized stale root was not quarantined" +quarantine_count="$( + find "$stale_parent" -mindepth 1 -maxdepth 1 -type d \ + -name '.oliphaunt-quarantine-default-*' | wc -l | tr -d '[:space:]' +)" +[ "$quarantine_count" = "1" ] || fail "stale root was not renamed exactly once" +grep -Fqx 'status=quarantined' "$report" || fail "stale report did not record quarantine" +grep -Fqx 'startupConfigurationDigest=ios-native-broker-spike-v1' "$report" || \ + fail "stale report lost the recognized digest" +if grep -Fq "$test_root" "$report"; then + fail "storage report exposed an absolute host path" +fi +grep -Fqx "simctl terminate $udid dev.oliphaunt.brokerspike" "$xcrun_log" || \ + fail "host termination was not attempted" +grep -Fqx "simctl terminate $udid dev.oliphaunt.brokerspike.extension" "$xcrun_log" || \ + fail "extension termination was not attempted" +grep -Fqx "simctl uninstall $udid dev.oliphaunt.brokerspike" "$xcrun_log" || \ + fail "host uninstall was not attempted" + +reset_case +write_manifest "$data_root" "ios-native-broker-spike-v2-restricted-role" +run_helper +[ -d "$(broker_root "$data_root")" ] || fail "current root was moved" +grep -Fqx 'status=retained-current' "$report" || fail "current root was not retained" + +reset_case +write_manifest "$data_root" "unknown-fixture-digest" +if run_helper >"$test_root/unknown.stdout" 2>"$test_root/unknown.stderr"; then + fail "unknown root was accepted" +fi +[ -d "$(broker_root "$data_root")" ] || fail "unknown root was moved" +grep -Fqx 'reason=unrecognized-manifest' "$report" || \ + fail "unknown root refusal was not recorded" + +reset_case +write_manifest "$data_root" "ios-native-broker-spike-v1" +root="$(broker_root "$data_root")" +parent="$(broker_parent "$data_root")" +lock_suffix="$(printf '%s' "$root" | shasum -a 256 | awk '{ print substr($1, 1, 32) }')" +lock_path="$parent/.oliphaunt-root-$lock_suffix.lock" +lock_ready="$test_root/lock-ready" +ruby -e ' + lock_path, ready = ARGV + File.open(lock_path, File::RDWR | File::CREAT, 0o600) do |file| + abort("lock failed") unless file.flock(File::LOCK_EX | File::LOCK_NB) + File.binwrite(ready, "ready") + sleep 30 + end +' "$lock_path" "$lock_ready" & +lock_pid=$! +for _ in $(seq 1 100); do + [ -f "$lock_ready" ] && break + /bin/sleep 0.01 +done +[ -f "$lock_ready" ] || fail "lock holder did not start" +if run_helper >"$test_root/lock.stdout" 2>"$test_root/lock.stderr"; then + fail "busy native root lock was ignored" +fi +[ -d "$root" ] || fail "locked root was moved" +grep -Fqx 'reason=root-lock-busy' "$report" || fail "busy lock refusal was not recorded" +kill "$lock_pid" +wait "$lock_pid" 2>/dev/null || true +lock_pid="" + +reset_case +write_manifest "$data_root" "ios-native-broker-spike-v1" +if run_helper active >"$test_root/active.stdout" 2>"$test_root/active.stderr"; then + fail "active target processes were ignored" +fi +[ -d "$(broker_root "$data_root")" ] || fail "active-process root was moved" +grep -Fqx 'reason=active-target-processes' "$report" || \ + fail "active target process refusal was not recorded" + +real_root="$test_root/real-root" +symlink_root="$test_root/symlink-root" +symlink_data_root="$symlink_root/CoreSimulator/Devices/$udid/data" +mkdir -p "$real_root/CoreSimulator/Devices/$udid/data" +ln -s "$real_root" "$symlink_root" +: >"$report" +if PATH="$stub_bin:$PATH" \ + TEST_XCRUN_LOG="$xcrun_log" \ + TEST_SIMULATOR_DATA_ROOT="$symlink_data_root" \ + TEST_PS_MODE=inactive \ + OLIPHAUNT_IOS_BROKER_QUARANTINE_PROCESS_WAIT_ATTEMPTS=3 \ + bash "$helper" \ + "$udid" \ + dev.oliphaunt.brokerspike \ + dev.oliphaunt.brokerspike.extension \ + OliphauntBrokerSpike \ + OliphauntBrokerSpike \ + BrokerAppExtension \ + BrokerAppExtension \ + "$report" >"$test_root/symlink.stdout" 2>"$test_root/symlink.stderr"; then + fail "symlinked simulator ancestry was accepted" +fi +grep -Fqx 'reason=unsafe-data-root' "$report" || \ + fail "symlink ancestry refusal was not recorded" + +reset_case +run_helper +grep -Fqx 'status=absent' "$report" || fail "absent root was not reported" + +printf 'simulator storage quarantine synthetic tests passed\n' diff --git a/src/sdks/swift/tools/run-ios-broker-device-hang.sh b/src/sdks/swift/tools/run-ios-broker-device-hang.sh new file mode 100755 index 00000000..8cee4203 --- /dev/null +++ b/src/sdks/swift/tools/run-ios-broker-device-hang.sh @@ -0,0 +1,760 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="${OLIPHAUNT_REPO_ROOT:-$(cd "$script_dir/../../../.." && pwd)}" + +fail() { + failure_reason="$*" + printf 'error: %s\n' "$failure_reason" >&2 + exit 1 +} + +absolute_path() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "$repo_root" "$1" ;; + esac +} + +validate_hang_report() { + local report_path="$1" + local validation_path="$2" + local expected_host_pid="$3" + ruby -rjson - "$report_path" "$validation_path" "$expected_host_pid" <<'RUBY' +report_path, validation_path, expected_host_pid = ARGV +expected_host_pid = Integer(expected_host_pid, 10) +report = JSON.parse(File.read(report_path)) +raise "app report must be a JSON object" unless report.is_a?(Hash) +error = report["error"] +raise "app reported failure: #{error}" unless error.nil? || error.empty? +result = report["result"] +raise "app report is missing result" unless result.is_a?(Hash) + +host_pid = result["hostPID"] +worker_pid = result["workerPID"] +raise "app report has an invalid host PID" unless host_pid.is_a?(Integer) && host_pid.positive? +raise "app report host PID disagrees with devicectl" unless host_pid == expected_host_pid +raise "app report has an invalid worker PID" unless worker_pid.is_a?(Integer) && worker_pid.positive? +raise "host and worker PIDs are identical" if host_pid == worker_pid + +uuid = /\A[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\z/ +epoch = result["epoch"] +raise "app report has an invalid epoch" unless epoch.is_a?(String) && epoch.match?(uuid) + +expected_checks = %w[ + hangCapabilityConservative + hangTimeout + mainActorResponsiveDuringHang + oldHangEpochInvalidated + replacementLaunchAttempted +] +checks = result["checks"] +raise "app report checks must be an array" unless checks.is_a?(Array) +raise "app report check matrix contains duplicates" unless checks.uniq.length == checks.length +raise "app report checks differ from the exact hang matrix" unless checks.sort == expected_checks.sort + +observations = result["observations"] +raise "hang observations must be an object" unless observations.is_a?(Hash) +raise "hang matrix overclaimed restartability" unless observations["hangRestartableCapability"] == "false" +raise "hang initial worker PID disagrees with the report" unless Integer(observations.fetch("initialWorkerPID"), 10) == worker_pid +raise "hang initial epoch disagrees with the report" unless observations.fetch("initialEpoch") == epoch +timeout = observations.fetch("timeout", "") +raise "hang matrix omitted its bounded terminal error" if timeout.empty? +raise "hang matrix terminal was not a deadline/interruption/outcome-unknown result" unless timeout.match?(/deadline|interrupt|outcome.*unknown/i) +raise "hang fault was not acknowledged before the trigger" unless observations["faultAcknowledged"] == "true" +raise "worker was not responsive after the fault acknowledgement" unless observations["postAckWorkerResponsive"] == "true" +raise "post-ack worker PID changed" unless Integer(observations.fetch("postAckWorkerPID"), 10) == worker_pid +raise "post-ack epoch changed" unless observations.fetch("postAckEpoch") == epoch + +initial_attempt_count = Integer(observations.fetch("initialLaunchAttemptCount"), 10) +interrupted_attempt_count = Integer(observations.fetch("interruptedLaunchAttemptCount"), 10) +post_attempt_count = Integer(observations.fetch("postRecoveryLaunchAttemptCount"), 10) +attempt_delta = Integer(observations.fetch("replacementLaunchAttemptDelta"), 10) +initial_launch_count = Integer(observations.fetch("initialLaunchCount"), 10) +interrupted_launch_count = Integer(observations.fetch("interruptedLaunchCount"), 10) +post_launch_count = Integer(observations.fetch("postRecoveryLaunchCount"), 10) +successful_launch_delta = Integer(observations.fetch("successfulLaunchCountDelta"), 10) +raise "initial launch-attempt count is invalid" unless initial_attempt_count.positive? +raise "initial successful-launch count is invalid" unless initial_launch_count.positive? +raise "initial attempts are fewer than successful launches" unless initial_attempt_count >= initial_launch_count +raise "hang invalidation regressed launch attempts" unless interrupted_attempt_count >= initial_attempt_count +raise "hang invalidation regressed successful launches" unless interrupted_launch_count >= initial_launch_count +raise "post-hang query did not attempt a replacement" unless post_attempt_count > interrupted_attempt_count +raise "replacement attempt delta is inconsistent" unless attempt_delta == post_attempt_count - interrupted_attempt_count +raise "successful launch count regressed" unless post_launch_count >= interrupted_launch_count +raise "successful launch delta is inconsistent" unless successful_launch_delta == post_launch_count - interrupted_launch_count + +fresh = observations["freshProcessObtained"] +raise "hang matrix omitted its fresh-process outcome" unless %w[true false].include?(fresh) +recovered_epochs = result["recoveredEpochs"] +raise "recoveredEpochs must be an array" unless recovered_epochs.is_a?(Array) +recovery_proven = false +recovery_outcome = "noFreshWorker" +recovered_pid = nil +recovered_epoch = nil + +if fresh == "true" + recovered_pid = Integer(observations.fetch("recoveredWorkerPID"), 10) + recovered_epoch = observations.fetch("recoveredEpoch") + raise "fresh worker PID is invalid" unless recovered_pid.positive? + raise "fresh worker PID collides with the host" if recovered_pid == host_pid + raise "fresh worker reused the initial PID" if recovered_pid == worker_pid + raise "fresh worker has an invalid epoch" unless recovered_epoch.is_a?(String) && recovered_epoch.match?(uuid) + raise "fresh worker reused the initial epoch" if recovered_epoch == epoch + raise "fresh recovery list is inconsistent" unless recovered_epochs == [recovered_epoch] + raise "fresh worker had no successful Ready launch" unless post_launch_count > interrupted_launch_count + raise "fresh worker launch delta is not positive" unless successful_launch_delta.positive? + raise "fresh recovery unexpectedly recorded a failure" if observations.key?("recoveryFailure") + recovery_proven = true + recovery_outcome = "freshWorkerObtained" +else + raise "unavailable recovery omitted its failure" if observations.fetch("recoveryFailure", "").empty? + raise "unavailable recovery published recovered epochs" unless recovered_epochs.empty? + if observations.key?("recoveredWorkerPID") || observations.key?("recoveredEpoch") + recovered_pid = Integer(observations.fetch("recoveredWorkerPID"), 10) + recovered_epoch = observations.fetch("recoveredEpoch") + raise "reported replacement PID is invalid" unless recovered_pid.positive? + raise "reported replacement PID collides with the host" if recovered_pid == host_pid + raise "reported replacement epoch is invalid" unless recovered_epoch.is_a?(String) && recovered_epoch.match?(uuid) + both_fresh = recovered_pid != worker_pid && recovered_epoch != epoch + raise "a fully fresh worker was mislabeled unavailable" if both_fresh + end +end + +payload = { + schema: "oliphaunt-ios-broker-device-hang-validation-v1", + status: "PASS", + evidenceStatus: "PASS", + recoveryProven: recovery_proven, + recoveryOutcome: recovery_outcome, + hostPID: host_pid, + initialWorkerPID: worker_pid, + initialEpoch: epoch, + recoveredWorkerPID: recovered_pid, + recoveredEpoch: recovered_epoch, + checks: checks, + timeout: timeout, + launchCounters: { + initialAttemptCount: initial_attempt_count, + interruptedAttemptCount: interrupted_attempt_count, + postRecoveryAttemptCount: post_attempt_count, + replacementAttemptDelta: attempt_delta, + initialSuccessfulLaunchCount: initial_launch_count, + interruptedSuccessfulLaunchCount: interrupted_launch_count, + postRecoverySuccessfulLaunchCount: post_launch_count, + successfulLaunchDelta: successful_launch_delta, + }, +} +File.write(validation_path, JSON.pretty_generate(payload) + "\n") +RUBY +} + +if [ "${1:-}" = "--validate-report" ]; then + [ "$#" -eq 4 ] || fail "usage: $0 --validate-report REPORT VALIDATION EXPECTED_HOST_PID" + validate_hang_report "$2" "$3" "$4" + exit 0 +fi +[ "$#" -eq 0 ] || fail "usage: $0 [--validate-report REPORT VALIDATION EXPECTED_HOST_PID]" + +device_id="${OLIPHAUNT_IOS_BROKER_DEVICE_ID:-7C01EC26-8B01-56E6-872D-82BB72421567}" +expected_udid="${OLIPHAUNT_IOS_BROKER_DEVICE_UDID:-00008120-001474980C47C01E}" +app_bundle_id="${OLIPHAUNT_IOS_BROKER_BUNDLE_ID:-dev.oliphaunt.brokerspike}" +extension_bundle_id="${OLIPHAUNT_IOS_BROKER_EXTENSION_BUNDLE_ID:-dev.oliphaunt.brokerspike.extension}" +expected_team="${OLIPHAUNT_IOS_BROKER_DEVELOPMENT_TEAM:-LCXFQNDD46}" +timeout_seconds="${OLIPHAUNT_IOS_BROKER_HANG_TIMEOUT_SECONDS:-90}" +retained_manifest="$(absolute_path "${OLIPHAUNT_IOS_BROKER_RETAINED_DEBUG_PRODUCT:-target/ios-native-broker-device-spike/reports/retained-semantic-debug-product.json}")" +canonical_device_report="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_RUNNER_REPORT:-target/ios-native-broker-device-spike/reports/device-runner-report.json}")" +expected_host_debug_sha="${OLIPHAUNT_IOS_BROKER_HOST_DEBUG_DYLIB_SHA256:-}" +expected_extension_debug_sha="${OLIPHAUNT_IOS_BROKER_EXTENSION_DEBUG_DYLIB_SHA256:-}" +run_token="${OLIPHAUNT_IOS_BROKER_HANG_RUN_TOKEN:-device-hang-$(date -u +%Y%m%dT%H%M%SZ)-$$}" +run_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_HANG_RUN_ROOT:-target/ios-native-broker-device-hang/$run_token}")" +reports_dir="$run_root/reports" +logs_dir="$run_root/logs" +mkdir -p "$reports_dir/pulled" "$reports_dir/cleanup" "$logs_dir" + +device_details="$reports_dir/devicectl-device-details.json" +device_lock_state="$reports_dir/devicectl-lock-state.json" +installed_apps_before="$reports_dir/devicectl-installed-apps-before.json" +artifact_validation="$reports_dir/retained-debug-validation.json" +install_json="$reports_dir/devicectl-install-debug.json" +install_log="$logs_dir/devicectl-install-debug.log" +launch_json="$reports_dir/devicectl-launch-hang.json" +console_log="$logs_dir/devicectl-console-hang.log" +console_report="$reports_dir/console-app-report.json" +pulled_report="$reports_dir/pulled/broker-spike-report.json" +copy_json="$reports_dir/devicectl-copy-report.json" +copy_log="$logs_dir/devicectl-copy-report.log" +validation_report="$reports_dir/hang-validation.json" +process_inventory="$reports_dir/devicectl-processes-after-report.json" +pass_marker="$reports_dir/pass-marker.txt" +runner_report="$reports_dir/device-hang-runner-report.json" +failure_file="$reports_dir/failure.txt" + +success_marker="OLIPHAUNT_BROKER_SPIKE PASS" +failure_marker="OLIPHAUNT_BROKER_SPIKE FAIL" +json_marker="OLIPHAUNT_BROKER_SPIKE_JSON " +failure_reason="" +console_pid="" +cleanup_complete=0 + +[ -n "$expected_host_debug_sha" ] || fail "OLIPHAUNT_IOS_BROKER_HOST_DEBUG_DYLIB_SHA256 is required" +[ -n "$expected_extension_debug_sha" ] || fail "OLIPHAUNT_IOS_BROKER_EXTENSION_DEBUG_DYLIB_SHA256 is required" + +stop_console() { + [ -n "$console_pid" ] || return 0 + if kill -0 "$console_pid" 2>/dev/null; then + kill -TERM "$console_pid" 2>/dev/null || true + local attempts=50 + while [ "$attempts" -gt 0 ] && kill -0 "$console_pid" 2>/dev/null; do + sleep 0.1 + attempts=$((attempts - 1)) + done + kill -KILL "$console_pid" 2>/dev/null || true + fi + wait "$console_pid" 2>/dev/null || true + console_pid="" +} + +terminate_exact_app_processes() { + local phase="$1" + local installation_url="$2" + local before_inventory="$reports_dir/cleanup/$phase-processes-before.json" + local after_inventory="$reports_dir/cleanup/$phase-processes-after.json" + + xcrun devicectl device info processes \ + --device "$device_id" \ + --columns '*' \ + --timeout 30 \ + --json-output "$before_inventory" \ + >"$logs_dir/$phase-processes-before.log" 2>&1 || return 1 + + ruby -rjson - "$before_inventory" "$installation_url" "$device_id" <<'RUBY' >"$reports_dir/cleanup/$phase-validated-pids.txt" || return 1 +inventory_path, installation_url, device_id = ARGV +inventory = JSON.parse(File.read(inventory_path)) +raise "process inventory outcome was not success" unless inventory.dig("info", "outcome") == "success" +raise "process inventory targeted a different device" unless inventory.dig("result", "deviceIdentifier") == device_id +processes = inventory.dig("result", "runningProcesses") +raise "process inventory omitted runningProcesses" unless processes.is_a?(Array) +if installation_url.empty? + exit 0 +end +root = installation_url.sub(%r{/\z}, "") +expected = { + "#{root}/OliphauntBrokerSpike" => "host", + "#{root}/Extensions/BrokerAppExtension.appex/BrokerAppExtension" => "extension", +} +seen = [] +relevant = processes.select do |process| + executable = process["executable"].to_s + executable.end_with?("/OliphauntBrokerSpike.app/OliphauntBrokerSpike") || + executable.end_with?("/BrokerAppExtension.appex/BrokerAppExtension") +end +relevant.each do |process| + pid = process["processIdentifier"] + executable = process["executable"] + raise "app process has an invalid PID" unless pid.is_a?(Integer) && pid.positive? + kind = expected[executable] + raise "spike process does not belong to the current exact installed URL" unless kind + raise "duplicate app process PID" if seen.include?(pid) + seen << pid + puts [pid, kind].join("\t") +end +RUBY + + while IFS=$'\t' read -r cleanup_pid cleanup_kind; do + [ -n "$cleanup_pid" ] || continue + xcrun devicectl device process terminate \ + --device "$device_id" \ + --pid "$cleanup_pid" \ + --kill \ + --timeout 30 \ + --json-output "$reports_dir/cleanup/$phase-terminate-$cleanup_kind-$cleanup_pid.json" \ + >"$logs_dir/$phase-terminate-$cleanup_kind-$cleanup_pid.log" 2>&1 || true + done <"$reports_dir/cleanup/$phase-validated-pids.txt" + + sleep 1 + xcrun devicectl device info processes \ + --device "$device_id" \ + --columns '*' \ + --timeout 30 \ + --json-output "$after_inventory" \ + >"$logs_dir/$phase-processes-after.log" 2>&1 || return 1 + ruby -rjson - "$after_inventory" "$device_id" "$installation_url" <<'RUBY' || return 1 +inventory_path, device_id, installation_url = ARGV +inventory = JSON.parse(File.read(inventory_path)) +raise "post-cleanup inventory outcome was not success" unless inventory.dig("info", "outcome") == "success" +raise "post-cleanup inventory targeted a different device" unless inventory.dig("result", "deviceIdentifier") == device_id +processes = inventory.dig("result", "runningProcesses") +raise "post-cleanup inventory omitted runningProcesses" unless processes.is_a?(Array) +unless installation_url.empty? + root = installation_url.sub(%r{/\z}, "") + expected = [ + "#{root}/OliphauntBrokerSpike", + "#{root}/Extensions/BrokerAppExtension.appex/BrokerAppExtension", + ] + survivors = processes.select do |process| + executable = process["executable"].to_s + executable.end_with?("/OliphauntBrokerSpike.app/OliphauntBrokerSpike") || + executable.end_with?("/BrokerAppExtension.appex/BrokerAppExtension") + end + unless survivors.all? { |process| expected.include?(process["executable"]) } + raise "stale spike process survived under a non-current install URL" + end + raise "an app or extension process survived cleanup" unless survivors.empty? +end +RUBY +} + +post_evidence_cleanup() { + [ "$cleanup_complete" = "0" ] || return 0 + set +e + stop_console + cleanup_installation_url="" + if [ -s "$install_json" ]; then + cleanup_installation_url="$(ruby -rjson -e ' + report = JSON.parse(File.read(ARGV.fetch(0))) + puts report.dig("result", "installedApplications", 0, "installationURL").to_s + ' "$install_json")" + fi + if terminate_exact_app_processes post-evidence "$cleanup_installation_url"; then + printf 'PASS\n' >"$reports_dir/cleanup/status.txt" + cleanup_complete=1 + else + printf 'FAIL\n' >"$reports_dir/cleanup/status.txt" + fi + set -e +} + +cleanup() { + local exit_code=$? + trap - EXIT INT TERM + set +e + post_evidence_cleanup + if [ "$exit_code" -ne 0 ]; then + [ -n "$failure_reason" ] || failure_reason="physical hang runner failed with status $exit_code" + printf '%s\n' "$failure_reason" >"$failure_file" + tail -120 "$console_log" "$copy_log" "$install_log" 2>/dev/null >&2 + fi + exit "$exit_code" +} +trap cleanup EXIT INT TERM + +[ -f "$retained_manifest" ] || fail "retained Debug manifest is missing: $retained_manifest" +artifact_fields="$(ruby -rjson - "$retained_manifest" <<'RUBY' +manifest = JSON.parse(File.read(ARGV.fetch(0))) +raise "retained Debug manifest schema mismatch" unless manifest["schema"] == "oliphaunt-ios-broker-retained-semantic-debug-product-v1" +raise "retained Debug manifest did not pass" unless manifest["status"] == "PASS" +puts [ + manifest.fetch("appPath"), manifest.fetch("extensionPath"), + manifest.fetch("resultBundle"), manifest.fetch("hostExecutableSHA256"), + manifest.fetch("extensionExecutableSHA256") +].join("\t") +RUBY +)" || fail "retained Debug manifest validation failed" +IFS=$'\t' read -r app_path extension_path result_bundle expected_host_sha expected_extension_sha <"$logs_dir/codesign.log" 2>&1 || \ + fail "retained Debug app failed strict code-signature verification" +codesign --verify --strict "$extension_path" >>"$logs_dir/codesign.log" 2>&1 || \ + fail "retained Debug extension failed strict code-signature verification" +host_identifier="$(codesign -dv --verbose=4 "$app_path" 2>&1 | sed -n 's/^Identifier=//p' | tail -1)" +extension_identifier="$(codesign -dv --verbose=4 "$extension_path" 2>&1 | sed -n 's/^Identifier=//p' | tail -1)" +host_team="$(codesign -dv --verbose=4 "$app_path" 2>&1 | sed -n 's/^TeamIdentifier=//p' | tail -1)" +extension_team="$(codesign -dv --verbose=4 "$extension_path" 2>&1 | sed -n 's/^TeamIdentifier=//p' | tail -1)" +[ "$host_identifier" = "$app_bundle_id" ] || fail "retained Debug host bundle identifier changed" +[ "$extension_identifier" = "$extension_bundle_id" ] || fail "retained Debug extension bundle identifier changed" +[ "$host_team" = "$expected_team" ] || fail "retained Debug host signing team changed" +[ "$extension_team" = "$expected_team" ] || fail "retained Debug extension signing team changed" + +host_sha="$(shasum -a 256 "$host_executable" | awk '{print $1}')" +extension_sha="$(shasum -a 256 "$extension_executable" | awk '{print $1}')" +host_debug_sha="$(shasum -a 256 "$host_debug_dylib" | awk '{print $1}')" +extension_debug_sha="$(shasum -a 256 "$extension_debug_dylib" | awk '{print $1}')" +[ "$host_sha" = "$expected_host_sha" ] || fail "retained Debug host executable hash changed" +[ "$extension_sha" = "$expected_extension_sha" ] || fail "retained Debug extension executable hash changed" +[ "$host_debug_sha" = "$expected_host_debug_sha" ] || fail "retained Debug host code dylib hash changed since preflight" +[ "$extension_debug_sha" = "$expected_extension_debug_sha" ] || fail "retained Debug extension code dylib hash changed since preflight" +nm "$host_debug_dylib" | xcrun swift-demangle >"$logs_dir/host-debug-symbols.txt" +nm "$extension_debug_dylib" | xcrun swift-demangle >"$logs_dir/extension-debug-symbols.txt" +strings "$host_debug_dylib" >"$logs_dir/host-debug-strings.txt" +grep -Fq 'HangFaultMatrix' "$logs_dir/host-debug-symbols.txt" || fail "retained Debug host omits HangFaultMatrix" +grep -Fq 'OLIPHAUNT_BROKER_FIXTURE_MODE' "$logs_dir/host-debug-strings.txt" || fail "retained Debug host omits fixture-mode selection" +grep -Fq 'armDeadlockAfterNativeRequestRegistration' "$logs_dir/extension-debug-symbols.txt" || fail "retained Debug extension omits the armed deadlock fault" + +ruby -rjson - "$artifact_validation" "$retained_manifest" "$app_path" "$extension_path" \ + "$result_bundle" "$host_sha" "$extension_sha" "$host_debug_sha" "$extension_debug_sha" \ + "$host_team" <<'RUBY' +output, manifest, app, extension, result, host_sha, extension_sha, host_debug_sha, + extension_debug_sha, team = ARGV +payload = { + schema: "oliphaunt-ios-broker-retained-debug-hang-artifact-v1", + status: "PASS", + manifest: manifest, + appPath: app, + extensionPath: extension, + resultBundle: result, + teamIdentifier: team, + hostExecutableSHA256: host_sha, + extensionExecutableSHA256: extension_sha, + hostDebugDylibSHA256: host_debug_sha, + extensionDebugDylibSHA256: extension_debug_sha, + faultSymbolsPresent: true, +} +File.write(output, JSON.pretty_generate(payload) + "\n") +RUBY + +xcrun devicectl device info details \ + --device "$device_id" \ + --timeout 20 \ + --json-output "$device_details" \ + >"$logs_dir/devicectl-device-details.log" 2>&1 || \ + fail "failed to inspect the pinned physical device" +xcrun devicectl device info lockState \ + --device "$device_id" \ + --timeout 20 \ + --json-output "$device_lock_state" \ + >"$logs_dir/devicectl-lock-state.log" 2>&1 || true +ruby -rjson - "$device_details" "$device_id" "$expected_udid" <<'RUBY' || \ + fail "pinned physical-device preflight failed" +path, identifier, udid = ARGV +report = JSON.parse(File.read(path)) +raise "device details outcome was not success" unless report.dig("info", "outcome") == "success" +device = report["result"] || {} +hardware = device["hardwareProperties"] || {} +properties = device["deviceProperties"] || {} +connection = device["connectionProperties"] || {} +raise "CoreDevice identifier mismatch" unless device["identifier"] == identifier +raise "hardware UDID mismatch" unless hardware["udid"] == udid +raise "target is not a physical iOS device" unless hardware["platform"] == "iOS" && hardware["reality"] == "physical" +raise "target is not booted" unless properties["bootState"] == "booted" +raise "Developer Mode is not enabled" unless properties["developerModeStatus"] == "enabled" +raise "DDI services are unavailable" unless properties["ddiServicesAvailable"] == true +raise "device is not paired" unless connection["pairingState"] == "paired" +raise "device tunnel is not connected" unless connection["tunnelState"] == "connected" +RUBY + +printf 'Removing only exact pre-existing spike processes before Debug replacement...\n' +xcrun devicectl device info apps \ + --device "$device_id" \ + --bundle-id "$app_bundle_id" \ + --timeout 30 \ + --json-output "$installed_apps_before" \ + >"$logs_dir/devicectl-installed-apps-before.log" 2>&1 || \ + fail "failed to inspect the currently installed spike app" +existing_installation_url="$(ruby -rjson - "$installed_apps_before" "$device_id" "$app_bundle_id" <<'RUBY' +path, device_id, bundle_id = ARGV +report = JSON.parse(File.read(path)) +raise "installed-app inventory outcome was not success" unless report.dig("info", "outcome") == "success" +raise "installed-app inventory targeted a different device" unless report.dig("result", "deviceIdentifier") == device_id +apps = report.dig("result", "apps") +raise "installed-app inventory omitted apps" unless apps.is_a?(Array) +raise "multiple apps matched the exact bundle ID" if apps.length > 1 +unless apps.empty? + raise "installed app bundle ID mismatch" unless apps.first["bundleIdentifier"] == bundle_id + puts apps.first.fetch("url") +end +RUBY +)" || fail "installed spike app identity validation failed" +terminate_exact_app_processes setup "$existing_installation_url" || \ + fail "failed to prove a clean pre-launch host/extension process state" + +printf 'Installing exact retained Debug app for the one-shot physical hang lane...\n' +xcrun devicectl device install app \ + --device "$device_id" \ + --timeout 120 \ + --json-output "$install_json" \ + "$app_path" >"$install_log" 2>&1 || fail "failed to install the retained Debug app" +installation_url="$(ruby -rjson - "$install_json" "$device_id" "$app_bundle_id" "$app_path" <<'RUBY' +path, device_id, bundle_id, app_path = ARGV +report = JSON.parse(File.read(path)) +raise "install outcome was not success" unless report.dig("info", "outcome") == "success" +raise "install targeted a different device" unless report.dig("result", "deviceIdentifier") == device_id +arguments = report.dig("info", "arguments") +raise "install command omitted its exact source app" unless arguments.is_a?(Array) && arguments.last == app_path +apps = report.dig("result", "installedApplications") +raise "install returned the wrong app" unless apps.is_a?(Array) && apps.length == 1 && apps.first["bundleID"] == bundle_id +raise "install omitted its device URL" if apps.first.fetch("installationURL", "").empty? +puts apps.first.fetch("installationURL") +RUBY +)" || fail "retained Debug install result validation failed" + +extract_console_report() { + ruby -rjson - "$console_log" "$console_report" "$json_marker" <<'RUBY' 2>/dev/null +log_path, output_path, marker = ARGV +line = File.foreach(log_path).select { |candidate| candidate.include?(marker) }.last +exit 1 unless line +payload = line.split(marker, 2).fetch(1).strip +report = JSON.parse(payload) +File.write(output_path, JSON.generate(report) + "\n") +RUBY +} + +is_explicit_locked_launch_failure() { + [ -s "$launch_json" ] || return 1 + ruby -rjson - "$launch_json" <<'RUBY' >/dev/null 2>&1 +report = JSON.parse(File.read(ARGV.fetch(0))) +error = report["error"] || {} +exit 1 unless report.dig("info", "outcome") == "failed" +exit 1 unless error["domain"] == "com.apple.dt.CoreDeviceError" && error["code"] == 10_002 +exit 1 if report.key?("result") +domains = [] +strings = [] +walk = lambda do |value| + case value + when Hash + domains << value["domain"] if value["domain"].is_a?(String) + value.each_value { |child| walk.call(child) } + when Array + value.each { |child| walk.call(child) } + when String + strings << value + end +end +walk.call(error) +locked = domains.include?("FBSOpenApplicationServiceErrorDomain") && + domains.include?("FBSOpenApplicationErrorDomain") && + strings.any? { |value| value.include?("reason: Locked") || value.include?("because the device was not, or could not be, unlocked") } +exit(locked ? 0 : 1) +RUBY +} + +lock_retry_deadline=$((SECONDS + 120)) +launch_attempt=0 +while :; do + launch_attempt=$((launch_attempt + 1)) + : >"$launch_json" + : >"$console_log" + printf 'Launching deliberate-hang attempt %s on the physical device...\n' "$launch_attempt" + xcrun devicectl device process launch \ + --device "$device_id" \ + --terminate-existing \ + --activate \ + --console \ + --environment-variables '{"NSUnbufferedIO":"YES","OLIPHAUNT_BROKER_FIXTURE_DISABLE_IDLE_TIMER":"YES","OLIPHAUNT_BROKER_FIXTURE_MODE":"hang"}' \ + --timeout "$((timeout_seconds + 60))" \ + --json-output "$launch_json" \ + "$app_bundle_id" >"$console_log" 2>&1 & + console_pid=$! + + deadline=$((SECONDS + timeout_seconds)) + report_ready=0 + pass_line="" + while [ "$SECONDS" -lt "$deadline" ]; do + failure_line="$(grep -F "$failure_marker " "$console_log" 2>/dev/null | tail -1 || true)" + [ -z "$failure_line" ] || fail "hang fixture emitted failure: $failure_line" + if [ "$report_ready" = "0" ] && extract_console_report; then + report_ready=1 + fi + pass_line="$(grep -F "$success_marker " "$console_log" 2>/dev/null | tail -1 || true)" + if [ "$report_ready" = "1" ] && [ -n "$pass_line" ]; then + break + fi + if ! kill -0 "$console_pid" 2>/dev/null; then + wait "$console_pid" 2>/dev/null || true + console_pid="" + if [ "$report_ready" = "0" ] && [ -z "$pass_line" ] && is_explicit_locked_launch_failure; then + [ "$SECONDS" -lt "$lock_retry_deadline" ] || fail "device remained locked for the bounded pre-launch retry window" + printf 'Device explicitly rejected the hang launch as Locked; waiting to retry...\n' + sleep 2 + continue 2 + fi + fail "hang app or console ended before authoritative evidence" + fi + sleep 1 + done + [ "$report_ready" = "1" ] || fail "timed out without a structured physical hang report" + [ -n "$pass_line" ] || fail "timed out without the authoritative physical hang PASS marker" + break +done + +mkdir -p "$(dirname "$pulled_report")" +xcrun devicectl device copy from \ + --device "$device_id" \ + --source "Documents/broker-spike-report.json" \ + --destination "$pulled_report" \ + --domain-type appDataContainer \ + --domain-identifier "$app_bundle_id" \ + --timeout 30 \ + --json-output "$copy_json" >"$copy_log" 2>&1 || fail "failed to pull the physical hang report" +[ -f "$pulled_report" ] || fail "device copy omitted the physical hang report" +ruby -rjson - "$console_report" "$pulled_report" <<'RUBY' || fail "console and pulled hang reports disagree" +console, pulled = ARGV.map { |path| JSON.parse(File.read(path)) } +raise "console/pulled report mismatch" unless console == pulled +RUBY + +report_identity="$(ruby -rjson -e ' + result = JSON.parse(File.read(ARGV.fetch(0))).fetch("result") + observations = result.fetch("observations") + puts [result.fetch("hostPID"), result.fetch("workerPID"), observations["recoveredWorkerPID"]].join("\t") +' "$pulled_report")" +IFS=$'\t' read -r report_host_pid initial_worker_pid recovered_worker_pid <"$logs_dir/devicectl-processes-after-report.log" 2>&1 || fail "failed to inventory the physical hang processes" +ruby -rjson - "$process_inventory" "$pulled_report" "$installation_url" "$device_id" <<'RUBY' || \ + fail "physical process inventory did not corroborate the hang report" +inventory_path, report_path, installation_url, expected_device = ARGV +inventory = JSON.parse(File.read(inventory_path)) +result = JSON.parse(File.read(report_path)).fetch("result") +observations = result.fetch("observations") +raise "process inventory outcome was not success" unless inventory.dig("info", "outcome") == "success" +raise "process inventory targeted a different device" unless inventory.dig("result", "deviceIdentifier") == expected_device +processes = inventory.dig("result", "runningProcesses") +raise "process inventory omitted runningProcesses" unless processes.is_a?(Array) +root = installation_url.sub(%r{/\z}, "") +host_pid = result.fetch("hostPID") +initial_worker_pid = result.fetch("workerPID") +recovered_worker_pid = observations["recoveredWorkerPID"]&.then { |value| Integer(value, 10) } +expected_pids = [host_pid, initial_worker_pid, recovered_worker_pid].compact +pids = processes.map do |process| + pid = process.fetch("processIdentifier") + executable = process.fetch("executable") + raise "process inventory contains an unexpected PID" unless expected_pids.include?(pid) + expected_executable = if pid == host_pid + "#{root}/OliphauntBrokerSpike" + else + "#{root}/Extensions/BrokerAppExtension.appex/BrokerAppExtension" + end + raise "process executable does not belong to the exact installed product" unless executable == expected_executable + pid +end +raise "process inventory has duplicate PIDs" unless pids.uniq.length == pids.length +raise "host process is absent after report publication" unless pids.include?(host_pid) +RUBY + +stop_console +[ -s "$launch_json" ] || fail "devicectl omitted the finalized hang launch JSON" +launch_host_pid="$(ruby -rjson - "$launch_json" "$device_id" "$installation_url" <<'RUBY' +path, device_id, installation_url = ARGV +report = JSON.parse(File.read(path)) +raise "launch outcome was not success" unless report.dig("info", "outcome") == "success" +raise "launch targeted a different device" unless report.dig("result", "deviceIdentifier") == device_id +host_pid = report.dig("result", "process", "processIdentifier") +raise "launch returned an invalid host PID" unless host_pid.is_a?(Integer) && host_pid.positive? +expected_executable = "#{installation_url.sub(%r{/\z}, "")}/OliphauntBrokerSpike" +raise "launch executable does not belong to the exact installed product" unless report.dig("result", "process", "executable") == expected_executable +options = report.dig("result", "launchOptions") || {} +raise "hang fixture mode was not supplied" unless options.dig("environmentVariables", "OLIPHAUNT_BROKER_FIXTURE_MODE") == "hang" +raise "hang launch was not activated" unless options["activatedWhenStarted"] == true +puts host_pid +RUBY +)" || fail "devicectl hang launch result validation failed" +validate_hang_report "$pulled_report" "$validation_report" "$launch_host_pid" || \ + fail "physical hang report validation failed" +printf '%s\n' "$pass_line" >"$pass_marker" +post_evidence_cleanup +[ "$(cat "$reports_dir/cleanup/status.txt" 2>/dev/null || true)" = "PASS" ] || \ + fail "post-evidence app/extension cleanup was not proven complete" + +ruby -rjson - "$runner_report" "$run_token" "$device_details" "$device_lock_state" \ + "$artifact_validation" "$install_json" "$launch_json" "$console_report" "$pulled_report" \ + "$validation_report" "$process_inventory" "$pass_marker" "$canonical_device_report" \ + "$reports_dir/cleanup/status.txt" "$reports_dir/cleanup/post-evidence-processes-after.json" <<'RUBY' +output, token, device_details_path, lock_path, artifact_path, install_path, + launch_path, console_report_path, pulled_report_path, validation_path, + process_inventory_path, pass_marker_path, canonical_device_report, + cleanup_status_path, cleanup_inventory_path = ARGV +details = JSON.parse(File.read(device_details_path)).fetch("result") +validation = JSON.parse(File.read(validation_path)) +artifact = JSON.parse(File.read(artifact_path)) +device_properties = details.fetch("deviceProperties") +hardware = details.fetch("hardwareProperties") +connection = details.fetch("connectionProperties") +payload = { + schema: "oliphaunt-ios-broker-physical-hang-run-v1", + status: "PASS", + evidenceStatus: validation.fetch("evidenceStatus"), + recoveryProven: validation.fetch("recoveryProven"), + recoveryOutcome: validation.fetch("recoveryOutcome"), + completedAt: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + runToken: token, + evidenceType: "signed-debug-physical-device-deliberate-hang", + device: { + coreDeviceIdentifier: details.fetch("identifier"), + udid: hardware.fetch("udid"), + name: device_properties.fetch("name"), + os: device_properties.fetch("osVersionNumber"), + osBuild: device_properties.fetch("osBuildUpdate"), + productType: hardware.fetch("productType"), + transport: connection.fetch("transportType"), + developerMode: device_properties.fetch("developerModeStatus"), + ddiServicesAvailable: device_properties.fetch("ddiServicesAvailable"), + }, + artifact: artifact, + report: JSON.parse(File.read(pulled_report_path)), + validation: validation, + evidence: { + deviceDetails: device_details_path, + lockState: File.size?(lock_path) ? lock_path : nil, + install: install_path, + launch: launch_path, + consoleReport: console_report_path, + pulledReport: pulled_report_path, + processInventoryAfterReport: process_inventory_path, + passMarker: pass_marker_path, + canonicalDeviceRunnerReport: canonical_device_report, + cleanupStatus: cleanup_status_path, + processInventoryAfterCleanup: cleanup_inventory_path, + }, + interpretation: { + genericFixturePassMeansRecovery: false, + oneSuccessfulRecoveryProvesReliability: false, + releaseOrDistributionQualified: false, + }, +} +File.write(output, JSON.pretty_generate(payload) + "\n") +RUBY + +rm -f "$failure_file" +printf 'Physical deliberate-hang evidence: %s\n' "$runner_report" +ruby -rjson -e ' + report = JSON.parse(File.read(ARGV.fetch(0))) + puts "Recovery outcome: #{report.fetch("recoveryOutcome")}" + puts "Recovery proven in this run: #{report.fetch("recoveryProven")}" +' "$runner_report" diff --git a/src/sdks/swift/tools/run-ios-broker-device.sh b/src/sdks/swift/tools/run-ios-broker-device.sh new file mode 100755 index 00000000..4e00a7be --- /dev/null +++ b/src/sdks/swift/tools/run-ios-broker-device.sh @@ -0,0 +1,2661 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +default_repo_root="$(cd "$script_dir/../../../.." && pwd)" +repo_root="${OLIPHAUNT_REPO_ROOT:-$default_repo_root}" + +. "$repo_root/src/sdks/react-native/tools/expo-runner-common.sh" + +absolute_path() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "$repo_root" "$1" ;; + esac +} + +normalize_yes_no() { + case "$1" in + 1|YES|yes|TRUE|true|ON|on) printf 'YES\n' ;; + 0|NO|no|FALSE|false|OFF|off) printf 'NO\n' ;; + *) fail "$2 must be YES or NO, got: $1" ;; + esac +} + +safe_bundle_identifier() { + case "$1" in + ''|*[!A-Za-z0-9.-]*) return 1 ;; + *) return 0 ;; + esac +} + +safe_process_name() { + case "$1" in + ''|*[!A-Za-z0-9._-]*) return 1 ;; + *) return 0 ;; + esac +} + +safe_build_name() { + case "$1" in + ''|.|..|*/*|*$'\n'*|*$'\r'*) return 1 ;; + *) return 0 ;; + esac +} + +select_xcode_development_team() { + { + defaults read com.apple.dt.Xcode IDEProvisioningTeams 2>/dev/null || true + defaults read com.apple.dt.Xcode IDEProvisioningTeamByIdentifier 2>/dev/null || true + } | + awk -F'= ' '/teamID =/ { value = $2; gsub(/[;[:space:]]/, "", value); print value }' | + sort -u | + awk 'NR == 1 { first = $0 } NR > 1 { multiple = 1 } END { if (!multiple && first != "") print first; else exit 1 }' +} + +valid_code_signing_identity_count() { + security find-identity -v -p codesigning 2>/dev/null | + awk '/valid identities found/ { print $1; found = 1 } END { if (!found) print 0 }' +} + +fixture_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_FIXTURE_ROOT:-spikes/ios-native-broker}")" +generator="$(absolute_path "${OLIPHAUNT_IOS_BROKER_PROJECT_GENERATOR:-$fixture_root/generate_project.rb}")" +build_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_BUILD_ROOT:-target/ios-native-broker-device-spike}")" +derived_data="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_DERIVED_DATA:-$build_root/DerivedData}")" +logs_dir="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_LOGS_DIR:-$build_root/logs}")" +reports_dir="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_REPORTS_DIR:-$build_root/reports}")" +artifact_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_ARTIFACT_ROOT:-target/ios-native-broker-device-artifacts}")" +artifact_preparer="$(absolute_path "${OLIPHAUNT_IOS_BROKER_ARTIFACT_PREPARER:-$script_dir/prepare-ios-broker-artifacts.sh}")" +artifact_environment="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_ARTIFACT_ENV:-$artifact_root/broker-artifacts.env}")" + +scheme="${OLIPHAUNT_IOS_BROKER_SCHEME:-OliphauntBrokerSpike}" +configuration="${OLIPHAUNT_IOS_BROKER_CONFIGURATION:-Debug}" +lifecycle_configuration="${OLIPHAUNT_IOS_BROKER_LIFECYCLE_CONFIGURATION:-Release}" +app_product_name="${OLIPHAUNT_IOS_BROKER_APP_PRODUCT_NAME:-OliphauntBrokerSpike}" +extension_product_name="${OLIPHAUNT_IOS_BROKER_EXTENSION_PRODUCT_NAME:-BrokerAppExtension}" +app_bundle_id="${OLIPHAUNT_IOS_BROKER_BUNDLE_ID:-dev.oliphaunt.brokerspike}" +extension_bundle_id="${OLIPHAUNT_IOS_BROKER_EXTENSION_BUNDLE_ID:-dev.oliphaunt.brokerspike.extension}" +requested_device_id="${OLIPHAUNT_IOS_BROKER_DEVICE_ID:-}" +requested_device_name="${OLIPHAUNT_IOS_BROKER_DEVICE_NAME:-}" +minimum_ios_major="${OLIPHAUNT_IOS_BROKER_MIN_IOS_MAJOR:-26}" +timeout_seconds="${OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS:-180}" +prepare_artifacts="${OLIPHAUNT_IOS_BROKER_PREPARE_ARTIFACTS:-YES}" +preflight_only="${OLIPHAUNT_IOS_BROKER_DEVICE_PREFLIGHT_ONLY:-NO}" +resume_lifecycle_only="${OLIPHAUNT_IOS_BROKER_RESUME_LIFECYCLE_ONLY:-NO}" +resume_after_debug_install="${OLIPHAUNT_IOS_BROKER_RESUME_AFTER_DEBUG_INSTALL:-NO}" +resume_debug_result_bundle_input="${OLIPHAUNT_IOS_BROKER_RESUME_DEBUG_RESULT_BUNDLE:-}" +clean_install="${OLIPHAUNT_IOS_BROKER_DEVICE_CLEAN_INSTALL:-YES}" +uninstall_after_run="${OLIPHAUNT_IOS_BROKER_UNINSTALL_AFTER_RUN:-NO}" +code_signing_allowed="${OLIPHAUNT_IOS_BROKER_CODE_SIGNING_ALLOWED:-YES}" +development_team="${OLIPHAUNT_IOS_BROKER_DEVELOPMENT_TEAM:-}" +code_sign_style="${OLIPHAUNT_IOS_BROKER_CODE_SIGN_STYLE:-}" +code_sign_identity="${OLIPHAUNT_IOS_BROKER_CODE_SIGN_IDENTITY:-}" +provisioning_profile_specifier="${OLIPHAUNT_IOS_BROKER_PROVISIONING_PROFILE_SPECIFIER:-}" +allow_provisioning_updates="${OLIPHAUNT_IOS_BROKER_ALLOW_PROVISIONING_UPDATES:-0}" +allow_device_registration="${OLIPHAUNT_IOS_BROKER_ALLOW_PROVISIONING_DEVICE_REGISTRATION:-0}" + +if [ "$#" -gt 0 ]; then + case "$1" in + --preflight-only) preflight_only=YES ;; + --resume-lifecycle) resume_lifecycle_only=YES ;; + --resume-after-debug-install) resume_after_debug_install=YES ;; + *) fail "usage: run-ios-broker-device.sh [--preflight-only|--resume-after-debug-install|--resume-lifecycle]" ;; + esac +fi +[ "$#" -le 1 ] || \ + fail "usage: run-ios-broker-device.sh [--preflight-only|--resume-after-debug-install|--resume-lifecycle]" + +success_marker="OLIPHAUNT_BROKER_SPIKE PASS" +failure_marker="OLIPHAUNT_BROKER_SPIKE FAIL" +json_marker="OLIPHAUNT_BROKER_SPIKE_JSON " +runner_report_path="$reports_dir/device-runner-report.json" +persistence_report="$reports_dir/extension-private-persistence.json" +device_inventory="$reports_dir/devicectl-devices.json" +device_details="$reports_dir/devicectl-device-details.json" +device_lock_state="$reports_dir/devicectl-lock-state.json" +installed_apps="$reports_dir/devicectl-installed-apps.json" +install_result="$reports_dir/devicectl-install.json" +preflight_report="$reports_dir/device-preflight.json" +artifact_validation_file="$reports_dir/broker-artifacts.txt" +embedded_extensions_file="$reports_dir/embedded-extensions.txt" +host_linkage_file="$reports_dir/host-otool.txt" +extension_linkage_file="$reports_dir/extension-otool.txt" +embedded_native_file="$reports_dir/embedded-native-library.txt" +extension_resources_file="$reports_dir/extension-resource-checks.txt" +signing_validation_file="$reports_dir/code-signing-checks.txt" +extension_host_sdk_symbol_file="$reports_dir/extension-host-sdk-symbol-checks.txt" +release_fault_symbol_file="$reports_dir/release-fault-symbol-checks.txt" +launch_one_app_report="$reports_dir/launch-1-app-report.json" +launch_two_app_report="$reports_dir/launch-2-app-report.json" +launch_one_console_report="$reports_dir/launch-1-console-report.json" +launch_two_console_report="$reports_dir/launch-2-console-report.json" +launch_one_validation="$reports_dir/launch-1-report-validation.json" +launch_two_validation="$reports_dir/launch-2-report-validation.json" +launch_one_result="$reports_dir/devicectl-launch-1.json" +launch_two_result="$reports_dir/devicectl-launch-2.json" +launch_one_copy_result="$reports_dir/devicectl-copy-report-1.json" +launch_two_copy_result="$reports_dir/devicectl-copy-report-2.json" +launch_one_pass_marker="$reports_dir/launch-1-pass-marker.txt" +launch_two_pass_marker="$reports_dir/launch-2-pass-marker.txt" +lifecycle_runner_report="$reports_dir/device-lifecycle-runner-report.json" +lifecycle_size_report="$reports_dir/release-product-sizes.json" +lifecycle_archive_validation="$reports_dir/release-archive-validation.json" +retained_semantic_product_validation="$reports_dir/retained-semantic-debug-product.json" +lifecycle_launch_one_report="$reports_dir/lifecycle-launch-1-app-report.json" +lifecycle_launch_two_report="$reports_dir/lifecycle-launch-2-app-report.json" +lifecycle_launch_one_validation="$reports_dir/lifecycle-launch-1-validation.json" +lifecycle_launch_two_validation="$reports_dir/lifecycle-launch-2-validation.json" +lifecycle_launch_one_result="$reports_dir/devicectl-lifecycle-launch-1.json" +lifecycle_launch_two_result="$reports_dir/devicectl-lifecycle-launch-2.json" +lifecycle_install_result="$reports_dir/devicectl-install-release-lifecycle.json" +lifecycle_run_token_from_environment="${OLIPHAUNT_IOS_BROKER_LIFECYCLE_RUN_TOKEN:-}" +lifecycle_run_token="${lifecycle_run_token_from_environment:-device-lifecycle-$(date -u +%Y%m%dT%H%M%SZ)-$$}" + +generator_log="$logs_dir/generate-project.log" +artifact_preparation_log="$logs_dir/prepare-broker-artifacts.log" +build_log="$logs_dir/xcodebuild-device.log" +install_log="$logs_dir/devicectl-install.log" +launch_one_log="$logs_dir/devicectl-console-launch-1.log" +launch_two_log="$logs_dir/devicectl-console-launch-2.log" +launch_one_copy_log="$logs_dir/devicectl-copy-report-1.log" +launch_two_copy_log="$logs_dir/devicectl-copy-report-2.log" +lifecycle_build_log="$logs_dir/xcodebuild-device-release-lifecycle.log" +lifecycle_archive_log="$logs_dir/xcodebuild-device-release-archive.log" +lifecycle_install_log="$logs_dir/devicectl-install-release-lifecycle.log" +lifecycle_launch_one_log="$logs_dir/devicectl-lifecycle-launch-1.log" +lifecycle_launch_two_log="$logs_dir/devicectl-lifecycle-launch-2.log" +lifecycle_build_result_bundle="" +lifecycle_archive_result_bundle="" +lifecycle_archive_path="" +lifecycle_release_build_app_path="" +validating_release_artifact=0 +semantic_app_path="" +semantic_extension_path="" + +selected_device_id="" +selected_device_udid="" +selected_device_name="" +selected_device_os="" +selected_device_product="" +selected_device_transport="" +selected_device_developer_mode="" +selected_device_ddi="" +host_executable="" +extension_executable="" +embedded_native_library="" +embedded_native_framework="" +app_path="" +extension_path="" +build_result_bundle="" +console_pid="" +installed=0 +failure_reason="" + +fail() { + failure_reason="$*" + printf 'error: %s\n' "$failure_reason" >&2 + exit 1 +} + +stop_device_console() { + [ -n "$console_pid" ] || return 0 + if kill -0 "$console_pid" 2>/dev/null; then + kill -TERM "$console_pid" 2>/dev/null || true + local attempts=50 + while [ "$attempts" -gt 0 ] && kill -0 "$console_pid" 2>/dev/null; do + sleep 0.1 + attempts=$((attempts - 1)) + done + kill -KILL "$console_pid" 2>/dev/null || true + fi + wait "$console_pid" 2>/dev/null || true + console_pid="" +} + +cleanup() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_device_console + if [ "$exit_code" -ne 0 ]; then + [ -n "$failure_reason" ] || failure_reason="device runner command failed with status $exit_code" + printf '%s\n' "$failure_reason" >"$reports_dir/failure.txt" + tail -120 "$launch_one_log" "$launch_two_log" "$launch_one_copy_log" \ + "$launch_two_copy_log" "$install_log" "$build_log" 2>/dev/null >&2 + fi + if [ "$installed" = "1" ] && [ "$uninstall_after_run" = "YES" ]; then + xcrun devicectl device uninstall app \ + --device "$selected_device_id" \ + --timeout 30 \ + "$app_bundle_id" >/dev/null 2>&1 || true + fi + exit "$exit_code" +} + +select_physical_device() { + xcrun devicectl list devices \ + --timeout 10 \ + --json-output "$device_inventory" \ + >"$logs_dir/devicectl-list.log" 2>&1 || \ + fail "failed to inventory physical iOS devices with devicectl" + + local selection + selection="$(ruby -rjson - "$device_inventory" "$requested_device_id" \ + "$requested_device_name" "$minimum_ios_major" <<'RUBY' +inventory_path, requested_id, requested_name, minimum_major = ARGV +minimum_major = Integer(minimum_major, 10) +devices = JSON.parse(File.read(inventory_path)).dig("result", "devices") || [] +candidates = devices.select do |device| + hardware = device["hardwareProperties"] || {} + connection = device["connectionProperties"] || {} + properties = device["deviceProperties"] || {} + identifier = device["identifier"] || hardware["udid"] + version = properties["osVersionNumber"].to_s + major = version[/\A\d+/].to_i + next false unless hardware["platform"] == "iOS" + next false unless hardware["reality"] == "physical" + next false unless connection["pairingState"] == "paired" + next false unless major >= minimum_major + next false if !requested_id.empty? && identifier != requested_id && hardware["udid"] != requested_id + next false if !requested_name.empty? && properties["name"] != requested_name + true +end + +if candidates.empty? + warn "no paired physical iOS #{minimum_major}+ device matched the requested selector" + exit 1 +end +if candidates.length > 1 + warn "multiple physical iOS devices matched; set OLIPHAUNT_IOS_BROKER_DEVICE_ID" + exit 1 +end + +device = candidates.fetch(0) +hardware = device["hardwareProperties"] || {} +connection = device["connectionProperties"] || {} +properties = device["deviceProperties"] || {} +fields = [ + device["identifier"] || hardware["udid"], + hardware["udid"], + properties["name"], + properties["osVersionNumber"], + hardware["productType"], + connection["transportType"], + properties["developerModeStatus"], + properties.fetch("ddiServicesAvailable", "unknown"), +] +abort("selected device metadata contains a tab or newline") if fields.any? { |field| field.to_s.match?(/[\t\r\n]/) } +puts fields.map(&:to_s).join("\t") +RUBY + )" || fail "failed to select a unique paired physical iOS device" + + IFS=$'\t' read -r selected_device_id selected_device_udid selected_device_name selected_device_os \ + selected_device_product selected_device_transport selected_device_developer_mode \ + selected_device_ddi <"$logs_dir/devicectl-details.log" 2>&1 || \ + fail "failed to inspect the selected iOS device; unlock and trust it, then retry" + xcrun devicectl device info lockState \ + --device "$selected_device_id" \ + --timeout 10 \ + --json-output "$device_lock_state" \ + >"$logs_dir/devicectl-lock-state.log" 2>&1 || true + + local device_preflight + device_preflight="$(ruby -rjson - "$device_details" "$minimum_ios_major" \ + "$selected_device_id" "$selected_device_udid" <<'RUBY' +details_path, minimum_major, expected_identifier, expected_udid = ARGV +minimum_major = Integer(minimum_major, 10) +result = JSON.parse(File.read(details_path))["result"] || {} +properties = result["deviceProperties"] || {} +hardware = result["hardwareProperties"] || {} +name = properties["name"] || "physical iOS device" +version = properties["osVersionNumber"].to_s +major = version[/\A\d+/].to_i +raise "device details CoreDevice identifier changed after selection" unless result["identifier"] == expected_identifier +raise "device details hardware UDID changed after selection" unless hardware["udid"] == expected_udid +raise "selected device is not a physical iOS device" unless hardware["platform"] == "iOS" && hardware["reality"] == "physical" +raise "#{name} runs iOS #{version}, but iOS #{minimum_major}+ is required" if major < minimum_major +mode = properties["developerModeStatus"] || "unknown" +raise "Developer Mode is not enabled on #{name}: #{mode}" unless mode == "enabled" +ddi = properties.fetch("ddiServicesAvailable", "unknown") +raise "Developer Disk Image services are unavailable on #{name}; reconnect/unlock it and let Xcode prepare it" unless ddi == true +puts [name, version, mode, ddi].join("\t") +RUBY + )" || fail "physical-device preflight failed" + + IFS=$'\t' read -r selected_device_name selected_device_os \ + selected_device_developer_mode selected_device_ddi </dev/null || true)" + [ -n "$framework_executable" ] || fail "broker framework slice has no executable" + native_library="$slice_product/$framework_executable" + [ -f "$native_library" ] || fail "broker device native library is missing: $native_library" + [ "$(xcrun vtool -show-build "$native_library" 2>/dev/null | awk '/platform / { print $2; exit }')" = "IOS" ] || \ + fail "broker XCFramework selected a non-device native library: $native_library" + local native_symbols required_symbol + native_symbols="$(nm -g "$native_library" 2>/dev/null)" + for required_symbol in \ + _liboliphaunt_selected_static_extensions \ + _oliphaunt_static_vector_Pg_magic_func \ + _oliphaunt_static_pg_trgm_Pg_magic_func; do + case "$native_symbols" in + *"$required_symbol"*) ;; + *) fail "broker device library is missing $required_symbol" ;; + esac + done + + local resource_root="$resources/oliphaunt" + local runtime_manifest="$resource_root/runtime/manifest.properties" + local template_manifest="$resource_root/template-pgdata/manifest.properties" + local static_manifest="$resource_root/static-registry/manifest.properties" + local required_resource + for required_resource in \ + "$runtime_manifest" \ + "$template_manifest" \ + "$static_manifest" \ + "$resource_root/runtime/files/share/postgresql/postgres.bki" \ + "$resource_root/runtime/files/share/postgresql/extension/vector.control" \ + "$resource_root/runtime/files/share/postgresql/extension/pg_trgm.control" \ + "$resource_root/template-pgdata/files/PG_VERSION"; do + [ -f "$required_resource" ] || fail "broker resources are incomplete: $required_resource" + done + grep -Fqx 'selectedExtensions=pg_trgm,vector' "$runtime_manifest" || \ + fail "broker runtime resources do not select exactly vector,pg_trgm" + grep -Fqx 'brokerDatabaseRole=oliphaunt_broker' "$template_manifest" || \ + fail "broker template does not seed the restricted database role" + grep -Fqx 'registeredExtensions=vector,pg_trgm' "$static_manifest" || \ + fail "broker static registry does not register exactly vector,pg_trgm" + + export OLIPHAUNT_IOS_BROKER_XCFRAMEWORK="$xcframework" + export OLIPHAUNT_IOS_BROKER_RESOURCES="$resources" + { + printf 'platform=ios-device\n' + printf 'architecture=arm64\n' + printf 'xcframework=%s\n' "$xcframework" + printf 'deviceLibrary=%s\n' "$native_library" + printf 'deviceLibrarySHA256=%s\n' "$(shasum -a 256 "$native_library" | awk '{ print $1 }')" + printf 'resources=%s\n' "$resources" + printf 'selectedExtensions=pg_trgm,vector\n' + } >"$artifact_validation_file" +} + +validate_built_app() { + [ -f "$app_path/Info.plist" ] || fail "built host app has no Info.plist: $app_path" + local observed_bundle_id + observed_bundle_id="$(plutil -extract CFBundleIdentifier raw -o - "$app_path/Info.plist" 2>/dev/null || true)" + [ "$observed_bundle_id" = "$app_bundle_id" ] || \ + fail "built host bundle identifier is $observed_bundle_id, expected $app_bundle_id" + host_executable="$(plutil -extract CFBundleExecutable raw -o - "$app_path/Info.plist" 2>/dev/null || true)" + safe_process_name "$host_executable" || fail "unsafe or missing host executable: $host_executable" + [ -x "$app_path/$host_executable" ] || fail "host executable is missing" + + extension_path="$app_path/Extensions/$extension_product_name.appex" + [ ! -e "$app_path/PlugIns/$extension_product_name.appex" ] || \ + fail "host app contains stale legacy extension packaging" + [ -d "$extension_path" ] || fail "host app is missing its embedded ExtensionKit extension" + find "$app_path/Extensions" -mindepth 1 -maxdepth 1 -type d -name '*.appex' -print | \ + LC_ALL=C sort >"$embedded_extensions_file" + observed_bundle_id="$(plutil -extract CFBundleIdentifier raw -o - "$extension_path/Info.plist" 2>/dev/null || true)" + [ "$observed_bundle_id" = "$extension_bundle_id" ] || \ + fail "embedded extension bundle identifier is $observed_bundle_id, expected $extension_bundle_id" + extension_executable="$(plutil -extract CFBundleExecutable raw -o - "$extension_path/Info.plist" 2>/dev/null || true)" + safe_process_name "$extension_executable" || fail "unsafe or missing extension executable" + [ -x "$extension_path/$extension_executable" ] || fail "embedded extension executable is missing" + + local host_binary extension_binary + : >"$host_linkage_file" + for host_binary in "$app_path/$host_executable" "$app_path/$host_executable.debug.dylib"; do + [ -f "$host_binary" ] || continue + otool -L "$host_binary" >>"$host_linkage_file" + done + : >"$extension_linkage_file" + for extension_binary in \ + "$extension_path/$extension_executable" \ + "$extension_path/$extension_executable.debug.dylib"; do + [ -f "$extension_binary" ] || continue + otool -L "$extension_binary" >>"$extension_linkage_file" + done + local native_framework_link='@rpath/liboliphaunt[.]framework/liboliphaunt' + local any_native_link='[/@]liboliphaunt([.]framework/liboliphaunt|[.]dylib)' + ! grep -Eq "$any_native_link" "$host_linkage_file" || \ + fail "broker host unexpectedly links liboliphaunt" + grep -Eq "$native_framework_link" "$extension_linkage_file" || \ + fail "broker extension does not load @rpath/liboliphaunt.framework/liboliphaunt" + + : >"$extension_host_sdk_symbol_file" + for extension_binary in \ + "$extension_path/$extension_executable" \ + "$extension_path/$extension_executable.debug.dylib"; do + [ -f "$extension_binary" ] || continue + { + nm "$extension_binary" 2>/dev/null || true + } | xcrun swift-demangle >>"$extension_host_sdk_symbol_file" + done + local host_adapter_symbol_pattern='OliphauntIOSBroker[.]IOSBroker(Manager|Engine|Session)([ .:$]|$)' + if grep -Eq "$host_adapter_symbol_pattern" "$extension_host_sdk_symbol_file"; then + fail "broker extension still contains host-only IOSBrokerManager/Engine/Session symbols" + fi + printf 'PASS: no IOSBrokerManager/IOSBrokerEngine/IOSBrokerSession symbols\n' \ + >>"$extension_host_sdk_symbol_file" + + : >"$release_fault_symbol_file" + for extension_binary in \ + "$app_path/$host_executable" \ + "$app_path/$host_executable.debug.dylib" \ + "$extension_path/$extension_executable" \ + "$extension_path/$extension_executable.debug.dylib"; do + [ -f "$extension_binary" ] || continue + { + nm "$extension_binary" 2>/dev/null || true + } | xcrun swift-demangle >>"$release_fault_symbol_file" + done + case "$validating_release_artifact" in + 1) + if grep -Eq 'BrokerFaultInjector|WorkerCore.*injectFault|IOSBrokerSession.*injectFault|ExtendedFaultMatrix|HangFaultMatrix' \ + "$release_fault_symbol_file"; then + fail "Release broker products still contain DEBUG fault-injection implementation symbols" + fi + printf 'PASS: no DEBUG fault-injection implementation symbols\n' \ + >>"$release_fault_symbol_file" + ;; + *) + printf 'INFO: DEBUG product fault symbols intentionally not gated here\n' \ + >>"$release_fault_symbol_file" + ;; + esac + + if find "$app_path" -type f -name 'liboliphaunt.dylib' -print -quit | grep -q .; then + fail "signed device app contains a forbidden loose liboliphaunt.dylib" + fi + if [ -d "$extension_path/Frameworks/liboliphaunt.framework" ]; then + fail "device broker framework must be embedded by the host, not duplicated in the extension" + fi + [ -d "$app_path/Frameworks" ] || fail "signed device host is missing its Frameworks directory" + find "$app_path/Frameworks" -type f -path '*/liboliphaunt.framework/liboliphaunt' \ + -print | LC_ALL=C sort >"$embedded_native_file" + [ "$(wc -l <"$embedded_native_file" | tr -d '[:space:]')" = "1" ] || \ + fail "broker host must embed exactly one liboliphaunt framework" + embedded_native_library="$(cat "$embedded_native_file")" + embedded_native_framework="$(dirname "$embedded_native_library")" + [ "$embedded_native_framework" = "$app_path/Frameworks/liboliphaunt.framework" ] || \ + fail "broker framework is not in the host Frameworks directory" + [ "$(xcrun vtool -show-build "$embedded_native_library" 2>/dev/null | awk '/platform / { print $2; exit }')" = "IOS" ] || \ + fail "embedded liboliphaunt is not an iOS device binary" + + local resource_root="$extension_path/oliphaunt" + local relative resource_file + : >"$extension_resources_file" + for relative in \ + runtime/manifest.properties \ + template-pgdata/manifest.properties \ + static-registry/manifest.properties \ + runtime/files/share/postgresql/postgres.bki \ + runtime/files/share/postgresql/extension/vector.control \ + runtime/files/share/postgresql/extension/pg_trgm.control \ + template-pgdata/files/PG_VERSION; do + resource_file="$resource_root/$relative" + [ -f "$resource_file" ] || fail "embedded broker extension resource is missing: $relative" + printf '%s\t%s\t%s\n' "$relative" \ + "$(wc -c <"$resource_file" | tr -d '[:space:]')" \ + "$(shasum -a 256 "$resource_file" | awk '{ print $1 }')" \ + >>"$extension_resources_file" + done + grep -Fqx 'brokerDatabaseRole=oliphaunt_broker' \ + "$resource_root/template-pgdata/manifest.properties" || \ + fail "embedded broker template lost its restricted database role" + if find "$resource_root" -type f \( -name '*.dylib' -o -name '*.so' \) -print -quit | grep -q .; then + fail "embedded broker resource tree contains a dynamic extension module" + fi + + for resource_file in \ + "$app_path/embedded.mobileprovision" \ + "$extension_path/embedded.mobileprovision"; do + [ -f "$resource_file" ] || fail "signed device bundle is missing $(basename "$resource_file")" + done + codesign --verify --deep --strict "$app_path" >"$signing_validation_file" 2>&1 || \ + fail "host app failed strict code-signature verification" + codesign --verify --strict "$extension_path" >>"$signing_validation_file" 2>&1 || \ + fail "extension failed strict code-signature verification" + codesign --verify --strict "$embedded_native_framework" >>"$signing_validation_file" 2>&1 || \ + fail "embedded liboliphaunt failed strict code-signature verification" + local signed_item observed_team + for signed_item in "$app_path" "$extension_path" "$embedded_native_framework"; do + observed_team="$(codesign -dv --verbose=4 "$signed_item" 2>&1 | sed -n 's/^TeamIdentifier=//p' | tail -1)" + [ "$observed_team" = "$development_team" ] || \ + fail "signed product team does not match OLIPHAUNT_IOS_BROKER_DEVELOPMENT_TEAM" + printf '%s\tteam-match\n' "$signed_item" >>"$signing_validation_file" + done +} + +extract_console_report() { + local console_log="$1" + local output_report="$2" + ruby -rjson - "$console_log" "$output_report" "$json_marker" <<'RUBY' 2>/dev/null +log_path, output_path, marker = ARGV +line = File.foreach(log_path).select { |candidate| candidate.include?(marker) }.last +exit 1 unless line +payload = line.split(marker, 2).fetch(1).strip +report = JSON.parse(payload) +File.write(output_path, JSON.generate(report) + "\n") +RUBY +} + +latest_console_marker() { + local marker="$1" + local console_log="$2" + grep -F "$marker " "$console_log" 2>/dev/null | tail -1 || true +} + +validate_app_report() { + local report_path="$1" + local validation_path="$2" + ruby -rjson - "$report_path" "$validation_path" <<'RUBY' +report_path, validation_path = ARGV +report = JSON.parse(File.read(report_path)) +raise "app report must be a JSON object" unless report.is_a?(Hash) +error = report["error"] +raise "app reported failure: #{error}" unless error.nil? || error.empty? +result = report["result"] +raise "app report is missing result" unless result.is_a?(Hash) +host_pid = result["hostPID"] +worker_pid = result["workerPID"] +raise "app report has invalid host PID" unless host_pid.is_a?(Integer) && host_pid.positive? +raise "app report has invalid worker PID" unless worker_pid.is_a?(Integer) && worker_pid.positive? +raise "host and extension PIDs are identical" if host_pid == worker_pid +epoch = result["epoch"] +uuid = /\A[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\z/ +raise "app report has invalid epoch" unless epoch.is_a?(String) && epoch.match?(uuid) +expected_checks = %w[ + extensionDiscovery separatePID xpcSession fdTransfer workerDiagnostics + openedIdleMemory capabilities realSelect ddl write parameterizedQuery + pgdataPathConfidentiality + postgresErrorRecovery vectorExtension pgTrgmExtension fragmentedFrame + boundedRequestAssembly multiFrameRequest streamingResponse simultaneousHandles + fifoSerialization referenceCounting transactionHandlePinning cancellation + postCancelLiveness checkpointControl backgroundLifecycle sameRootReopen + outcomeUnknown postCommitAmbiguity crashRecovery preCommitRollbackRecovery + noAutomaticReplay +] +checks = result["checks"] +raise "app report checks must be an array" unless checks.is_a?(Array) +missing = expected_checks - checks +raise "app report is missing checks: #{missing.join(",")}" unless missing.empty? +unexpected = checks - expected_checks +raise "app report has unexpected checks: #{unexpected.join(",")}" unless unexpected.empty? +raise "app report check matrix contains duplicates" unless checks.uniq.length == expected_checks.length + +recovered_epochs = result["recoveredEpochs"] +raise "app report must contain exactly three recovered epochs" unless recovered_epochs.is_a?(Array) && recovered_epochs.length == 3 +raise "app report recovered epochs are not unique" unless recovered_epochs.uniq.length == 3 +raise "app report contains an invalid recovered epoch" unless recovered_epochs.all? { |value| value.is_a?(String) && value.match?(uuid) } +raise "initial epoch appears in recovered epochs" if recovered_epochs.include?(epoch) + +diagnostics = result["diagnostics"] +raise "app report diagnostics must be an array" unless diagnostics.is_a?(Array) +recovery_phases = %w[openedIdle sameRootReopen postCommitRecovery preCommitRecovery] +recovery_evidence = recovery_phases.map do |phase| + matches = diagnostics.select { |entry| entry["phase"] == phase } + raise "app report must contain exactly one #{phase} diagnostic" unless matches.length == 1 + matches.fetch(0) +end +diagnostic_epochs = recovery_evidence.map { |entry| entry["epoch"] } +raise "diagnostic recovery epochs are invalid" unless diagnostic_epochs.all? { |value| value.is_a?(String) && value.match?(uuid) } +raise "diagnostic recovery epochs are not unique" unless diagnostic_epochs.uniq.length == 4 +raise "initial epoch disagrees with openedIdle diagnostic" unless diagnostic_epochs.first == epoch +raise "diagnostic recovered epochs disagree with result" unless diagnostic_epochs.drop(1).sort == recovered_epochs.sort +diagnostic_pids = recovery_evidence.map { |entry| entry["workerPID"] } +raise "diagnostic worker PIDs are invalid" unless diagnostic_pids.all? { |value| value.is_a?(Integer) && value.positive? && value != host_pid } +raise "initial worker PID disagrees with openedIdle diagnostic" unless diagnostic_pids.first == worker_pid +raise "same-root reopen did not reuse the live worker process" unless diagnostic_pids[1] == diagnostic_pids[0] +raise "post-commit recovery did not start a fresh worker process" if diagnostic_pids[2] == diagnostic_pids[0] +raise "pre-commit recovery did not start a fresh worker process" if diagnostic_pids[3] == diagnostic_pids[0] +raise "crash recovery cycles reused a worker process" if diagnostic_pids[3] == diagnostic_pids[2] +raise "recovery diagnostics must prove exactly three worker processes" unless diagnostic_pids.uniq.length == 3 + +observations = result["observations"] +raise "app report observations must be an object" unless observations.is_a?(Hash) +raise "app report used the wrong database role" unless observations["restrictedDatabaseRole"] == "oliphaunt_broker" +raise "app report assigned the database to the broker role" unless observations["databaseOwner"] == "postgres" +raise "app report assigned selected extensions to the broker role" unless observations["selectedExtensionOwners"] == "pg_trgm:postgres,vector:postgres" +raise "app report omitted the broker-owned working schema" unless observations["brokerSchemaOwner"] == "oliphaunt_broker" +%w[ + dataDirectorySQLState parameterizedDataDirectorySQLState serverFileSQLState + bootstrapEscalationSQLState sessionAuthorizationEscalationSQLState + databaseOwnerEscalationSQLState relationPathSQLState tablespacePathSQLState + listDirectorySQLState statFileSQLState largeObjectImportSQLState + externalCopySQLState externalCopyFromSQLState alterSystemSQLState + createRoleSQLState selfSuperuserEscalationSQLState grantFileRoleSQLState + dropSelectedExtensionSQLState + createTablespaceSQLState createNativeFunctionSQLState loadLibrarySQLState + afterResetDataDirectorySQLState afterDiscardDataDirectorySQLState +].each do |key| + raise "app report did not deny #{key}" unless observations[key] == "42501" +end +raise "app report did not preserve the sanitized backend SQLSTATE" unless observations["sanitizedBackendErrorSQLState"] == "F0000" +raise "app report did not restore the broker search path after DISCARD ALL" unless observations["afterDiscardSearchPath"] == "{oliphaunt_broker,public}" +raise "app report exposed data_directory through pg_settings" unless observations["pgSettingsDataDirectoryRows"] == "0" +%w[ + restrictedFunctionExecuteCount restrictedViewSelectCount + pgSettingsSourcePathRows visiblePrivatePathSettingRows +].each do |key| + raise "app report exposed private catalog/path evidence through #{key}" unless observations[key] == "0" +end +raise "app report found a non-default tablespace" unless observations["nonDefaultTablespaceCount"] == "0" +raise "manager launch count is not 4" unless Integer(observations.fetch("launchCount"), 10) == 4 +raise "manager interruption count is not 2" unless Integer(observations.fetch("interruptionCount"), 10) == 2 +streamed_bytes = Integer(observations.fetch("streamedBytes"), 10) +streamed_chunks = Integer(observations.fetch("streamedChunks"), 10) +raise "streamed response byte evidence is too small" unless streamed_bytes > 2 * 1024 * 1024 +raise "streamed response was not delivered in multiple chunks" unless streamed_chunks > 1 +File.write(validation_path, JSON.pretty_generate({ + status: "PASS", + hostPID: host_pid, + workerPID: worker_pid, + epoch: epoch, + recoveredEpochs: recovered_epochs, + recoveryWorkerPIDs: diagnostic_pids, + checks: checks, + launchCount: 4, + interruptionCount: 2, + streamedBytes: streamed_bytes, + streamedChunks: streamed_chunks, +}) + "\n") +RUBY +} + +pull_app_report() { + local launch_index="$1" + local destination_report="$2" + local copy_result="$3" + local copy_log="$4" + local pull_directory="$reports_dir/pulled-launch-$launch_index-$$" + local pulled_report="$pull_directory/broker-spike-report.json" + mkdir -p "$pull_directory" + if ! xcrun devicectl device copy from \ + --device "$selected_device_id" \ + --source "Documents/broker-spike-report.json" \ + --destination "$pulled_report" \ + --domain-type appDataContainer \ + --domain-identifier "$app_bundle_id" \ + --timeout 30 \ + --json-output "$copy_result" \ + >"$copy_log" 2>&1; then + tail -80 "$copy_log" >&2 || true + fail "failed to pull launch $launch_index app report from the device" + fi + [ -f "$pulled_report" ] || \ + fail "device copy did not produce the launch $launch_index broker report" + cp "$pulled_report" "$destination_report" +} + +is_explicit_locked_launch_failure() { + local launch_json="$1" + [ -s "$launch_json" ] || return 1 + ruby -rjson - "$launch_json" <<'RUBY' >/dev/null 2>&1 +report = JSON.parse(File.read(ARGV.fetch(0))) +error = report["error"] || {} +exit 1 unless report.dig("info", "outcome") == "failed" +exit 1 unless error["domain"] == "com.apple.dt.CoreDeviceError" +exit 1 unless error["code"] == 10_002 +exit 1 if report.key?("result") + +domains = [] +strings = [] +walk = lambda do |value| + case value + when Hash + domains << value["domain"] if value["domain"].is_a?(String) + value.each_value { |child| walk.call(child) } + when Array + value.each { |child| walk.call(child) } + when String + strings << value + end +end +walk.call(error) +exit 1 unless domains.include?("FBSOpenApplicationServiceErrorDomain") +exit 1 unless domains.include?("FBSOpenApplicationErrorDomain") +locked = strings.any? do |value| + value.include?("reason: Locked") || + value.include?("because the device was not, or could not be, unlocked") +end +exit(locked ? 0 : 1) +RUBY +} + +run_probe_launch() { + local launch_index="$1" + local app_report="$2" + local console_report="$3" + local validation_report="$4" + local console_log="$5" + local launch_json="$6" + local copy_json="$7" + local copy_log="$8" + local pass_marker="$9" + local deadline report_valid pass_line failure_line + local lock_retry_deadline launch_attempt + + : >"$console_log" + : >"$console_report" + : >"$app_report" + : >"$validation_report" + : >"$pass_marker" + printf 'Launching physical-device probe %s of 2 without reinstall...\n' "$launch_index" + lock_retry_deadline=$((SECONDS + 120)) + launch_attempt=0 + while :; do + launch_attempt=$((launch_attempt + 1)) + : >"$console_log" + : >"$launch_json" + xcrun devicectl device process launch \ + --device "$selected_device_id" \ + --terminate-existing \ + --console \ + --environment-variables "{\"NSUnbufferedIO\":\"YES\",\"OLIPHAUNT_BROKER_DEVICE_LAUNCH_INDEX\":\"$launch_index\",\"OLIPHAUNT_BROKER_FIXTURE_DISABLE_IDLE_TIMER\":\"YES\"}" \ + --timeout "$((timeout_seconds + 60))" \ + --json-output "$launch_json" \ + "$app_bundle_id" >"$console_log" 2>&1 & + console_pid=$! + + deadline=$((SECONDS + timeout_seconds)) + report_valid=0 + pass_line="" + while [ "$SECONDS" -lt "$deadline" ]; do + failure_line="$(latest_console_marker "$failure_marker" "$console_log")" + [ -z "$failure_line" ] || \ + fail "broker spike launch $launch_index emitted failure marker: $failure_line" + if [ "$report_valid" = "0" ] && extract_console_report "$console_log" "$console_report"; then + validate_app_report "$console_report" "$validation_report" || \ + fail "broker spike launch $launch_index emitted an invalid console report" + report_valid=1 + fi + pass_line="$(latest_console_marker "$success_marker" "$console_log")" + if [ "$report_valid" = "1" ] && [ -n "$pass_line" ]; then + break + fi + if ! kill -0 "$console_pid" 2>/dev/null; then + wait "$console_pid" 2>/dev/null || true + console_pid="" + if [ "$report_valid" = "0" ] && [ -z "$pass_line" ] && \ + is_explicit_locked_launch_failure "$launch_json"; then + [ "$SECONDS" -lt "$lock_retry_deadline" ] || \ + fail "device remained locked for the bounded pre-launch retry window" + printf 'Device explicitly rejected probe %s pre-launch as Locked; waiting to retry (%s)...\n' \ + "$launch_index" "$launch_attempt" + sleep 2 + continue 2 + fi + fail "physical-device app or console ended before launch $launch_index PASS" + fi + sleep 1 + done + [ "$report_valid" = "1" ] || \ + fail "timed out waiting for physical-device launch $launch_index console report" + [ -n "$pass_line" ] || \ + fail "timed out waiting for launch $launch_index authoritative $success_marker marker" + + pull_app_report "$launch_index" "$app_report" "$copy_json" "$copy_log" + validate_app_report "$app_report" "$validation_report" || \ + fail "pulled launch $launch_index app report is invalid" + ruby -rjson - "$console_report" "$app_report" <<'RUBY' || \ + fail "launch $launch_index console and pulled reports disagree" +console_path, pulled_path = ARGV +console = JSON.parse(File.read(console_path)) +pulled = JSON.parse(File.read(pulled_path)) +raise "console/pulled report mismatch" unless console == pulled +RUBY + printf '%s\n' "$pass_line" >"$pass_marker" + stop_device_console + [ -s "$launch_json" ] || fail "devicectl did not write launch $launch_index result JSON" + sleep 2 + return 0 + done +} + +copy_lifecycle_report() { + local launch_index="$1" + local attempt="$2" + local destination_report="$3" + local poll_directory="$reports_dir/lifecycle-pulls-$launch_index-$$/attempt-$attempt" + local pulled_report="$poll_directory/broker-lifecycle-report.json" + local copy_result="$poll_directory/devicectl-copy.json" + local copy_log="$poll_directory/devicectl-copy.log" + mkdir -p "$poll_directory" + xcrun devicectl device copy from \ + --device "$selected_device_id" \ + --source "Documents/broker-lifecycle-report.json" \ + --destination "$pulled_report" \ + --domain-type appDataContainer \ + --domain-identifier "$app_bundle_id" \ + --timeout 15 \ + --json-output "$copy_result" \ + >"$copy_log" 2>&1 || return 1 + [ -f "$pulled_report" ] || return 1 + cp "$pulled_report" "$destination_report" +} + +wait_for_lifecycle_phase() { + local launch_index="$1" + local expected_phase="$2" + local destination_report="$3" + local deadline=$((SECONDS + timeout_seconds)) + local attempt=0 state + while [ "$SECONDS" -lt "$deadline" ]; do + attempt=$((attempt + 1)) + if copy_lifecycle_report "$launch_index" "$attempt" "$destination_report"; then + state="$(ruby -rjson - "$destination_report" "$lifecycle_run_token" \ + "$launch_index" "$expected_phase" <<'RUBY' 2>/dev/null || true +path, token, launch_index, expected_phase = ARGV +report = JSON.parse(File.read(path)) +exit 1 unless report["runToken"] == token +exit 1 unless report["launchIndex"] == Integer(launch_index, 10) +if report["status"] == "fail" || report["phase"] == "failed" + puts "FAIL:#{report["error"] || "unspecified lifecycle failure"}" +elsif report["phase"] == expected_phase + puts "MATCH" +else + puts "WAIT:#{report["phase"]}" +end +RUBY + )" + case "$state" in + MATCH) return 0 ;; + FAIL:*) fail "lifecycle launch $launch_index failed: ${state#FAIL:}" ;; + esac + fi + sleep 1 + done + fail "timed out waiting for lifecycle launch $launch_index phase $expected_phase" +} + +wait_for_lifecycle_foreground_active() { + local launch_index="$1" + local destination_report="$2" + local deadline=$((SECONDS + 30)) + local attempt=0 state + while [ "$SECONDS" -lt "$deadline" ]; do + attempt=$((attempt + 1)) + if copy_lifecycle_report \ + "$launch_index" "foreground-active-$attempt" "$destination_report"; then + state="$(ruby -rjson - "$destination_report" "$lifecycle_run_token" \ + "$launch_index" <<'RUBY' 2>/dev/null || true +path, token, launch_index = ARGV +report = JSON.parse(File.read(path)) +exit 1 unless report["runToken"] == token +exit 1 unless report["launchIndex"] == Integer(launch_index, 10) +if report["status"] == "fail" || report["phase"] == "failed" + puts "FAIL:#{report["error"] || "unspecified lifecycle failure"}" +elsif Array(report["events"]).last&.fetch("kind", nil) == "active" + puts "MATCH" +else + puts "WAIT" +end +RUBY + )" + case "$state" in + MATCH) return 0 ;; + FAIL:*) fail "lifecycle launch $launch_index failed: ${state#FAIL:}" ;; + esac + fi + sleep 1 + done + fail "lifecycle launch $launch_index never entered an active foreground scene" +} + +lifecycle_report_integer() { + ruby -rjson -e ' + value = ARGV.drop(1).reduce(JSON.parse(File.read(ARGV.fetch(0)))) { |memo, key| memo.fetch(key) } + abort("lifecycle report value is not a positive integer") unless value.is_a?(Integer) && value.positive? + puts value + ' "$@" +} + +lifecycle_report_string() { + ruby -rjson -e ' + value = ARGV.drop(1).reduce(JSON.parse(File.read(ARGV.fetch(0)))) { |memo, key| memo.fetch(key) } + abort("lifecycle report value is not a string") unless value.is_a?(String) && !value.empty? + puts value + ' "$@" +} + +classify_suspended_process_inventory() { + local inventory_path="$1" + local host_pid="$2" + local worker_pid="$3" + local expected_device_id="$4" + ruby -rjson - "$inventory_path" "$host_pid" "$worker_pid" \ + "$expected_device_id" <<'RUBY' +inventory_path, host_pid, worker_pid, expected_device_id = ARGV +host_pid = Integer(host_pid, 10) +worker_pid = Integer(worker_pid, 10) +raise "host and worker PIDs collide" if host_pid == worker_pid + +inventory = JSON.parse(File.read(inventory_path)) +raise "process inventory outcome was not success" unless inventory.dig("info", "outcome") == "success" +raise "process inventory belongs to a different device" unless inventory.dig("result", "deviceIdentifier") == expected_device_id +processes = inventory.dig("result", "runningProcesses") +raise "process inventory omitted runningProcesses" unless processes.is_a?(Array) +process_ids = processes.map do |process| + raise "process inventory entry is not an object" unless process.is_a?(Hash) + pid = process["processIdentifier"] + raise "process inventory entry has an invalid PID" unless pid.is_a?(Integer) && pid.positive? + pid +end +raise "process inventory contains duplicate PIDs" unless process_ids.uniq.length == process_ids.length +unexpected_pids = process_ids - [host_pid, worker_pid] +raise "process inventory contains unexpected PIDs" unless unexpected_pids.empty? +raise "suspended host is absent from process inventory" unless process_ids.include?(host_pid) + +case process_ids.sort +when [host_pid].sort + puts "workerAbsent" +when [host_pid, worker_pid].sort + puts "workerPresent" +else + raise "process inventory has an unrecognized host/worker state" +end +RUBY +} + +is_exact_devicectl_esrch_failure() { + ruby -rjson -e ' + result = JSON.parse(File.read(ARGV.fetch(0))) + exact_esrch = result.dig("info", "outcome") == "failed" && + result.dig("error", "domain") == "NSPOSIXErrorDomain" && + result.dig("error", "code") == 3 + exit(exact_esrch ? 0 : 1) + ' "$1" 2>/dev/null +} + +validate_lifecycle_report() { + local report_path="$1" + local validation_path="$2" + local launch_index="$3" + local expect_worker_kill="$4" + local worker_termination_mode="$5" + local foreground_inventory_path="$6" + local suspend_result_path="$7" + local suspended_inventory_path="$8" + local worker_terminate_result_path="$9" + local post_terminate_inventory_path="${10}" + local expected_device_id="${11}" + ruby -rjson - "$report_path" "$validation_path" "$lifecycle_run_token" \ + "$launch_index" "$expect_worker_kill" "$worker_termination_mode" \ + "$timeout_seconds" "$foreground_inventory_path" "$suspend_result_path" \ + "$suspended_inventory_path" "$worker_terminate_result_path" \ + "$post_terminate_inventory_path" "$expected_device_id" <<'RUBY' +report_path, validation_path, run_token, launch_index, expect_worker_kill, + worker_termination_mode, runner_timeout_seconds, foreground_inventory_path, + suspend_result_path, suspended_inventory_path, + worker_terminate_result_path, post_terminate_inventory_path, + expected_device_id = ARGV +launch_index = Integer(launch_index, 10) +expect_worker_kill = expect_worker_kill == "YES" +runner_timeout_seconds = Integer(runner_timeout_seconds, 10) +report = JSON.parse(File.read(report_path)) +raise "lifecycle schema mismatch" unless report["schemaVersion"] == 1 +raise "lifecycle run token mismatch" unless report["runToken"] == run_token +raise "lifecycle launch index mismatch" unless report["launchIndex"] == launch_index +raise "lifecycle report did not pass: #{report["error"]}" unless report["status"] == "pass" && report["phase"] == "completed" +raise "lifecycle worker-kill expectation mismatch" unless report["expectWorkerKill"] == expect_worker_kill +accepted_worker_termination_modes = %w[ + explicitSIGKILL workerAbsentAtPostSuspendInventory exitedDuringKillRace +] +if expect_worker_kill + raise "worker termination mode did not prove suspended unavailability" unless accepted_worker_termination_modes.include?(worker_termination_mode) +else + raise "ordinary resume unexpectedly recorded worker termination" unless worker_termination_mode == "notRequested" +end + +uuid = /\A[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\z/ +host_pid = report.fetch("hostPID") +initial_pid = report.fetch("initialWorkerPID") +resumed_pid = report.fetch("currentWorkerPID") +initial_epoch = report.fetch("initialEpoch") +resumed_epoch = report.fetch("currentEpoch") +raise "invalid lifecycle host/worker PID" unless [host_pid, initial_pid, resumed_pid].all? { |pid| pid.is_a?(Integer) && pid.positive? } +raise "host and lifecycle worker PID collide" if host_pid == initial_pid || host_pid == resumed_pid +raise "invalid lifecycle epoch" unless [initial_epoch, resumed_epoch].all? { |epoch| epoch.is_a?(String) && epoch.match?(uuid) } +if expect_worker_kill + raise "killed background worker reused its PID" if resumed_pid == initial_pid + raise "killed background worker reused its epoch" if resumed_epoch == initial_epoch +else + same_worker = resumed_pid == initial_pid && resumed_epoch == initial_epoch + fresh_worker = resumed_pid != initial_pid && resumed_epoch != initial_epoch + raise "resume produced a mixed stale PID/epoch identity" unless same_worker || fresh_worker +end +raise "lifecycle report omitted its manifest digest" if report.fetch("manifestDigest").to_s.empty? + +parse_inventory = lambda do |path| + inventory = JSON.parse(File.read(path)) + raise "process inventory outcome was not success" unless inventory.dig("info", "outcome") == "success" + raise "process inventory belongs to a different device" unless inventory.dig("result", "deviceIdentifier") == expected_device_id + processes = inventory.dig("result", "runningProcesses") + raise "process inventory omitted runningProcesses" unless processes.is_a?(Array) + process_ids = processes.map do |process| + raise "process inventory entry is not an object" unless process.is_a?(Hash) + pid = process["processIdentifier"] + raise "process inventory entry has an invalid PID" unless pid.is_a?(Integer) && pid.positive? + pid + end + raise "process inventory contains duplicate PIDs" unless process_ids.uniq.length == process_ids.length + raise "process inventory contains unexpected PIDs" unless (process_ids - [host_pid, initial_pid]).empty? + raise "suspended host is not present exactly once" unless process_ids.count(host_pid) == 1 + process_ids +end + +foreground_process_ids = parse_inventory.call(foreground_inventory_path) +raise "foreground-ready inventory did not include the worker" unless foreground_process_ids.include?(initial_pid) +suspend_result = JSON.parse(File.read(suspend_result_path)) +raise "suspend command outcome was not success" unless suspend_result.dig("info", "outcome") == "success" +raise "suspend result belongs to a different device" unless suspend_result.dig("result", "deviceIdentifier") == expected_device_id +raise "suspend result names the wrong host PID" unless suspend_result.dig("result", "process", "processIdentifier") == host_pid +suspend_signal = suspend_result.dig("result", "signal") +raise "suspend result has an invalid signal object" unless suspend_signal.is_a?(Hash) +raise "suspend result did not deliver SIGSTOP" unless suspend_signal["name"] == "SIGSTOP" && suspend_signal["value"] == 17 +suspended_process_ids = parse_inventory.call(suspended_inventory_path) +worker_present_at_post_suspend_inventory = suspended_process_ids.include?(initial_pid) + +post_terminate_process_ids = nil +worker_terminate_result = nil +case worker_termination_mode +when "explicitSIGKILL" + worker_terminate_result = JSON.parse(File.read(worker_terminate_result_path)) + raise "worker terminate command outcome was not success" unless worker_terminate_result.dig("info", "outcome") == "success" + raise "worker terminate result belongs to a different device" unless worker_terminate_result.dig("result", "deviceIdentifier") == expected_device_id + raise "worker terminate result names the wrong PID" unless worker_terminate_result.dig("result", "process", "processIdentifier") == initial_pid + terminate_signal = worker_terminate_result.dig("result", "signal") + raise "worker terminate result has an invalid signal object" unless terminate_signal.is_a?(Hash) + raise "worker terminate result did not deliver SIGKILL" unless terminate_signal["name"] == "SIGKILL" && terminate_signal["value"] == 9 + post_terminate_process_ids = parse_inventory.call(post_terminate_inventory_path) + raise "explicitly killed worker remained in the post-terminate inventory" if post_terminate_process_ids.include?(initial_pid) +when "workerAbsentAtPostSuspendInventory" + raise "absent-worker mode found the worker in the post-suspend inventory" if worker_present_at_post_suspend_inventory + worker_terminate_result = JSON.parse(File.read(worker_terminate_result_path)) + exact_esrch = worker_terminate_result.dig("info", "outcome") == "failed" && + worker_terminate_result.dig("error", "domain") == "NSPOSIXErrorDomain" && + worker_terminate_result.dig("error", "code") == 3 + raise "absent-worker mode did not record exact ESRCH" unless exact_esrch + post_terminate_process_ids = parse_inventory.call(post_terminate_inventory_path) + raise "worker appeared in the post-ESRCH inventory" if post_terminate_process_ids.include?(initial_pid) +when "exitedDuringKillRace" + raise "kill-race mode began with an absent worker" unless worker_present_at_post_suspend_inventory + worker_terminate_result = JSON.parse(File.read(worker_terminate_result_path)) + exact_esrch = worker_terminate_result.dig("info", "outcome") == "failed" && + worker_terminate_result.dig("error", "domain") == "NSPOSIXErrorDomain" && + worker_terminate_result.dig("error", "code") == 3 + raise "kill-race mode did not record exact ESRCH" unless exact_esrch + post_terminate_process_ids = parse_inventory.call(post_terminate_inventory_path) + raise "worker remained in the post-ESRCH inventory" if post_terminate_process_ids.include?(initial_pid) +when "notRequested" + raise "ordinary resume unexpectedly attempted worker termination" unless File.zero?(worker_terminate_result_path) + raise "ordinary resume unexpectedly wrote a post-terminate inventory" unless File.zero?(post_terminate_inventory_path) +else + raise "unrecognized worker termination evidence mode" +end +if !expect_worker_kill && !worker_present_at_post_suspend_inventory + raise "worker absent before ordinary resume reused its PID" if resumed_pid == initial_pid + raise "worker absent before ordinary resume reused its epoch" if resumed_epoch == initial_epoch +end + +checks = Array(report["checks"]) +required_checks = %w[ + extensionDiscovery separatePID workerDiagnostics backgroundContinuableFalse + openedIdleMemory availableMemory capabilities crossLaunchPersistence sizableRelation + protocolRTT cancellation postCancelLiveness executingMemory slowStreamThroughput + slowStreamTwoSizes slowStreamBoundedHeadroom + checkpointControl checkpointDiagnostics expiredDeadlineAdmission + backgroundCancellation backgroundAdmissionClosed backgroundDeadline + recursiveStorageProtection relationAndWALFreshness actualBackground + backgroundResume postResumeHealth postResumeMemory postResumePersistence +] +if expect_worker_kill + required_checks << "backgroundWorkerKillRecovery" +elsif !(checks.include?("backgroundSameWorkerResume") || checks.include?("backgroundFreshWorkerResume")) + raise "ordinary resume proved neither same-worker liveness nor fresh-worker recovery" +end +missing = required_checks - checks +raise "lifecycle report is missing checks: #{missing.join(",")}" unless missing.empty? +raise "lifecycle checks contain duplicates" unless checks.uniq.length == checks.length + +events = Array(report["events"]) +scene_phases = events.map { |event| event["kind"] } +event_uptimes = events.map { |event| Integer(event.fetch("observedAtUptimeNanoseconds")) } +raise "lifecycle event uptimes are not strictly monotonic" unless event_uptimes.each_cons(2).all? { |left, right| left < right } +application_state_events = events.reject { |event| event["kind"] == "memoryWarning" } +application_state_kinds = application_state_events.map { |event| event["kind"] } +raise "lifecycle application-state events contain adjacent duplicates" unless application_state_kinds.each_cons(2).all? { |left, right| left != right } +raise "app did not observe inactive lifecycle phase" unless scene_phases.include?("inactive") +raise "app did not observe background lifecycle phase" unless scene_phases.include?("background") +raise "app did not observe foreground resume" unless scene_phases.count("active") >= 2 + +observations = report.fetch("observations") +foreground_active_uptime = Integer(observations.fetch("foregroundActiveUptimeNanoseconds"), 10) +background_cutoff_uptime = Integer( + observations.fetch("backgroundTransitionNotBeforeUptimeNanoseconds"), 10 +) +background_transition_uptime = Integer( + observations.fetch("backgroundTransitionUptimeNanoseconds"), 10 +) +background_observed_uptime = Integer( + observations.fetch("backgroundObservedUptimeNanoseconds"), 10 +) +resumed_active_uptime = Integer( + observations.fetch("resumedActiveUptimeNanoseconds"), 10 +) +initial_active = events.find do |event| + event["kind"] == "active" && event["observedAtUptimeNanoseconds"] == foreground_active_uptime +end +raise "initial foreground-active observation is inconsistent" unless initial_active +latest_state_at_cutoff = application_state_events + .select { |event| event["observedAtUptimeNanoseconds"].to_i <= background_cutoff_uptime } + .last +raise "host was not active when the background handoff was armed" unless latest_state_at_cutoff&.fetch("kind") == "active" +transition = events.find do |event| + event["kind"] == "inactive" && + event["observedAtUptimeNanoseconds"].to_i == background_transition_uptime && + background_transition_uptime > background_cutoff_uptime +end +raise "no deliberate post-ready background transition was observed" unless transition +background = events.find do |event| + event["kind"] == "background" && + event["observedAtUptimeNanoseconds"].to_i == background_observed_uptime && + background_observed_uptime > background_transition_uptime +end +raise "no actual background event followed the deliberate transition" unless background +foregrounding = events.find do |event| + event["kind"] == "inactive" && + event["observedAtUptimeNanoseconds"].to_i > background_observed_uptime && + event["observedAtUptimeNanoseconds"].to_i < resumed_active_uptime +end +raise "no inactive foregrounding transition followed actual background" unless foregrounding +resumed_active = events.find do |event| + event["kind"] == "active" && + event["observedAtUptimeNanoseconds"].to_i == resumed_active_uptime && + resumed_active_uptime > background_observed_uptime +end +raise "no foreground-active event followed actual background" unless resumed_active + +raise "lifecycle qualification was not a Release build" unless observations.fetch("buildConfiguration") == "release" +orchestration_timeout = Integer(observations.fetch("orchestrationTimeoutSeconds"), 10) +background_work_seconds = Integer(observations.fetch("backgroundActiveWorkSeconds"), 10) +request_deadline_seconds = Integer(observations.fetch("requestDeadlineSeconds"), 10) +raise "fixture orchestration timeout disagrees with the runner" unless orchestration_timeout == runner_timeout_seconds +raise "background active-work duration is not twice the orchestration window" unless background_work_seconds == 2 * orchestration_timeout +raise "broker request deadline is not three times the orchestration window" unless request_deadline_seconds == 3 * orchestration_timeout +large_stream_bytes = Integer(observations.fetch("slowStreamBytes"), 10) +small_stream_bytes = Integer(observations.fetch("smallSlowStreamBytes"), 10) +small_stream_deadline_seconds = Integer(observations.fetch("smallSlowStreamSamplingDeadlineSeconds"), 10) +large_stream_deadline_seconds = Integer(observations.fetch("slowStreamSamplingDeadlineSeconds"), 10) +small_stream_elapsed_nanoseconds = Integer(observations.fetch("smallSlowStreamElapsedNanoseconds"), 10) +large_stream_elapsed_nanoseconds = Integer(observations.fetch("slowStreamElapsedNanoseconds"), 10) +raise "small slow-reader sampling deadline is not the reviewed 30-second bound" unless small_stream_deadline_seconds == 30 +raise "large slow-reader sampling deadline is not the reviewed 120-second bound" unless large_stream_deadline_seconds == 120 +raise "small slow-reader exceeded its sampling deadline" unless small_stream_elapsed_nanoseconds <= (small_stream_deadline_seconds + 1) * 1_000_000_000 +raise "large slow-reader exceeded its sampling deadline" unless large_stream_elapsed_nanoseconds <= (large_stream_deadline_seconds + 1) * 1_000_000_000 +raise "streaming byte evidence is too small" unless large_stream_bytes > 32 * 1024 * 1024 +raise "streaming chunk evidence is not framed" unless Integer(observations.fetch("slowStreamChunks"), 10) > 1 +raise "small slow-reader byte evidence is too small" unless small_stream_bytes > 8 * 1024 * 1024 +raise "small slow-reader response is not framed" unless Integer(observations.fetch("smallSlowStreamChunks"), 10) > 1 +raise "small slow-reader run lacks repeated active samples" unless Integer(observations.fetch("smallSlowStreamActiveSampleCount"), 10) > 1 +raise "large slow-reader run lacks repeated active samples" unless Integer(observations.fetch("slowStreamActiveSampleCount"), 10) > 1 +raise "protocol RTT was not measured" unless Float(observations.fetch("protocolRTTMedianMilliseconds")) > 0 +raise "stream throughput was not measured" unless Integer(observations.fetch("slowStreamBytesPerSecond"), 10) > 0 +queue_ceiling = Integer(observations.fetch("declaredQueueCeilingBytes"), 10) +maximum_footprint_delta = Integer(observations.fetch("maximumSlowStreamFootprintDeltaBytes"), 10) +required_headroom = Integer(observations.fetch("requiredSlowStreamAvailableMemoryHeadroomBytes"), 10) +minimum_headroom = Integer(observations.fetch("minimumSlowStreamAvailableMemoryBytes"), 10) +small_footprint = Integer(observations.fetch("smallSlowStreamPhysFootprintBytes"), 10) +large_footprint = Integer(observations.fetch("largeSlowStreamPhysFootprintBytes"), 10) +observed_delta = Integer(observations.fetch("slowStreamFootprintDeltaBytes"), 10) +peak_footprint = Integer(observations.fetch("slowStreamPeakPhysFootprintBytes"), 10) +response_size_delta = Integer(observations.fetch("slowStreamResponseSizeDeltaBytes"), 10) +raise "slow-reader footprint samples were not recorded" unless small_footprint.positive? && large_footprint.positive? +raise "declared slow-reader queue ceiling is not exactly 8 MiB" unless queue_ceiling == 8 * 1024 * 1024 +raise "declared slow-reader footprint delta is not exactly two queue ceilings" unless maximum_footprint_delta == 2 * queue_ceiling +raise "slow-reader response-size delta is inconsistent" unless response_size_delta == large_stream_bytes - small_stream_bytes +raise "slow-reader response-size delta does not exceed footprint bound" unless response_size_delta > maximum_footprint_delta +raise "slow-reader footprint delta is inconsistent" unless observed_delta == [large_footprint - small_footprint, 0].max +raise "slow-reader footprint delta exceeds its declared bound" unless observed_delta <= maximum_footprint_delta +raise "slow-reader headroom bound is not exactly one full queue ceiling" unless required_headroom == queue_ceiling +raise "slow-reader available headroom did not exceed its declared bound" unless minimum_headroom > required_headroom +raise "slow-reader peak footprint is inconsistent" unless peak_footprint == [small_footprint, large_footprint].max +prior_checkpoint_sample_sequence = Integer(observations.fetch("priorCheckpointMemorySampleSequence"), 10) +checkpoint_sample_sequence = Integer(observations.fetch("checkpointMemorySampleSequence"), 10) +checkpoint_started_at = Integer(observations.fetch("checkpointMemorySampleStartedAtUptimeNanoseconds"), 10) +checkpoint_sampled_at = Integer(observations.fetch("checkpointMemorySampledAtUptimeNanoseconds"), 10) +checkpoint_completed_at = Integer(observations.fetch("checkpointMemorySampleCompletedAtUptimeNanoseconds"), 10) +raise "checkpoint memory evidence was stale" unless checkpoint_sample_sequence > prior_checkpoint_sample_sequence +raise "checkpoint memory sample predates the checkpoint interval" unless checkpoint_started_at <= checkpoint_sampled_at +raise "checkpoint memory sample follows checkpoint completion" unless checkpoint_sampled_at <= checkpoint_completed_at +raise "checkpoint live flag remained set after completion" unless observations.fetch("checkpointInProgressAfterCompletion") == "false" + +diagnostics = Array(report["diagnostics"]) +required_phases = %w[openedIdle executingBeforeCancel slowStreaming8MiB slowStreaming32MiB checkpointMemorySample afterCheckpoint quiesced resumed] +missing_phases = required_phases - diagnostics.map { |entry| entry["phase"] } +raise "lifecycle memory evidence is missing phases: #{missing_phases.join(",")}" unless missing_phases.empty? +%w[slowStreaming8MiB slowStreaming32MiB].each do |phase| + entry = diagnostics.find { |candidate| candidate["phase"] == phase } + raise "#{phase} did not overlap an active request" unless entry["activeRequestID"].to_i.positive? + raise "#{phase} did not overlap native dispatch" unless entry["nativeDispatchStarted"] == true +end +checkpoint_entry = diagnostics.find { |entry| entry["phase"] == "checkpointMemorySample" } +raise "checkpoint live flag was reported as sticky" unless checkpoint_entry["checkpointInProgress"] == false +diagnostics.each do |entry| + next unless required_phases.include?(entry["phase"]) + raise "#{entry["phase"]} lacks physical-footprint evidence" unless entry["currentPhysFootprintBytes"].to_i.positive? + raise "#{entry["phase"]} lacks resident-memory evidence" unless entry["currentResidentBytes"].to_i.positive? + raise "#{entry["phase"]} lacks available-memory headroom" unless entry["availableMemoryBytes"].to_i.positive? +end + +protection = report.fetch("storageProtection") +expected_protection = "NSFileProtectionCompleteUntilFirstUserAuthentication" +raise "recursive protection expected the wrong protection class" unless protection.fetch("expectedProtection") == expected_protection +raise "recursive protection enumeration failed" if protection["enumerationFailed"] +raise "recursive protection evidence is empty" unless protection["entryCount"].to_i.positive? +raise "recursive protection found symlinks" unless protection["symbolicLinkCount"].to_i.zero? +raise "recursive protection has unreadable entries" unless protection["unreadableEntryCount"].to_i.zero? +raise "recursive protection is missing metadata" unless protection["missingProtectionCount"].to_i.zero? +raise "recursive protection has mismatches" unless protection["mismatchedProtectionCount"].to_i.zero? +raise "recursive protection metadata was unavailable" unless protection["protectionMetadataUnavailableCount"].to_i.zero? +raise "recursive protection count does not cover every entry" unless protection["matchingProtectionCount"] == protection["entryCount"] +raise "no relation files were audited" unless protection["relationFileCount"].to_i.positive? +raise "no WAL files were audited" unless protection["walFileCount"].to_i.positive? +write_started = report.fetch("writeStartedAtUnixNanoseconds") +raise "lifecycle write timestamp is invalid" unless write_started.is_a?(Integer) && write_started.positive? +timestamp_tolerance = 2_000_000_000 +earliest_fresh_modification = [write_started - timestamp_tolerance, 0].max +newest_relation = protection.fetch("newestRelationModificationUnixNanoseconds") +newest_wal = protection.fetch("newestWALModificationUnixNanoseconds") +raise "newest relation timestamp is invalid" unless newest_relation.is_a?(Integer) && newest_relation.positive? +raise "newest WAL timestamp is invalid" unless newest_wal.is_a?(Integer) && newest_wal.positive? +raise "newest relation predates the lifecycle write" unless newest_relation >= earliest_fresh_modification +raise "newest WAL predates the lifecycle write" unless newest_wal >= earliest_fresh_modification + +worker_termination_evidence = case worker_termination_mode +when "explicitSIGKILL" + { + workerAbsentAtPostSuspendInventory: !worker_present_at_post_suspend_inventory, + workerLossWindow: "explicitSIGKILLAfterPostSuspendInventory", + terminationCause: "explicitSIGKILL", + intentionalSIGKILLDelivered: true, + workerUnavailableBeforeResume: true, + postTerminationInventoryConfirmed: true, + } +when "workerAbsentAtPostSuspendInventory" + { + workerAbsentAtPostSuspendInventory: true, + workerLossWindow: "afterQuiescedEvidenceThroughPostSuspendInventory", + terminationCause: "unattributed", + intentionalSIGKILLDelivered: false, + workerUnavailableBeforeResume: true, + postTerminationInventoryConfirmed: true, + } +when "exitedDuringKillRace" + { + workerAbsentAtPostSuspendInventory: false, + workerLossWindow: "afterPostSuspendInventoryThroughPostESRCHInventory", + terminationCause: "unattributed", + intentionalSIGKILLDelivered: false, + workerUnavailableBeforeResume: true, + postTerminationInventoryConfirmed: true, + } +when "notRequested" + { + workerAbsentAtPostSuspendInventory: !worker_present_at_post_suspend_inventory, + workerLossWindow: worker_present_at_post_suspend_inventory ? + "notApplicable" : "afterQuiescedEvidenceThroughPostSuspendInventory", + terminationCause: worker_present_at_post_suspend_inventory ? + "notRequested" : "unattributed", + intentionalSIGKILLDelivered: false, + workerUnavailableBeforeResume: !worker_present_at_post_suspend_inventory, + postTerminationInventoryConfirmed: false, + } +else + raise "unrecognized worker termination evidence mode" +end + +File.write(validation_path, JSON.pretty_generate({ + status: "PASS", + launchIndex: launch_index, + expectedWorkerKill: expect_worker_kill, + workerTerminationMode: worker_termination_mode, + workerTerminationEvidence: worker_termination_evidence, + suspensionEvidence: { + externallyConfirmed: true, + deviceIdentifier: expected_device_id, + hostPID: host_pid, + signalName: suspend_signal&.fetch("name", nil), + signalValue: suspend_signal&.fetch("value", nil), + hostCountAtPostSuspendInventory: suspended_process_ids.count(host_pid), + workerCountAtPostSuspendInventory: suspended_process_ids.count(initial_pid), + foregroundInventoryPath: foreground_inventory_path, + foregroundHostCount: foreground_process_ids.count(host_pid), + foregroundWorkerCount: foreground_process_ids.count(initial_pid), + suspendResultPath: suspend_result_path, + postSuspendInventoryPath: suspended_inventory_path, + workerTerminateResultPath: worker_terminate_result_path, + postTerminateInventoryPath: post_terminate_inventory_path, + }, + hostPID: host_pid, + initialWorkerPID: initial_pid, + resumedWorkerPID: resumed_pid, + initialEpoch: initial_epoch, + resumedEpoch: resumed_epoch, + checks: checks, + diagnosticPhases: diagnostics.map { |entry| entry["phase"] }, + storageProtection: protection, +}) + "\n") +RUBY +} + +run_lifecycle_launch() { + local launch_index="$1" + local expect_worker_kill="$2" + local report_path validation_path launch_json lifecycle_log + if [ "$launch_index" = "1" ]; then + report_path="$lifecycle_launch_one_report" + validation_path="$lifecycle_launch_one_validation" + launch_json="$lifecycle_launch_one_result" + lifecycle_log="$lifecycle_launch_one_log" + else + report_path="$lifecycle_launch_two_report" + validation_path="$lifecycle_launch_two_validation" + launch_json="$lifecycle_launch_two_result" + lifecycle_log="$lifecycle_launch_two_log" + fi + local background_json="$reports_dir/devicectl-lifecycle-background-$launch_index.json" + local foreground_processes_json="$reports_dir/devicectl-lifecycle-processes-foreground-$launch_index.json" + local suspend_json="$reports_dir/devicectl-lifecycle-suspend-$launch_index.json" + local suspended_processes_json="$reports_dir/devicectl-lifecycle-processes-suspended-$launch_index.json" + local kill_json="$reports_dir/devicectl-lifecycle-worker-kill-$launch_index.json" + local post_kill_processes_json="$reports_dir/devicectl-lifecycle-processes-post-worker-termination-$launch_index.json" + local resume_json="$reports_dir/devicectl-lifecycle-resume-$launch_index.json" + local activate_json="$reports_dir/devicectl-lifecycle-reactivate-$launch_index.json" + local terminate_json="$reports_dir/devicectl-lifecycle-terminate-$launch_index.json" + local host_pid worker_pid activated_pid foreground_worker_state + local suspended_worker_state post_kill_worker_state + local worker_termination_mode="notRequested" + local launch_attempt=0 + local lock_retry_deadline=$((SECONDS + 120)) + + printf 'Launching detached Release lifecycle probe %s of 2 (workerKill=%s)...\n' \ + "$launch_index" "$expect_worker_kill" + : >"$kill_json" + : >"$post_kill_processes_json" + while :; do + launch_attempt=$((launch_attempt + 1)) + : >"$launch_json" + : >"$lifecycle_log" + if xcrun devicectl device process launch \ + --device "$selected_device_id" \ + --terminate-existing \ + --activate \ + --environment-variables "{\"NSUnbufferedIO\":\"YES\",\"OLIPHAUNT_BROKER_FIXTURE_MODE\":\"lifecycle\",\"OLIPHAUNT_BROKER_FIXTURE_DISABLE_IDLE_TIMER\":\"YES\",\"OLIPHAUNT_BROKER_LIFECYCLE_RUN_TOKEN\":\"$lifecycle_run_token\",\"OLIPHAUNT_BROKER_LIFECYCLE_LAUNCH_INDEX\":\"$launch_index\",\"OLIPHAUNT_BROKER_LIFECYCLE_EXPECT_WORKER_KILL\":\"$expect_worker_kill\",\"OLIPHAUNT_BROKER_LIFECYCLE_ORCHESTRATION_TIMEOUT_SECONDS\":\"$timeout_seconds\",\"OLIPHAUNT_BROKER_BUILD_CONFIGURATION\":\"$lifecycle_configuration\"}" \ + --timeout 30 \ + --json-output "$launch_json" \ + "$app_bundle_id" >"$lifecycle_log" 2>&1; then + break + fi + if is_explicit_locked_launch_failure "$launch_json"; then + [ "$SECONDS" -lt "$lock_retry_deadline" ] || \ + fail "device remained locked for the bounded lifecycle pre-launch retry window" + printf 'Device explicitly rejected lifecycle probe %s pre-launch as Locked; waiting to retry (%s)...\n' \ + "$launch_index" "$launch_attempt" + sleep 2 + continue + fi + fail "failed to launch detached lifecycle probe $launch_index" + done + host_pid="$(ruby -rjson -e 'puts JSON.parse(File.read(ARGV.fetch(0))).dig("result", "process", "processIdentifier")' "$launch_json")" + case "$host_pid" in ''|*[!0-9]*) fail "lifecycle launch $launch_index returned an invalid host PID" ;; esac + + wait_for_lifecycle_foreground_active "$launch_index" "$report_path" + wait_for_lifecycle_phase "$launch_index" readyForBackground "$report_path" + [ "$(lifecycle_report_integer "$report_path" hostPID)" = "$host_pid" ] || \ + fail "lifecycle launch $launch_index report host PID disagrees with devicectl" + worker_pid="$(lifecycle_report_integer "$report_path" initialWorkerPID)" + + : >"$foreground_processes_json" + xcrun devicectl device info processes \ + --device "$selected_device_id" \ + --filter "processIdentifier == $host_pid OR processIdentifier == $worker_pid" \ + --columns '*' \ + --timeout 30 \ + --json-output "$foreground_processes_json" \ + >>"$lifecycle_log" 2>&1 || \ + fail "failed to inventory lifecycle processes while foreground-ready" + if ! foreground_worker_state="$(classify_suspended_process_inventory \ + "$foreground_processes_json" "$host_pid" "$worker_pid" \ + "$selected_device_id")"; then + fail "invalid lifecycle process inventory while foreground-ready" + fi + [ "$foreground_worker_state" = "workerPresent" ] || \ + fail "foreground-ready inventory did not prove worker PID $worker_pid visibility" + + # Activating a different public system app causes a real scene transition. + # The fixture disables only the display idle timer while its scene is active; + # inactive/background phases disable it and request no background execution. + xcrun devicectl device process launch \ + --device "$selected_device_id" \ + --terminate-existing \ + --activate \ + --timeout 30 \ + --json-output "$background_json" \ + com.apple.Preferences >>"$lifecycle_log" 2>&1 || \ + fail "failed to foreground Settings for lifecycle launch $launch_index" + wait_for_lifecycle_phase "$launch_index" quiesced "$report_path" + + xcrun devicectl device process suspend \ + --device "$selected_device_id" \ + --pid "$host_pid" \ + --timeout 30 \ + --json-output "$suspend_json" \ + >>"$lifecycle_log" 2>&1 || \ + fail "failed to suspend backgrounded broker host PID $host_pid" + sleep 4 + : >"$suspended_processes_json" + xcrun devicectl device info processes \ + --device "$selected_device_id" \ + --filter "processIdentifier == $host_pid OR processIdentifier == $worker_pid" \ + --columns '*' \ + --timeout 30 \ + --json-output "$suspended_processes_json" \ + >>"$lifecycle_log" 2>&1 || \ + fail "failed to inventory lifecycle processes while suspended" + if ! suspended_worker_state="$(classify_suspended_process_inventory \ + "$suspended_processes_json" "$host_pid" "$worker_pid" \ + "$selected_device_id")"; then + fail "invalid lifecycle process inventory while host PID $host_pid was suspended" + fi + + if [ "$expect_worker_kill" = "YES" ]; then + if xcrun devicectl device process terminate \ + --device "$selected_device_id" \ + --pid "$worker_pid" \ + --kill \ + --timeout 30 \ + --json-output "$kill_json" \ + >>"$lifecycle_log" 2>&1; then + worker_termination_mode="explicitSIGKILL" + elif is_exact_devicectl_esrch_failure "$kill_json"; then + case "$suspended_worker_state" in + workerAbsent) + worker_termination_mode="workerAbsentAtPostSuspendInventory" + ;; + workerPresent) + worker_termination_mode="exitedDuringKillRace" + ;; + *) + fail "unrecognized suspended worker inventory state: $suspended_worker_state" + ;; + esac + else + fail "failed to SIGKILL background worker PID $worker_pid" + fi + + : >"$post_kill_processes_json" + xcrun devicectl device info processes \ + --device "$selected_device_id" \ + --filter "processIdentifier == $host_pid OR processIdentifier == $worker_pid" \ + --columns '*' \ + --timeout 30 \ + --json-output "$post_kill_processes_json" \ + >>"$lifecycle_log" 2>&1 || \ + fail "failed to inventory lifecycle processes after worker termination" + if ! post_kill_worker_state="$(classify_suspended_process_inventory \ + "$post_kill_processes_json" "$host_pid" "$worker_pid" \ + "$selected_device_id")"; then + fail "invalid lifecycle process inventory after worker termination" + fi + [ "$post_kill_worker_state" = "workerAbsent" ] || \ + fail "background worker PID $worker_pid remained present after worker termination" + fi + + xcrun devicectl device process resume \ + --device "$selected_device_id" \ + --pid "$host_pid" \ + --timeout 30 \ + --json-output "$resume_json" \ + >>"$lifecycle_log" 2>&1 || \ + fail "failed to resume lifecycle host PID $host_pid" + xcrun devicectl device process launch \ + --device "$selected_device_id" \ + --activate \ + --timeout 30 \ + --json-output "$activate_json" \ + "$app_bundle_id" >>"$lifecycle_log" 2>&1 || \ + fail "failed to reactivate lifecycle host PID $host_pid" + activated_pid="$(ruby -rjson -e 'puts JSON.parse(File.read(ARGV.fetch(0))).dig("result", "process", "processIdentifier")' "$activate_json")" + [ "$activated_pid" = "$host_pid" ] || \ + fail "reactivation replaced lifecycle host PID $host_pid with $activated_pid" + + wait_for_lifecycle_phase "$launch_index" completed "$report_path" + validate_lifecycle_report \ + "$report_path" "$validation_path" "$launch_index" "$expect_worker_kill" \ + "$worker_termination_mode" "$foreground_processes_json" "$suspend_json" \ + "$suspended_processes_json" "$kill_json" "$post_kill_processes_json" \ + "$selected_device_id" || \ + fail "lifecycle launch $launch_index report validation failed" + xcrun devicectl device process terminate \ + --device "$selected_device_id" \ + --pid "$host_pid" \ + --timeout 30 \ + --json-output "$terminate_json" \ + >>"$lifecycle_log" 2>&1 || \ + fail "failed to terminate completed lifecycle host PID $host_pid" +} + +validate_extension_private_persistence() { + ruby -rjson - "$launch_one_app_report" "$launch_two_app_report" \ + "$persistence_report" <<'RUBY' +first_path, second_path, output_path = ARGV +reports = [first_path, second_path].map { |path| JSON.parse(File.read(path)) } +results = reports.map { |report| report.fetch("result") } +digests = results.map do |result| + values = Array(result["diagnostics"]).map { |entry| entry["manifestDigest"] }.compact + .reject(&:empty?).uniq + raise "launch has no unique worker manifestDigest" unless values.length == 1 + values.fetch(0) +end +raise "two launches reused the same host process" if results[0]["hostPID"] == results[1]["hostPID"] +raise "extension-private manifest digest changed across launches" unless digests[0] == digests[1] +observations = results.map { |result| result.fetch("observations") } +first_marker = observations[0].fetch("currentLaunchMarker") +second_marker = observations[1].fetch("currentLaunchMarker") +raise "first launch marker is empty" if first_marker.empty? +raise "second launch marker is empty" if second_marker.empty? +raise "two launches reused the same durable marker" if first_marker == second_marker +second_prior_markers = observations[1].fetch("priorLaunchMarkers").split(",").reject(&:empty?) +raise "second launch did not observe the first launch marker" unless second_prior_markers.include?(first_marker) +File.write(output_path, JSON.pretty_generate({ + schema: "oliphaunt-ios-broker-device-persistence-v1", + status: "PASS", + evidence: "two-full-launches-without-reinstall", + manifestDigest: digests.fetch(0), + launchHostPIDs: results.map { |result| result.fetch("hostPID") }, + launchWorkerPIDs: results.map { |result| result.fetch("workerPID") }, + launchEpochs: results.map { |result| result.fetch("epoch") }, + firstLaunchMarker: first_marker, + secondLaunchMarker: second_marker, + secondLaunchPriorMarkers: second_prior_markers, + cleanupRequestedByLaunch: 2, + reports: [first_path, second_path], +}) + "\n") +RUBY +} + +retain_semantic_debug_product() { + local retained_root retained_app retained_extension + local host_hash extension_hash + [ -d "$semantic_app_path" ] || fail "semantic Debug app disappeared before retention" + [ -d "$semantic_extension_path" ] || \ + fail "semantic Debug extension disappeared before retention" + retained_root="$(mktemp -d "$build_root/retained-semantic-debug.XXXXXX")" + retained_app="$retained_root/$app_product_name.app" + retained_extension="$retained_app/Extensions/$extension_product_name.appex" + ditto "$semantic_app_path" "$retained_app" || \ + fail "failed to retain the signed semantic Debug app" + [ -d "$retained_extension" ] || \ + fail "retained semantic Debug app omitted its extension" + codesign --verify --strict --deep "$retained_app" || \ + fail "retained semantic Debug app signature is invalid" + codesign --verify --strict "$retained_extension" || \ + fail "retained semantic Debug extension signature is invalid" + [ "$(plutil -extract CFBundleIdentifier raw -o - "$retained_app/Info.plist")" = \ + "$app_bundle_id" ] || fail "retained semantic Debug app has the wrong bundle identifier" + [ "$(plutil -extract CFBundleIdentifier raw -o - "$retained_extension/Info.plist")" = \ + "$extension_bundle_id" ] || \ + fail "retained semantic Debug extension has the wrong bundle identifier" + host_hash="$(shasum -a 256 "$retained_app/$host_executable" | awk '{print $1}')" + extension_hash="$(shasum -a 256 \ + "$retained_extension/$extension_executable" | awk '{print $1}')" + ruby -rjson - "$retained_semantic_product_validation" "$retained_app" \ + "$retained_extension" "$build_result_bundle" "$host_hash" \ + "$extension_hash" <<'RUBY' +output, app, extension, result_bundle, host_hash, extension_hash = ARGV +File.write(output, JSON.pretty_generate({ + schema: "oliphaunt-ios-broker-retained-semantic-debug-product-v1", + status: "PASS", + appPath: app, + extensionPath: extension, + resultBundle: result_bundle, + hostExecutableSHA256: host_hash, + extensionExecutableSHA256: extension_hash, +}) + "\n") +RUBY + semantic_app_path="$retained_app" + semantic_extension_path="$retained_extension" +} + +load_retained_semantic_debug_product() { + local expected_result_bundle="$1" + local retained_result_bundle retained_host_hash retained_extension_hash + [ -s "$retained_semantic_product_validation" ] || \ + fail "resume lifecycle requires a retained semantic Debug product report" + semantic_app_path="$(ruby -rjson -e ' + report = JSON.parse(File.read(ARGV.fetch(0))) + abort unless report["schema"] == "oliphaunt-ios-broker-retained-semantic-debug-product-v1" + abort unless report["status"] == "PASS" + puts report.fetch("appPath") + ' "$retained_semantic_product_validation")" + semantic_extension_path="$(ruby -rjson -e ' + puts JSON.parse(File.read(ARGV.fetch(0))).fetch("extensionPath") + ' "$retained_semantic_product_validation")" + retained_result_bundle="$(ruby -rjson -e ' + puts JSON.parse(File.read(ARGV.fetch(0))).fetch("resultBundle") + ' "$retained_semantic_product_validation")" + case "$semantic_app_path" in + "$build_root"/retained-semantic-debug.*/*) ;; + *) fail "retained semantic Debug app is outside the device build root" ;; + esac + [ "$semantic_extension_path" = \ + "$semantic_app_path/Extensions/$extension_product_name.appex" ] || \ + fail "retained semantic Debug extension path is inconsistent" + [ "$retained_result_bundle" = "$expected_result_bundle" ] || \ + fail "retained semantic Debug product names a different result bundle" + [ -d "$semantic_app_path" ] || fail "retained semantic Debug app is missing" + [ -d "$semantic_extension_path" ] || \ + fail "retained semantic Debug extension is missing" + codesign --verify --strict --deep "$semantic_app_path" || \ + fail "retained semantic Debug app signature is invalid" + codesign --verify --strict "$semantic_extension_path" || \ + fail "retained semantic Debug extension signature is invalid" + retained_host_hash="$(ruby -rjson -e ' + puts JSON.parse(File.read(ARGV.fetch(0))).fetch("hostExecutableSHA256") + ' "$retained_semantic_product_validation")" + retained_extension_hash="$(ruby -rjson -e ' + puts JSON.parse(File.read(ARGV.fetch(0))).fetch("extensionExecutableSHA256") + ' "$retained_semantic_product_validation")" + [ "$(shasum -a 256 "$semantic_app_path/$host_executable" | awk '{print $1}')" = \ + "$retained_host_hash" ] || fail "retained semantic Debug host executable changed" + [ "$(shasum -a 256 \ + "$semantic_extension_path/$extension_executable" | awk '{print $1}')" = \ + "$retained_extension_hash" ] || \ + fail "retained semantic Debug extension executable changed" +} + +write_release_product_sizes() { + ruby -rjson - "$lifecycle_size_report" "$app_path" "$extension_path" \ + "$embedded_native_framework" "$app_path/$host_executable" \ + "$extension_path/$extension_executable" "$extension_path/oliphaunt" <<'RUBY' +output, app, extension, framework, host_executable, extension_executable, resources = ARGV +def allocated_bytes(path) + paths = [path] + Dir.glob(File.join(path, "**", "*"), File::FNM_DOTMATCH) + paths.uniq.sum do |entry| + next 0 unless File.file?(entry) + File.size(entry) + rescue Errno::ENOENT + 0 + end +end +payload = { + schema: "oliphaunt-ios-broker-release-product-sizes-v1", + status: "PASS", + appBundleBytes: allocated_bytes(app), + extensionBundleBytes: allocated_bytes(extension), + nativeFrameworkBytes: allocated_bytes(framework), + runtimeResourcesBytes: allocated_bytes(resources), + hostExecutableBytes: File.size(host_executable), + extensionExecutableBytes: File.size(extension_executable), +} +raise "release size evidence contains an empty product" unless payload.values_at( + :appBundleBytes, :extensionBundleBytes, :nativeFrameworkBytes, + :runtimeResourcesBytes, :hostExecutableBytes, :extensionExecutableBytes +).all?(&:positive?) +File.write(output, JSON.pretty_generate(payload) + "\n") +RUBY +} + +write_lifecycle_runner_report() { + local completed_at + completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + ruby -rjson - "$lifecycle_runner_report" "$lifecycle_launch_one_report" \ + "$lifecycle_launch_two_report" "$lifecycle_launch_one_validation" \ + "$lifecycle_launch_two_validation" "$lifecycle_size_report" \ + "$lifecycle_run_token" "$selected_device_id" "$selected_device_udid" \ + "$selected_device_name" "$selected_device_os" "$selected_device_product" \ + "$selected_device_transport" "$lifecycle_configuration" "$app_path" \ + "$extension_path" "$lifecycle_build_result_bundle" "$lifecycle_build_log" \ + "$lifecycle_install_log" "$lifecycle_release_build_app_path" \ + "$lifecycle_archive_path" "$lifecycle_archive_result_bundle" \ + "$lifecycle_archive_validation" "$lifecycle_archive_log" "$completed_at" <<'RUBY' +output, first_report, second_report, first_validation, second_validation, sizes, + run_token, identifier, udid, name, os, product, transport, configuration, + app_path, extension_path, result_bundle, build_log, install_log, + release_build_app_path, archive_path, archive_result_bundle, + archive_validation, archive_log, completed_at = ARGV +first = JSON.parse(File.read(first_report)) +second = JSON.parse(File.read(second_report)) +first_validation_payload = JSON.parse(File.read(first_validation)) +second_validation_payload = JSON.parse(File.read(second_validation)) +validations = [first_validation_payload, second_validation_payload] +reports = [first, second] +raise "lifecycle validation artifact did not pass" unless validations.all? { |validation| validation["status"] == "PASS" } +raise "launch one validation unexpectedly requested worker termination" unless first_validation_payload["expectedWorkerKill"] == false +raise "launch two validation omitted worker-unavailability recovery" unless second_validation_payload["expectedWorkerKill"] == true +raise "launch one validation recorded a termination mode" unless first_validation_payload["workerTerminationMode"] == "notRequested" +first_worker_termination_evidence = first_validation_payload.fetch("workerTerminationEvidence") +raise "launch one validation falsely claimed SIGKILL" unless first_worker_termination_evidence["intentionalSIGKILLDelivered"] == false +raise "launch one worker-unavailability evidence is inconsistent" unless first_worker_termination_evidence["workerUnavailableBeforeResume"] == first_worker_termination_evidence["workerAbsentAtPostSuspendInventory"] +reports.zip(validations).each_with_index do |(report, validation), index| + raise "lifecycle validation launch index mismatch" unless validation["launchIndex"] == report["launchIndex"] + raise "lifecycle validation ordinal mismatch" unless validation["launchIndex"] == index + 1 + raise "lifecycle validation host PID mismatch" unless validation["hostPID"] == report["hostPID"] + raise "lifecycle validation initial worker PID mismatch" unless validation["initialWorkerPID"] == report["initialWorkerPID"] + raise "lifecycle validation resumed worker PID mismatch" unless validation["resumedWorkerPID"] == report["currentWorkerPID"] + raise "lifecycle validation initial epoch mismatch" unless validation["initialEpoch"] == report["initialEpoch"] + raise "lifecycle validation resumed epoch mismatch" unless validation["resumedEpoch"] == report["currentEpoch"] +end +raise "lifecycle launches reused one host process" if first.fetch("hostPID") == second.fetch("hostPID") +raise "extension-private manifest changed across lifecycle launches" unless first.fetch("manifestDigest") == second.fetch("manifestDigest") +raise "lifecycle launch one found stale run-token markers" unless Integer(first.fetch("observations").fetch("priorLaunchMarkerCount"), 10) == 0 +raise "lifecycle launch two did not observe launch one's marker" unless Integer(second.fetch("observations").fetch("priorLaunchMarkerCount"), 10) == 1 +suspensions = validations.map { |validation| validation.fetch("suspensionEvidence") } +externally_confirmed_suspension = suspensions.all? do |evidence| + evidence["externallyConfirmed"] == true && + evidence["deviceIdentifier"] == identifier && + evidence["signalName"] == "SIGSTOP" && + evidence["signalValue"] == 17 && + evidence["foregroundHostCount"] == 1 && + evidence["foregroundWorkerCount"] == 1 && + evidence["hostCountAtPostSuspendInventory"] == 1 +end +raise "lifecycle validations did not independently confirm both suspensions" unless externally_confirmed_suspension +worker_termination_mode = second_validation_payload.fetch("workerTerminationMode") +worker_termination_evidence = second_validation_payload.fetch("workerTerminationEvidence") +case worker_termination_mode +when "explicitSIGKILL" + raise "explicit SIGKILL validation denied delivery" unless worker_termination_evidence["intentionalSIGKILLDelivered"] == true +when "workerAbsentAtPostSuspendInventory" + raise "absent-worker validation did not observe absence" unless worker_termination_evidence["workerAbsentAtPostSuspendInventory"] == true + raise "absent-worker validation falsely claimed SIGKILL" unless worker_termination_evidence["intentionalSIGKILLDelivered"] == false +when "exitedDuringKillRace" + raise "kill-race validation falsely claimed SIGKILL" unless worker_termination_evidence["intentionalSIGKILLDelivered"] == false +else + raise "launch two validation has an unrecognized worker termination mode" +end +worker_unavailable_before_resume = worker_termination_evidence.fetch("workerUnavailableBeforeResume") == true +raise "launch two did not prove worker unavailability before resume" unless worker_unavailable_before_resume +actual_foreground_background_foreground = validations.all? do |validation| + checks = Array(validation["checks"]) + checks.include?("actualBackground") && checks.include?("backgroundResume") +end +payload = { + schema: "oliphaunt-ios-broker-device-lifecycle-run-v2", + status: "PASS", + evidenceType: "signed-release-physical-device-lifecycle", + completedAt: completed_at, + runToken: run_token, + device: { + coreDeviceIdentifier: identifier, + udid: udid, + name: name, + os: os, + productType: product, + transport: transport, + }, + build: { + sdk: "iphoneos", + architecture: "arm64", + configuration: configuration, + builtAppPath: release_build_app_path, + appPath: app_path, + extensionPath: extension_path, + resultBundle: result_bundle, + archivePath: archive_path, + archiveResultBundle: archive_result_bundle, + }, + scope: { + actualForegroundBackgroundForeground: actual_foreground_background_foreground, + externallyConfirmedSuspension: externally_confirmed_suspension, + workerUnavailableBeforeResume: worker_unavailable_before_resume, + workerTerminationMode: worker_termination_mode, + workerKilledWhileHostSuspended: + worker_termination_evidence.fetch("intentionalSIGKILLDelivered"), + workerAbsentAtPostSuspendInventory: + worker_termination_evidence.fetch("workerAbsentAtPostSuspendInventory"), + workerLossWindow: worker_termination_evidence.fetch("workerLossWindow"), + terminationCause: worker_termination_evidence.fetch("terminationCause"), + intentionalSIGKILLDelivered: + worker_termination_evidence.fetch("intentionalSIGKILLDelivered"), + backgroundKeepaliveUsed: false, + distributionQualification: false, + }, + launches: [ + { + ordinal: 1, + workerTerminationMode: first_validation_payload.fetch("workerTerminationMode"), + workerKilledWhileSuspended: + first_validation_payload.fetch("workerTerminationEvidence").fetch("intentionalSIGKILLDelivered"), + workerUnavailableBeforeResume: + first_validation_payload.fetch("workerTerminationEvidence").fetch("workerUnavailableBeforeResume"), + reportPath: first_report, + report: first, + validation: first_validation_payload, + }, + { + ordinal: 2, + workerTerminationMode: worker_termination_mode, + workerKilledWhileSuspended: + worker_termination_evidence.fetch("intentionalSIGKILLDelivered"), + workerUnavailableBeforeResume: worker_unavailable_before_resume, + reportPath: second_report, + report: second, + validation: second_validation_payload, + }, + ], + productSizes: JSON.parse(File.read(sizes)), + archiveValidation: JSON.parse(File.read(archive_validation)), + performance: { + broker: [first, second].map do |report| + observations = report.fetch("observations") + { + launchIndex: report.fetch("launchIndex"), + protocolRTTMedianMilliseconds: + Float(observations.fetch("protocolRTTMedianMilliseconds")), + protocolRTTSampleCount: + Integer(observations.fetch("protocolRTTSampleCount"), 10), + smallSlowStreamBytes: + Integer(observations.fetch("smallSlowStreamBytes"), 10), + smallSlowStreamActiveSampleCount: + Integer(observations.fetch("smallSlowStreamActiveSampleCount"), 10), + smallSlowStreamElapsedNanoseconds: + Integer(observations.fetch("smallSlowStreamElapsedNanoseconds"), 10), + smallSlowStreamSamplingDeadlineSeconds: + Integer(observations.fetch("smallSlowStreamSamplingDeadlineSeconds"), 10), + slowStreamBytes: Integer(observations.fetch("slowStreamBytes"), 10), + slowStreamActiveSampleCount: + Integer(observations.fetch("slowStreamActiveSampleCount"), 10), + slowStreamElapsedNanoseconds: + Integer(observations.fetch("slowStreamElapsedNanoseconds"), 10), + slowStreamSamplingDeadlineSeconds: + Integer(observations.fetch("slowStreamSamplingDeadlineSeconds"), 10), + slowStreamBytesPerSecond: + Integer(observations.fetch("slowStreamBytesPerSecond"), 10), + declaredQueueCeilingBytes: + Integer(observations.fetch("declaredQueueCeilingBytes"), 10), + maximumSlowStreamFootprintDeltaBytes: + Integer(observations.fetch("maximumSlowStreamFootprintDeltaBytes"), 10), + slowStreamResponseSizeDeltaBytes: + Integer(observations.fetch("slowStreamResponseSizeDeltaBytes"), 10), + slowStreamFootprintDeltaBytes: + Integer(observations.fetch("slowStreamFootprintDeltaBytes"), 10), + minimumSlowStreamAvailableMemoryBytes: + Integer(observations.fetch("minimumSlowStreamAvailableMemoryBytes"), 10), + slowStreamPeakPhysFootprintBytes: + Integer(observations.fetch("slowStreamPeakPhysFootprintBytes"), 10), + } + end, + directModeComparison: { + status: "not-run", + reason: "the signed ExtensionFoundation fixture has no in-process nativeDirect host target; compare against the dedicated nativeDirect benchmark before production sizing", + }, + }, + logs: { build: build_log, archive: archive_log, install: install_log }, +} +File.write(output, JSON.pretty_generate(payload) + "\n") +RUBY +} + +write_runner_report() { + local completed_at + completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + ruby -rjson - "$runner_report_path" "$launch_one_app_report" \ + "$launch_two_app_report" "$persistence_report" "$selected_device_id" \ + "$selected_device_udid" \ + "$selected_device_name" "$selected_device_os" "$selected_device_product" \ + "$selected_device_transport" "$scheme" "$configuration" "$app_bundle_id" \ + "$extension_bundle_id" "$app_path" "$extension_path" \ + "$OLIPHAUNT_IOS_BROKER_XCFRAMEWORK" "$OLIPHAUNT_IOS_BROKER_RESOURCES" \ + "$embedded_native_library" "$artifact_validation_file" "$host_linkage_file" \ + "$extension_linkage_file" "$extension_resources_file" "$signing_validation_file" \ + "$device_inventory" "$device_details" "$device_lock_state" "$installed_apps" \ + "$install_result" "$launch_one_result" "$launch_two_result" \ + "$launch_one_copy_result" "$launch_two_copy_result" \ + "$artifact_preparation_log" "$build_log" "$install_log" "$launch_one_log" \ + "$launch_two_log" "$build_result_bundle" "$launch_one_pass_marker" \ + "$launch_two_pass_marker" "$completed_at" <<'RUBY' +output, first_app_report, second_app_report, persistence_report, identifier, + udid, name, os, product, transport, scheme, + configuration, app_bundle_id, extension_bundle_id, app_path, extension_path, + xcframework, runtime_resources, embedded_native_library, artifact_validation, + host_linkage, extension_linkage, extension_resources, signing_validation, + device_inventory, device_details, lock_state, installed_apps, install_result, + first_launch_result, second_launch_result, first_copy_result, second_copy_result, + artifact_preparation_log, build_log, install_log, first_console_log, + second_console_log, result_bundle, first_pass_marker_path, + second_pass_marker_path, completed_at = ARGV +payload = { + schema: "oliphaunt-ios-broker-device-run-v1", + status: "PASS", + evidenceType: "physical-device", + completedAt: completed_at, + device: { + coreDeviceIdentifier: identifier, + udid: udid, + name: name, + os: os, + productType: product, + transport: transport, + }, + build: { + sdk: "iphoneos", + architecture: "arm64", + scheme: scheme, + configuration: configuration, + appPath: app_path, + embeddedExtensionPath: extension_path, + resultBundle: result_bundle, + }, + bundleIdentifiers: { host: app_bundle_id, extension: extension_bundle_id }, + artifacts: { + platform: "ios-device", + xcframework: xcframework, + runtimeResources: runtime_resources, + embeddedNativeLibrary: embedded_native_library, + }, + validations: { + artifacts: artifact_validation, + hostLinkage: host_linkage, + extensionLinkage: extension_linkage, + extensionResources: extension_resources, + codeSigning: signing_validation, + }, + evidence: { + deviceInventory: device_inventory, + deviceDetails: device_details, + lockState: lock_state, + installedApps: installed_apps, + installResult: install_result, + launchResults: [first_launch_result, second_launch_result], + reportCopyResults: [first_copy_result, second_copy_result], + }, + logs: { + artifactPreparation: artifact_preparation_log, + xcodebuild: build_log, + install: install_log, + deviceConsoles: [first_console_log, second_console_log], + }, + launches: [ + { + ordinal: 1, + passMarker: File.read(first_pass_marker_path).strip, + appReportPath: first_app_report, + appReport: JSON.parse(File.read(first_app_report)), + }, + { + ordinal: 2, + passMarker: File.read(second_pass_marker_path).strip, + appReportPath: second_app_report, + appReport: JSON.parse(File.read(second_app_report)), + }, + ], + persistence: JSON.parse(File.read(persistence_report)), +} +File.write(output, JSON.pretty_generate(payload) + "\n") +RUBY +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +resume_lifecycle_only="$(normalize_yes_no \ + "$resume_lifecycle_only" OLIPHAUNT_IOS_BROKER_RESUME_LIFECYCLE_ONLY)" +resume_after_debug_install="$(normalize_yes_no \ + "$resume_after_debug_install" OLIPHAUNT_IOS_BROKER_RESUME_AFTER_DEBUG_INSTALL)" +[ "$resume_lifecycle_only" != "YES" ] || [ "$resume_after_debug_install" != "YES" ] || \ + fail "resume-after-debug-install and resume-lifecycle are mutually exclusive" +mkdir -p "$build_root" "$derived_data" "$logs_dir" "$reports_dir" +if [ "$resume_lifecycle_only" = "YES" ]; then + for output_file in \ + "$lifecycle_launch_one_log" "$lifecycle_launch_two_log" \ + "$lifecycle_launch_one_report" "$lifecycle_launch_two_report" \ + "$lifecycle_launch_one_validation" "$lifecycle_launch_two_validation" \ + "$lifecycle_launch_one_result" "$lifecycle_launch_two_result" \ + "$lifecycle_runner_report" "$runner_report_path" \ + "$device_inventory" "$device_details" "$device_lock_state" \ + "$preflight_report" "$artifact_validation_file" \ + "$embedded_extensions_file" "$host_linkage_file" "$extension_linkage_file" \ + "$embedded_native_file" "$extension_resources_file" "$signing_validation_file" \ + "$extension_host_sdk_symbol_file" "$release_fault_symbol_file" \ + "$reports_dir/failure.txt"; do + : >"$output_file" + done +elif [ "$resume_after_debug_install" = "YES" ]; then + for output_file in \ + "$launch_one_log" "$launch_two_log" "$launch_one_copy_log" "$launch_two_copy_log" \ + "$artifact_validation_file" \ + "$embedded_extensions_file" "$host_linkage_file" "$extension_linkage_file" \ + "$embedded_native_file" "$extension_resources_file" "$signing_validation_file" \ + "$extension_host_sdk_symbol_file" \ + "$release_fault_symbol_file" \ + "$launch_one_pass_marker" "$launch_two_pass_marker" \ + "$launch_one_app_report" "$launch_two_app_report" \ + "$launch_one_console_report" "$launch_two_console_report" \ + "$launch_one_validation" "$launch_two_validation" "$persistence_report" \ + "$lifecycle_build_log" "$lifecycle_install_log" \ + "$lifecycle_archive_log" "$lifecycle_archive_validation" \ + "$lifecycle_launch_one_log" "$lifecycle_launch_two_log" \ + "$lifecycle_launch_one_report" "$lifecycle_launch_two_report" \ + "$lifecycle_launch_one_validation" "$lifecycle_launch_two_validation" \ + "$lifecycle_runner_report" "$lifecycle_size_report" "$lifecycle_install_result" \ + "$retained_semantic_product_validation" \ + "$runner_report_path" "$device_inventory" "$device_details" \ + "$device_lock_state" "$installed_apps" \ + "$preflight_report" "$launch_one_result" "$launch_two_result" \ + "$launch_one_copy_result" "$launch_two_copy_result" \ + "$reports_dir/failure.txt"; do + : >"$output_file" + done +else + for output_file in \ + "$generator_log" "$artifact_preparation_log" "$build_log" "$install_log" \ + "$launch_one_log" "$launch_two_log" "$launch_one_copy_log" "$launch_two_copy_log" \ + "$artifact_validation_file" \ + "$embedded_extensions_file" "$host_linkage_file" "$extension_linkage_file" \ + "$embedded_native_file" "$extension_resources_file" "$signing_validation_file" \ + "$extension_host_sdk_symbol_file" \ + "$release_fault_symbol_file" \ + "$launch_one_pass_marker" "$launch_two_pass_marker" \ + "$launch_one_app_report" "$launch_two_app_report" \ + "$launch_one_console_report" "$launch_two_console_report" \ + "$launch_one_validation" "$launch_two_validation" "$persistence_report" \ + "$lifecycle_build_log" "$lifecycle_install_log" \ + "$lifecycle_archive_log" "$lifecycle_archive_validation" \ + "$lifecycle_launch_one_log" "$lifecycle_launch_two_log" \ + "$lifecycle_launch_one_report" "$lifecycle_launch_two_report" \ + "$lifecycle_launch_one_validation" "$lifecycle_launch_two_validation" \ + "$lifecycle_runner_report" "$lifecycle_size_report" "$lifecycle_install_result" \ + "$retained_semantic_product_validation" \ + "$runner_report_path" "$device_inventory" "$device_details" \ + "$device_lock_state" "$installed_apps" "$install_result" \ + "$preflight_report" "$launch_one_result" "$launch_two_result" \ + "$launch_one_copy_result" "$launch_two_copy_result" \ + "$reports_dir/failure.txt"; do + : >"$output_file" + done +fi + +case "$minimum_ios_major" in + ''|*[!0-9]*) fail "OLIPHAUNT_IOS_BROKER_MIN_IOS_MAJOR must be an integer" ;; +esac +[ "$minimum_ios_major" -ge 26 ] || fail "the broker fixture requires iOS 26 or newer" +case "$timeout_seconds" in + ''|*[!0-9]*) fail "OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS must be a positive integer" ;; +esac +[ "$timeout_seconds" -ge 30 ] && [ "$timeout_seconds" -le 600 ] || \ + fail "OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS must be in 30...600" +prepare_artifacts="$(normalize_yes_no "$prepare_artifacts" OLIPHAUNT_IOS_BROKER_PREPARE_ARTIFACTS)" +preflight_only="$(normalize_yes_no "$preflight_only" OLIPHAUNT_IOS_BROKER_DEVICE_PREFLIGHT_ONLY)" +if [ "$resume_lifecycle_only" = "YES" ]; then + [ -n "$lifecycle_run_token_from_environment" ] || \ + fail "--resume-lifecycle requires OLIPHAUNT_IOS_BROKER_LIFECYCLE_RUN_TOKEN" + prepare_artifacts=NO + clean_install=NO +fi +if [ "$resume_after_debug_install" = "YES" ]; then + [ -n "$resume_debug_result_bundle_input" ] || \ + fail "--resume-after-debug-install requires OLIPHAUNT_IOS_BROKER_RESUME_DEBUG_RESULT_BUNDLE" + prepare_artifacts=NO + clean_install=NO +fi +case "$lifecycle_run_token" in + ''|*[!A-Za-z0-9._:-]*) fail "unsafe lifecycle run token" ;; +esac +[ "${#lifecycle_run_token}" -le 256 ] || fail "lifecycle run token exceeds 256 characters" +clean_install="$(normalize_yes_no "$clean_install" OLIPHAUNT_IOS_BROKER_DEVICE_CLEAN_INSTALL)" +uninstall_after_run="$(normalize_yes_no "$uninstall_after_run" OLIPHAUNT_IOS_BROKER_UNINSTALL_AFTER_RUN)" +code_signing_allowed="$(normalize_yes_no "$code_signing_allowed" OLIPHAUNT_IOS_BROKER_CODE_SIGNING_ALLOWED)" +[ "$code_signing_allowed" = "YES" ] || fail "physical iOS install/launch requires code signing" +safe_bundle_identifier "$app_bundle_id" || fail "unsafe host bundle identifier: $app_bundle_id" +safe_bundle_identifier "$extension_bundle_id" || fail "unsafe extension bundle identifier: $extension_bundle_id" +[ "$app_bundle_id" = "dev.oliphaunt.brokerspike" ] || \ + fail "the ExtensionFoundation fixture requires host bundle ID dev.oliphaunt.brokerspike" +[ "$extension_bundle_id" = "dev.oliphaunt.brokerspike.extension" ] || \ + fail "the ExtensionFoundation fixture requires extension bundle ID dev.oliphaunt.brokerspike.extension" +case "$extension_bundle_id" in + "$app_bundle_id".*) ;; + *) fail "extension bundle identifier must be prefixed by the host bundle identifier" ;; +esac +safe_build_name "$scheme" || fail "unsafe Xcode scheme name: $scheme" +safe_build_name "$configuration" || fail "unsafe Xcode configuration name: $configuration" +[ "$configuration" = "Debug" ] || \ + fail "physical broker qualification requires Debug fault-injection coverage" +safe_build_name "$lifecycle_configuration" || \ + fail "unsafe lifecycle Xcode configuration name: $lifecycle_configuration" +[ "$lifecycle_configuration" = "Release" ] || \ + fail "physical lifecycle/memory qualification requires a Release build" +safe_build_name "$app_product_name" || fail "unsafe host product name: $app_product_name" +safe_build_name "$extension_product_name" || fail "unsafe extension product name: $extension_product_name" + +[ "$(uname -s)" = "Darwin" ] || fail "the iOS device runner requires macOS" +for command_name in awk basename codesign cp date defaults dirname ditto find grep kill mktemp \ + nm otool plutil ruby security sed shasum sleep sort tail tee wc xcodebuild xcrun; do + need_cmd "$command_name" +done +[ -f "$generator" ] || fail "missing project generator: $generator" +[ -x "$artifact_preparer" ] || fail "missing executable broker artifact preparer: $artifact_preparer" +if ! ruby -e 'require "xcodeproj"' >"$logs_dir/xcodeproj-preflight.log" 2>&1; then + fail "Ruby xcodeproj is required to generate the fixture" +fi +xcode_major="$(xcodebuild -version | awk 'NR == 1 { split($2, version, "."); print version[1] }')" +case "$xcode_major" in + ''|*[!0-9]*) fail "could not determine the Xcode major version" ;; +esac +[ "$xcode_major" -ge 26 ] || fail "the ExtensionFoundation fixture requires Xcode 26 or newer" + +printf 'Selecting a paired physical iOS %s+ device...\n' "$minimum_ios_major" +select_physical_device +printf 'Selected device: %s (%s, iOS %s, %s)\n' \ + "$selected_device_name" "$selected_device_id" "$selected_device_os" "$selected_device_transport" +preflight_device +configure_broker_device_signing +identity_count="$(valid_code_signing_identity_count)" +write_preflight_report "$identity_count" +if [ "$preflight_only" = "YES" ]; then + printf 'OLIPHAUNT_IOS_BROKER_DEVICE_PREFLIGHT_PASS report=%s\n' "$preflight_report" + exit 0 +fi + +if [ "$prepare_artifacts" = "YES" ]; then + printf 'Preparing arm64 iOS device broker artifacts...\n' + if ! env \ + OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM=device \ + OLIPHAUNT_IOS_BROKER_ARTIFACT_ROOT="$artifact_root" \ + bash "$artifact_preparer" >"$artifact_preparation_log" 2>&1; then + tail -120 "$artifact_preparation_log" >&2 || true + fail "failed to prepare iOS device broker artifacts" + fi + [ -f "$artifact_environment" ] || \ + fail "device artifact preparer did not write its environment file" + # shellcheck disable=SC1090 + . "$artifact_environment" +elif { [ -z "${OLIPHAUNT_IOS_BROKER_XCFRAMEWORK:-}" ] || \ + [ -z "${OLIPHAUNT_IOS_BROKER_RESOURCES:-}" ]; } && [ -f "$artifact_environment" ]; then + # shellcheck disable=SC1090 + . "$artifact_environment" +fi +[ "${OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM:-device}" = "device" ] || \ + fail "configured broker artifacts are not marked for iOS device use" +export OLIPHAUNT_IOS_BROKER_ARTIFACT_PLATFORM=device +validate_broker_artifacts + +if [ "$resume_lifecycle_only" = "YES" ]; then + [ -s "$lifecycle_archive_validation" ] || \ + fail "resume lifecycle requires retained Release archive validation" + lifecycle_archive_path="$(ruby -rjson -e ' + report = JSON.parse(File.read(ARGV.fetch(0))) + abort unless report["schema"] == "oliphaunt-ios-broker-release-archive-validation-v1" + abort unless report["status"] == "PASS" + puts report.fetch("archivePath") + ' "$lifecycle_archive_validation")" + app_path="$(ruby -rjson -e 'puts JSON.parse(File.read(ARGV.fetch(0))).fetch("appPath")' \ + "$lifecycle_archive_validation")" + lifecycle_archive_result_bundle="$(ruby -rjson -e \ + 'puts JSON.parse(File.read(ARGV.fetch(0))).fetch("resultBundle")' \ + "$lifecycle_archive_validation")" + case "$lifecycle_archive_path" in + "$build_root"/archives/*.xcarchive) ;; + *) fail "retained Release archive is outside the device build root" ;; + esac + [ "$app_path" = "$lifecycle_archive_path/Products/Applications/$app_product_name.app" ] || \ + fail "retained Release archive validation names an unexpected app" + [ -d "$app_path" ] || fail "retained signed Release app is missing" + [ -d "$lifecycle_archive_result_bundle" ] || \ + fail "retained Release archive result bundle is missing" + ruby -rjson - "$lifecycle_install_result" "$app_path" "$selected_device_id" \ + "$app_bundle_id" <<'RUBY' +install_path, expected_app, expected_device, expected_bundle = ARGV +report = JSON.parse(File.read(install_path)) +raise "retained Release install did not succeed" unless report.dig("info", "outcome") == "success" +raise "retained Release install used a different archive app" unless report.dig("info", "arguments")&.last == expected_app +raise "retained Release install targeted a different device" unless report.dig("result", "deviceIdentifier") == expected_device +bundles = Array(report.dig("result", "installedApplications")).map { |entry| entry["bundleID"] } +raise "retained Release install omitted the broker app" unless bundles.include?(expected_bundle) +RUBY + xcrun devicectl device info apps \ + --device "$selected_device_id" \ + --bundle-id "$app_bundle_id" \ + --timeout 30 \ + --json-output "$installed_apps" \ + >"$logs_dir/devicectl-installed-apps-resume.log" 2>&1 || \ + fail "failed to verify the retained installed Release app" + ruby -rjson -e ' + apps = JSON.parse(File.read(ARGV.fetch(0))).dig("result", "apps") || [] + matches = apps.select { |app| app["bundleIdentifier"] == ARGV.fetch(1) } + abort unless matches.length == 1 + ' "$installed_apps" "$app_bundle_id" || \ + fail "retained Release broker app is not installed exactly once" + + validating_release_artifact=1 + validate_built_app + write_release_product_sizes || fail "failed to refresh Release archive product sizes" + lifecycle_release_build_app_path="$derived_data/Build/Products/$lifecycle_configuration-iphoneos/$app_product_name.app" + build_result_bundle="$(find "$reports_dir" -mindepth 1 -maxdepth 1 \ + -type d -name 'device-build-*.xcresult' -print | LC_ALL=C sort | tail -1)" + lifecycle_build_result_bundle="$(find "$reports_dir" -mindepth 1 -maxdepth 1 \ + -type d -name 'device-lifecycle-release-build-*.xcresult' -print | \ + LC_ALL=C sort | tail -1)" + [ -d "$build_result_bundle" ] || fail "retained Debug result bundle is missing" + [ -d "$lifecycle_build_result_bundle" ] || \ + fail "retained Release build result bundle is missing" + load_retained_semantic_debug_product "$build_result_bundle" + printf 'Resuming lifecycle qualification from installed audited archive (no build/install)...\n' +else +export OLIPHAUNT_BROKER_INCLUDE_SDK=1 +export OLIPHAUNT_IOS_BROKER_BUNDLE_ID="$app_bundle_id" +export OLIPHAUNT_IOS_BROKER_EXTENSION_BUNDLE_ID="$extension_bundle_id" +export OLIPHAUNT_IOS_BROKER_DEVELOPMENT_TEAM="$development_team" +if [ "$resume_after_debug_install" = "YES" ]; then + project_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_PROJECT_PATH:-$fixture_root/Generated/OliphauntBrokerSpike.xcodeproj}")" + [ -d "$project_path" ] || fail "retained generated Xcode project is missing: $project_path" + build_result_bundle="$(absolute_path "$resume_debug_result_bundle_input")" + case "$build_result_bundle" in + "$reports_dir"/device-build-*.xcresult) ;; + *) fail "retained Debug result bundle is outside the device reports directory" ;; + esac + [ -d "$build_result_bundle" ] || fail "retained Debug result bundle is missing" + grep -Fq '** BUILD SUCCEEDED **' "$build_log" || \ + fail "retained Debug build log does not record BUILD SUCCEEDED" + + app_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_APP_PATH:-$derived_data/Build/Products/$configuration-iphoneos/$app_product_name.app}")" + [ -d "$app_path" ] || fail "retained Debug host app is missing: $app_path" + validate_built_app + semantic_app_path="$app_path" + semantic_extension_path="$extension_path" + + [ -s "$install_result" ] || fail "retained Debug install result is missing" + ruby -rjson - "$install_result" "$app_path" "$selected_device_id" \ + "$app_bundle_id" <<'RUBY' +install_path, expected_app, expected_device, expected_bundle = ARGV +report = JSON.parse(File.read(install_path)) +raise "retained Debug install did not succeed" unless report.dig("info", "outcome") == "success" +raise "retained Debug install used a different built app" unless report.dig("info", "arguments")&.last == expected_app +raise "retained Debug install targeted a different device" unless report.dig("result", "deviceIdentifier") == expected_device +bundles = Array(report.dig("result", "installedApplications")).map { |entry| entry["bundleID"] } +raise "retained Debug install omitted the broker app" unless bundles.include?(expected_bundle) +RUBY + xcrun devicectl device info apps \ + --device "$selected_device_id" \ + --bundle-id "$app_bundle_id" \ + --timeout 30 \ + --json-output "$installed_apps" \ + >"$logs_dir/devicectl-installed-apps-resume-debug.log" 2>&1 || \ + fail "failed to verify the retained installed Debug app" + ruby -rjson - "$installed_apps" "$install_result" "$app_bundle_id" <<'RUBY' +installed_path, install_path, expected_bundle = ARGV +apps = JSON.parse(File.read(installed_path)).dig("result", "apps") || [] +matches = apps.select { |app| app["bundleIdentifier"] == expected_bundle } +raise "retained Debug broker app is not installed exactly once" unless matches.length == 1 +installed_url = matches.first.fetch("url") +recorded_url = JSON.parse(File.read(install_path)).dig( + "result", "installedApplications", 0, "installationURL" +) +raise "installed Debug broker app no longer matches the retained install" unless installed_url == recorded_url +RUBY + installed=1 + printf 'Resuming semantic qualification from retained signed Debug build/install (no rebuild/reinstall)...\n' +else +printf 'Generating signed device Xcode project...\n' +if ! ruby "$generator" >"$generator_log" 2>&1; then + tail -120 "$generator_log" >&2 || true + fail "failed to generate the broker spike Xcode project" +fi +generated_project="$(tail -1 "$generator_log")" +project_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_PROJECT_PATH:-$generated_project}")" +[ -d "$project_path" ] || fail "generated Xcode project is missing: $project_path" + +build_result_bundle="$reports_dir/device-build-$(date -u +%Y%m%dT%H%M%SZ)-$$.xcresult" +xcodebuild_arguments=( + -project "$project_path" + -scheme "$scheme" + -configuration "$configuration" + -sdk iphoneos + -destination "id=$selected_device_udid" + -derivedDataPath "$derived_data" + -resultBundlePath "$build_result_bundle" +) +if is_truthy "$allow_provisioning_updates"; then + xcodebuild_arguments+=( -allowProvisioningUpdates ) +fi +if is_truthy "$allow_device_registration"; then + xcodebuild_arguments+=( -allowProvisioningDeviceRegistration ) +fi +xcodebuild_arguments+=( + CODE_SIGNING_ALLOWED=YES + "DEVELOPMENT_TEAM=$development_team" + "CODE_SIGN_STYLE=$code_sign_style" + COMPILER_INDEX_STORE_ENABLE=NO +) +[ -z "$code_sign_identity" ] || xcodebuild_arguments+=( "CODE_SIGN_IDENTITY=$code_sign_identity" ) +[ -z "$provisioning_profile_specifier" ] || \ + xcodebuild_arguments+=( "PROVISIONING_PROFILE_SPECIFIER=$provisioning_profile_specifier" ) +printf 'Building %s for physical device %s (hardware UDID selected by Xcode)...\n' \ + "$scheme" "$selected_device_name" +if ! xcodebuild "${xcodebuild_arguments[@]}" clean build 2>&1 | tee "$build_log"; then + fail "signed iPhoneOS xcodebuild failed; see $build_log" +fi + +app_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_APP_PATH:-$derived_data/Build/Products/$configuration-iphoneos/$app_product_name.app}")" +[ -d "$app_path" ] || fail "built device host app is missing: $app_path" +validate_built_app +semantic_app_path="$app_path" +semantic_extension_path="$extension_path" + +if [ "$clean_install" = "YES" ]; then + xcrun devicectl device uninstall app \ + --device "$selected_device_id" \ + --timeout 30 \ + "$app_bundle_id" >/dev/null 2>&1 || true +fi +printf 'Installing signed host app on the physical device...\n' +if ! xcrun devicectl device install app \ + --device "$selected_device_id" \ + --timeout 120 \ + --json-output "$install_result" \ + "$app_path" >"$install_log" 2>&1; then + tail -80 "$install_log" >&2 || true + fail "failed to install $app_bundle_id on the selected device" +fi +installed=1 +xcrun devicectl device info apps \ + --device "$selected_device_id" \ + --bundle-id "$app_bundle_id" \ + --timeout 30 \ + --json-output "$installed_apps" \ + >"$logs_dir/devicectl-installed-apps.log" 2>&1 || \ + fail "failed to verify the installed host app" +ruby -rjson -e ' + apps = JSON.parse(File.read(ARGV.fetch(0))).dig("result", "apps") || [] + matches = apps.select { |app| app["bundleIdentifier"] == ARGV.fetch(1) } + abort("expected exactly one installed host app, found #{matches.length}") unless matches.length == 1 +' "$installed_apps" "$app_bundle_id" || fail "devicectl did not report exactly one installed host app" +fi + +run_probe_launch \ + 1 \ + "$launch_one_app_report" \ + "$launch_one_console_report" \ + "$launch_one_validation" \ + "$launch_one_log" \ + "$launch_one_result" \ + "$launch_one_copy_result" \ + "$launch_one_copy_log" \ + "$launch_one_pass_marker" +run_probe_launch \ + 2 \ + "$launch_two_app_report" \ + "$launch_two_console_report" \ + "$launch_two_validation" \ + "$launch_two_log" \ + "$launch_two_result" \ + "$launch_two_copy_result" \ + "$launch_two_copy_log" \ + "$launch_two_pass_marker" +validate_extension_private_persistence || \ + fail "extension-private root identity was not stable across two launches" +retain_semantic_debug_product + +lifecycle_build_result_bundle="$reports_dir/device-lifecycle-release-build-$(date -u +%Y%m%dT%H%M%SZ)-$$.xcresult" +lifecycle_xcodebuild_arguments=( + -project "$project_path" + -scheme "$scheme" + -configuration "$lifecycle_configuration" + -sdk iphoneos + -destination "id=$selected_device_udid" + -derivedDataPath "$derived_data" + -resultBundlePath "$lifecycle_build_result_bundle" +) +if is_truthy "$allow_provisioning_updates"; then + lifecycle_xcodebuild_arguments+=( -allowProvisioningUpdates ) +fi +if is_truthy "$allow_device_registration"; then + lifecycle_xcodebuild_arguments+=( -allowProvisioningDeviceRegistration ) +fi +lifecycle_xcodebuild_arguments+=( + CODE_SIGNING_ALLOWED=YES + "DEVELOPMENT_TEAM=$development_team" + "CODE_SIGN_STYLE=$code_sign_style" + COMPILER_INDEX_STORE_ENABLE=NO +) +[ -z "$code_sign_identity" ] || \ + lifecycle_xcodebuild_arguments+=( "CODE_SIGN_IDENTITY=$code_sign_identity" ) +[ -z "$provisioning_profile_specifier" ] || \ + lifecycle_xcodebuild_arguments+=( "PROVISIONING_PROFILE_SPECIFIER=$provisioning_profile_specifier" ) +printf 'Building signed Release lifecycle fixture for physical device %s...\n' \ + "$selected_device_name" +if ! xcodebuild "${lifecycle_xcodebuild_arguments[@]}" clean build 2>&1 | \ + tee "$lifecycle_build_log"; then + fail "signed Release iPhoneOS lifecycle build failed; see $lifecycle_build_log" +fi +app_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DEVICE_RELEASE_APP_PATH:-$derived_data/Build/Products/$lifecycle_configuration-iphoneos/$app_product_name.app}")" +[ -d "$app_path" ] || fail "built Release lifecycle host app is missing: $app_path" +validating_release_artifact=1 +validate_built_app +lifecycle_release_build_app_path="$app_path" + +mkdir -p "$build_root/archives" +lifecycle_archive_path="$build_root/archives/$app_product_name-$(date -u +%Y%m%dT%H%M%SZ)-$$.xcarchive" +lifecycle_archive_result_bundle="$reports_dir/device-lifecycle-release-archive-$(date -u +%Y%m%dT%H%M%SZ)-$$.xcresult" +lifecycle_archive_arguments=( + -project "$project_path" + -scheme "$scheme" + -configuration "$lifecycle_configuration" + -sdk iphoneos + -destination "generic/platform=iOS" + -derivedDataPath "$derived_data" + -archivePath "$lifecycle_archive_path" + -resultBundlePath "$lifecycle_archive_result_bundle" +) +if is_truthy "$allow_provisioning_updates"; then + lifecycle_archive_arguments+=( -allowProvisioningUpdates ) +fi +if is_truthy "$allow_device_registration"; then + lifecycle_archive_arguments+=( -allowProvisioningDeviceRegistration ) +fi +lifecycle_archive_arguments+=( + CODE_SIGNING_ALLOWED=YES + "DEVELOPMENT_TEAM=$development_team" + "CODE_SIGN_STYLE=$code_sign_style" + COMPILER_INDEX_STORE_ENABLE=NO +) +[ -z "$code_sign_identity" ] || \ + lifecycle_archive_arguments+=( "CODE_SIGN_IDENTITY=$code_sign_identity" ) +[ -z "$provisioning_profile_specifier" ] || \ + lifecycle_archive_arguments+=( "PROVISIONING_PROFILE_SPECIFIER=$provisioning_profile_specifier" ) +printf 'Archiving signed Release lifecycle fixture...\n' +if ! xcodebuild "${lifecycle_archive_arguments[@]}" archive 2>&1 | \ + tee "$lifecycle_archive_log"; then + fail "signed Release iPhoneOS archive failed; see $lifecycle_archive_log" +fi +[ -f "$lifecycle_archive_path/Info.plist" ] || \ + fail "Release archive has no Info.plist" +archive_application_path="$(plutil -extract ApplicationProperties.ApplicationPath raw -o - \ + "$lifecycle_archive_path/Info.plist" 2>/dev/null || true)" +[ "$archive_application_path" = "Applications/$app_product_name.app" ] || \ + fail "Release archive records an unexpected application path: $archive_application_path" +app_path="$lifecycle_archive_path/Products/$archive_application_path" +[ -d "$app_path" ] || fail "Release archive is missing its application product" +validate_built_app +write_release_product_sizes || fail "failed to record Release archive product sizes" +ruby -rjson - "$lifecycle_archive_validation" "$lifecycle_archive_path" \ + "$app_path" "$extension_path" "$embedded_native_framework" \ + "$lifecycle_archive_result_bundle" "$signing_validation_file" \ + "$extension_host_sdk_symbol_file" "$release_fault_symbol_file" <<'RUBY' +output, archive, app, extension, framework, result_bundle, signing, + extension_symbols, fault_symbols = ARGV +File.write(output, JSON.pretty_generate({ + schema: "oliphaunt-ios-broker-release-archive-validation-v1", + status: "PASS", + archivePath: archive, + appPath: app, + extensionPath: extension, + nativeFrameworkPath: framework, + resultBundle: result_bundle, + validations: { + recursiveCodeSigning: signing, + extensionHostSDKSymbols: extension_symbols, + releaseFaultSymbols: fault_symbols, + }, + export: { + status: "not-run", + reason: "App Store/TestFlight export, upload, and review require distribution credentials and external service qualification", + }, +}) + "\n") +RUBY + +# Upgrade in place: do not uninstall between Debug semantic smoke and Release +# lifecycle launches, because the extension-private root must survive. +printf 'Installing signed Release lifecycle fixture without uninstall...\n' +if ! xcrun devicectl device install app \ + --device "$selected_device_id" \ + --timeout 120 \ + --json-output "$lifecycle_install_result" \ + "$app_path" >"$lifecycle_install_log" 2>&1; then + tail -80 "$lifecycle_install_log" >&2 || true + fail "failed to install signed Release lifecycle fixture" +fi +fi + +run_lifecycle_launch 1 NO +run_lifecycle_launch 2 YES +write_lifecycle_runner_report +write_runner_report + +ruby -rjson - "$runner_report_path" "$lifecycle_runner_report" \ + "$semantic_app_path" "$semantic_extension_path" "$configuration" \ + "$app_path" "$extension_path" "$lifecycle_configuration" \ + "$build_result_bundle" "$lifecycle_build_result_bundle" \ + "$extension_host_sdk_symbol_file" "$release_fault_symbol_file" \ + "$retained_semantic_product_validation" <<'RUBY' +runner_path, lifecycle_path, debug_app, debug_extension, debug_configuration, + release_app, release_extension, release_configuration, debug_result_bundle, + release_result_bundle, extension_host_sdk_symbols, release_fault_symbols, + retained_semantic_product = ARGV +runner = JSON.parse(File.read(runner_path)) +runner["schema"] = "oliphaunt-ios-broker-device-run-v2" +runner["build"] = { + "sdk" => "iphoneos", + "architecture" => "arm64", + "semanticDebug" => { + "configuration" => debug_configuration, + "appPath" => debug_app, + "embeddedExtensionPath" => debug_extension, + "resultBundle" => debug_result_bundle, + }, + "lifecycleRelease" => { + "configuration" => release_configuration, + "appPath" => release_app, + "embeddedExtensionPath" => release_extension, + "resultBundle" => release_result_bundle, + }, +} +runner["lifecycleQualification"] = JSON.parse(File.read(lifecycle_path)) +runner["validations"]["extensionHostSDKSymbols"] = extension_host_sdk_symbols +runner["validations"]["releaseFaultSymbols"] = release_fault_symbols +runner["validations"]["semanticDebugRetainedProduct"] = retained_semantic_product +File.write(runner_path, JSON.pretty_generate(runner) + "\n") +RUBY + +printf 'OLIPHAUNT_IOS_BROKER_DEVICE_PASS report=%s launch1=%s launch2=%s persistence=%s lifecycle=%s logs=%s\n' \ + "$runner_report_path" "$launch_one_app_report" "$launch_two_app_report" \ + "$persistence_report" "$lifecycle_runner_report" "$logs_dir" diff --git a/src/sdks/swift/tools/run-ios-broker-full-simulator-matrix.sh b/src/sdks/swift/tools/run-ios-broker-full-simulator-matrix.sh new file mode 100755 index 00000000..744e28ec --- /dev/null +++ b/src/sdks/swift/tools/run-ios-broker-full-simulator-matrix.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="${OLIPHAUNT_REPO_ROOT:-$(cd "$script_dir/../../../.." && pwd)}" +runner="$script_dir/run-ios-broker-simulator.sh" +aggregate_root="$repo_root/target/ios-native-broker-full-matrix" +aggregate_report="$aggregate_root/simulator-matrix.json" +timeout_seconds="${OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS:-240}" +prepare_first="${OLIPHAUNT_IOS_BROKER_PREPARE_ARTIFACTS:-YES}" + +[ -x "$runner" ] || { + printf 'error: simulator runner is not executable: %s\n' "$runner" >&2 + exit 1 +} +mkdir -p "$aggregate_root" + +run_mode() { + local mode="$1" + local build_leaf="$2" + local prepare="$3" + local reset_storage="$4" + env \ + OLIPHAUNT_REPO_ROOT="$repo_root" \ + OLIPHAUNT_BROKER_FIXTURE_MODE="$mode" \ + OLIPHAUNT_IOS_BROKER_BUILD_ROOT="target/$build_leaf" \ + OLIPHAUNT_IOS_BROKER_PREPARE_ARTIFACTS="$prepare" \ + OLIPHAUNT_IOS_BROKER_RESET_SIMULATOR_STORAGE="$reset_storage" \ + OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS="$timeout_seconds" \ + "$runner" +} + +run_mode semantic ios-native-broker-full-matrix/semantic "$prepare_first" YES +run_mode handshakeNegatives ios-native-broker-full-matrix/handshake NO NO +run_mode extendedFaults ios-native-broker-full-matrix/faults NO NO +# A real WorkerCore deadlock may leave an unkillable/reused extension generation. +# Keep it last so it cannot contaminate the recoverable crash matrices. +run_mode hang ios-native-broker-full-matrix/hang NO NO + +ruby -rjson - "$aggregate_report" \ + "$aggregate_root/semantic/reports/runner-report.json" \ + "$aggregate_root/handshake/reports/runner-report.json" \ + "$aggregate_root/faults/reports/runner-report.json" \ + "$aggregate_root/hang/reports/runner-report.json" <<'RUBY' +output, *paths = ARGV +reports = paths.map { |path| [path, JSON.parse(File.read(path))] } +expected_modes = %w[semantic handshakeNegatives extendedFaults hang] +actual_modes = reports.map { |_, report| report["fixtureMode"] } +raise "simulator modes differ: #{actual_modes.inspect}" unless actual_modes == expected_modes +reports.each do |path, report| + raise "simulator lane failed: #{path}" unless report["status"] == "PASS" +end +File.write(output, JSON.pretty_generate({ + schema: "oliphaunt-ios-broker-full-simulator-matrix-v1", + status: "PASS", + completedAt: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + modes: reports.map { |path, report| { + mode: report.fetch("fixtureMode"), + report: path, + appReport: report.fetch("appReport"), + } }, +}) + "\n") +RUBY + +printf 'OLIPHAUNT_IOS_BROKER_FULL_SIMULATOR_MATRIX_PASS report=%s\n' \ + "$aggregate_report" diff --git a/src/sdks/swift/tools/run-ios-broker-simulator.sh b/src/sdks/swift/tools/run-ios-broker-simulator.sh new file mode 100755 index 00000000..e5df8775 --- /dev/null +++ b/src/sdks/swift/tools/run-ios-broker-simulator.sh @@ -0,0 +1,984 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +default_repo_root="$(cd "$script_dir/../../../.." && pwd)" +repo_root="${OLIPHAUNT_REPO_ROOT:-$default_repo_root}" + +absolute_path() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "$repo_root" "$1" ;; + esac +} + +fixture_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_FIXTURE_ROOT:-spikes/ios-native-broker}")" +generator="$(absolute_path "${OLIPHAUNT_IOS_BROKER_PROJECT_GENERATOR:-$fixture_root/generate_project.rb}")" +build_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_BUILD_ROOT:-target/ios-native-broker-spike}")" +derived_data="$(absolute_path "${OLIPHAUNT_IOS_BROKER_DERIVED_DATA:-$build_root/DerivedData}")" +logs_dir="$(absolute_path "${OLIPHAUNT_IOS_BROKER_LOGS_DIR:-$build_root/logs}")" +reports_dir="$(absolute_path "${OLIPHAUNT_IOS_BROKER_REPORTS_DIR:-$build_root/reports}")" +artifact_root="$(absolute_path "${OLIPHAUNT_IOS_BROKER_ARTIFACT_ROOT:-target/ios-native-broker-artifacts}")" +artifact_preparer="$(absolute_path "${OLIPHAUNT_IOS_BROKER_ARTIFACT_PREPARER:-$script_dir/prepare-ios-broker-artifacts.sh}")" +artifact_environment="$(absolute_path "${OLIPHAUNT_IOS_BROKER_ARTIFACT_ENV:-$artifact_root/broker-artifacts.env}")" +storage_quarantine_helper="$(absolute_path "${OLIPHAUNT_IOS_BROKER_STORAGE_QUARANTINE_HELPER:-$script_dir/quarantine-ios-broker-simulator-storage.sh}")" + +scheme="${OLIPHAUNT_IOS_BROKER_SCHEME:-OliphauntBrokerSpike}" +configuration="${OLIPHAUNT_IOS_BROKER_CONFIGURATION:-Debug}" +app_product_name="${OLIPHAUNT_IOS_BROKER_APP_PRODUCT_NAME:-OliphauntBrokerSpike}" +extension_product_name="${OLIPHAUNT_IOS_BROKER_EXTENSION_PRODUCT_NAME:-BrokerAppExtension}" +app_bundle_id="${OLIPHAUNT_IOS_BROKER_BUNDLE_ID:-dev.oliphaunt.brokerspike}" +extension_bundle_id="${OLIPHAUNT_IOS_BROKER_EXTENSION_BUNDLE_ID:-dev.oliphaunt.brokerspike.extension}" +requested_udid="${OLIPHAUNT_IOS_BROKER_SIMULATOR_UDID:-}" +requested_device_name="${OLIPHAUNT_IOS_BROKER_SIMULATOR_NAME:-iPhone 17 Pro}" +requested_runtime="${OLIPHAUNT_IOS_BROKER_SIMULATOR_RUNTIME:-}" +minimum_ios_major="${OLIPHAUNT_IOS_BROKER_MIN_IOS_MAJOR:-26}" +timeout_seconds="${OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS:-120}" +code_signing_allowed="${OLIPHAUNT_IOS_BROKER_CODE_SIGNING_ALLOWED:-YES}" +terminate_after_run="${OLIPHAUNT_IOS_BROKER_TERMINATE_AFTER_RUN:-YES}" +uninstall_after_run="${OLIPHAUNT_IOS_BROKER_UNINSTALL_AFTER_RUN:-NO}" +log_capture_startup_seconds="${OLIPHAUNT_IOS_BROKER_LOG_CAPTURE_STARTUP_SECONDS:-1}" +prepare_artifacts="${OLIPHAUNT_IOS_BROKER_PREPARE_ARTIFACTS:-YES}" +reset_simulator_storage="${OLIPHAUNT_IOS_BROKER_RESET_SIMULATOR_STORAGE:-NO}" +fixture_mode="${OLIPHAUNT_BROKER_FIXTURE_MODE:-semantic}" + +success_marker="OLIPHAUNT_BROKER_SPIKE PASS" +failure_marker="OLIPHAUNT_BROKER_SPIKE FAIL" +app_report_name="broker-spike-report.json" +app_report_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_APP_REPORT_PATH:-$reports_dir/$app_report_name}")" +runner_report_path="$(absolute_path "${OLIPHAUNT_IOS_BROKER_RUNNER_REPORT_PATH:-$reports_dir/runner-report.json}")" + +generator_log="$logs_dir/generate-project.log" +build_log="$logs_dir/xcodebuild.log" +boot_log="$logs_dir/simulator-boot.log" +install_log="$logs_dir/simctl-install.log" +launch_log="$logs_dir/simctl-launch.log" +app_stdout_log="$logs_dir/app-stdout.log" +app_stderr_log="$logs_dir/app-stderr.log" +unified_stream_log="$logs_dir/simulator-unified-stream.log" +unified_snapshot_log="$logs_dir/simulator-unified-snapshot.log" +report_validation_log="$logs_dir/report-validation.log" +artifact_preparation_log="$logs_dir/prepare-broker-artifacts.log" +artifact_validation_file="$reports_dir/broker-artifacts.txt" +embedded_extensions_file="$reports_dir/embedded-extensions.txt" +installed_extensions_file="$reports_dir/installed-extensions.txt" +host_linkage_file="$reports_dir/host-otool.txt" +extension_linkage_file="$reports_dir/extension-otool.txt" +extension_symbols_file="$reports_dir/extension-symbols.txt" +embedded_native_file="$reports_dir/embedded-native-library.txt" +extension_resources_file="$reports_dir/extension-resource-checks.txt" +pass_marker_file="$reports_dir/pass-marker.txt" +simulator_inventory="$reports_dir/simulators.json" +storage_reset_file="$reports_dir/simulator-storage-reset.txt" + +selected_udid="" +selected_name="" +selected_runtime="" +selected_state="" +host_executable="" +extension_executable="" +embedded_native_library="" +log_predicate="" +log_start_time="" +log_stream_pid="" +failure_reason="" + +fail() { + failure_reason="$*" + printf 'error: %s\n' "$failure_reason" >&2 + exit 1 +} + +need_command() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +normalize_yes_no() { + case "$1" in + 1|YES|yes|TRUE|true|ON|on) printf 'YES\n' ;; + 0|NO|no|FALSE|false|OFF|off) printf 'NO\n' ;; + *) fail "$2 must be YES or NO, got: $1" ;; + esac +} + +safe_bundle_identifier() { + case "$1" in + ''|*[!A-Za-z0-9.-]*) return 1 ;; + *) return 0 ;; + esac +} + +safe_process_name() { + case "$1" in + ''|*[!A-Za-z0-9._-]*) return 1 ;; + *) return 0 ;; + esac +} + +safe_build_name() { + case "$1" in + ''|.|..|*/*|*$'\n'*|*$'\r'*) return 1 ;; + *) return 0 ;; + esac +} + +validate_broker_artifacts() { + local xcframework="${OLIPHAUNT_IOS_BROKER_XCFRAMEWORK:-}" + local resources="${OLIPHAUNT_IOS_BROKER_RESOURCES:-}" + [ -n "$xcframework" ] || fail "OLIPHAUNT_IOS_BROKER_XCFRAMEWORK is not set" + [ -n "$resources" ] || fail "OLIPHAUNT_IOS_BROKER_RESOURCES is not set" + xcframework="$(absolute_path "$xcframework")" + resources="$(absolute_path "$resources")" + [ -d "$xcframework" ] || fail "broker XCFramework is missing: $xcframework" + [ -f "$xcframework/Info.plist" ] || fail "broker XCFramework has no Info.plist: $xcframework" + [ -d "$resources/oliphaunt" ] || fail "broker resources do not contain oliphaunt/: $resources" + + local simulator_library_metadata slice_identifier slice_library_path slice_product native_library + simulator_library_metadata="$( + plutil -convert json -o - "$xcframework/Info.plist" | + ruby -rjson -e ' + libraries = JSON.parse(STDIN.read).fetch("AvailableLibraries") + slice = libraries.find do |library| + library["SupportedPlatform"] == "ios" && + library["SupportedPlatformVariant"] == "simulator" && + Array(library["SupportedArchitectures"]).include?("arm64") + end + abort("missing arm64 iOS simulator slice") unless slice + puts [slice.fetch("LibraryIdentifier"), slice.fetch("LibraryPath")].join("\t") + ' + )" || fail "broker XCFramework has no arm64 iOS simulator slice" + IFS=$'\t' read -r slice_identifier slice_library_path </dev/null || true)" + [ -n "$framework_executable" ] || fail "broker framework slice has no executable name: $slice_product" + native_library="$slice_product/$framework_executable" + ;; + *) native_library="$slice_product" ;; + esac + [ -f "$native_library" ] || fail "broker simulator native library is missing: $native_library" + [ "$(xcrun vtool -show-build "$native_library" 2>/dev/null | awk '/platform / { print $2; exit }')" = "IOSSIMULATOR" ] || \ + fail "broker XCFramework selected a non-simulator native library: $native_library" + local native_symbols required_symbol + native_symbols="$(nm -g "$native_library" 2>/dev/null)" + case "$native_symbols" in + *"_liboliphaunt_selected_static_extensions"*) ;; + *) fail "broker simulator library does not contain its static-extension registry" ;; + esac + for required_symbol in _oliphaunt_static_vector_Pg_magic_func _oliphaunt_static_pg_trgm_Pg_magic_func; do + case "$native_symbols" in + *"$required_symbol"*) ;; + *) fail "broker simulator library is missing $required_symbol" ;; + esac + done + + local resource_root="$resources/oliphaunt" + local runtime_manifest="$resource_root/runtime/manifest.properties" + local template_manifest="$resource_root/template-pgdata/manifest.properties" + local static_manifest="$resource_root/static-registry/manifest.properties" + local runtime_files="$resource_root/runtime/files" + local template_files="$resource_root/template-pgdata/files" + local required_file + for required_file in \ + "$runtime_manifest" \ + "$template_manifest" \ + "$static_manifest" \ + "$runtime_files/share/postgresql/postgres.bki" \ + "$runtime_files/share/postgresql/extension/vector.control" \ + "$runtime_files/share/postgresql/extension/pg_trgm.control" \ + "$template_files/PG_VERSION"; do + [ -f "$required_file" ] || fail "broker resources are incomplete: $required_file" + done + grep -Fqx 'selectedExtensions=pg_trgm,vector' "$runtime_manifest" || \ + fail "broker runtime resources do not select exactly vector,pg_trgm" + grep -Fqx 'brokerDatabaseRole=oliphaunt_broker' "$template_manifest" || \ + fail "broker template does not seed the restricted database role" + grep -Fqx 'registeredExtensions=vector,pg_trgm' "$static_manifest" || \ + fail "broker static registry does not register exactly vector,pg_trgm" + for extension in vector pg_trgm; do + find "$runtime_files/share/postgresql/extension" -maxdepth 1 -type f \ + -name "$extension--*.sql" -print -quit | grep -q . || \ + fail "broker runtime resources are missing $extension SQL" + done + + export OLIPHAUNT_IOS_BROKER_XCFRAMEWORK="$xcframework" + export OLIPHAUNT_IOS_BROKER_RESOURCES="$resources" + { + printf 'xcframework=%s\n' "$xcframework" + printf 'simulatorLibrary=%s\n' "$native_library" + printf 'simulatorLibrarySHA256=%s\n' "$(shasum -a 256 "$native_library" | awk '{ print $1 }')" + printf 'resources=%s\n' "$resources" + printf 'runtimeManifest=%s\n' "$runtime_manifest" + printf 'templateManifest=%s\n' "$template_manifest" + printf 'staticRegistryManifest=%s\n' "$static_manifest" + printf 'selectedExtensions=pg_trgm,vector\n' + } >"$artifact_validation_file" +} + +validate_built_artifact_isolation() { + local binary + : >"$host_linkage_file" + for binary in \ + "$app_path/$host_executable" \ + "$app_path/$host_executable.debug.dylib"; do + [ -f "$binary" ] || continue + otool -L "$binary" >>"$host_linkage_file" + done + : >"$extension_linkage_file" + for binary in \ + "$extension_path/$extension_executable" \ + "$extension_path/$extension_executable.debug.dylib"; do + [ -f "$binary" ] || continue + otool -L "$binary" >>"$extension_linkage_file" + done + local native_link_pattern='[/@]liboliphaunt([.]framework/liboliphaunt|[.]dylib)' + if grep -Eq "$native_link_pattern" "$host_linkage_file"; then + fail "broker host unexpectedly links liboliphaunt; see $host_linkage_file" + fi + grep -Eq "$native_link_pattern" "$extension_linkage_file" || \ + fail "broker extension does not link liboliphaunt; see $extension_linkage_file" + + : >"$extension_symbols_file" + for binary in \ + "$extension_path/$extension_executable" \ + "$extension_path/$extension_executable.debug.dylib"; do + [ -f "$binary" ] || continue + { + nm "$binary" 2>/dev/null || true + } | xcrun swift-demangle >>"$extension_symbols_file" + done + local host_adapter_symbol_pattern='OliphauntIOSBroker[.]IOSBroker(Manager|Engine|Session)([ .:$]|$)' + if grep -Eq "$host_adapter_symbol_pattern" "$extension_symbols_file"; then + fail "broker extension contains host-adapter symbols; see $extension_symbols_file" + fi + + local frameworks_dir="$extension_path/Frameworks" + [ -d "$frameworks_dir" ] || fail "broker extension has no embedded Frameworks directory" + find "$frameworks_dir" -type f \( -name liboliphaunt -o -name liboliphaunt.dylib \) \ + -print | LC_ALL=C sort >"$embedded_native_file" + local embedded_native_count + embedded_native_count="$(wc -l <"$embedded_native_file" | tr -d '[:space:]')" + [ "$embedded_native_count" = "1" ] || \ + fail "broker extension must embed exactly one liboliphaunt library, found $embedded_native_count" + embedded_native_library="$(cat "$embedded_native_file")" + + local resource_root="$extension_path/oliphaunt" + local runtime_manifest="$resource_root/runtime/manifest.properties" + local static_manifest="$resource_root/static-registry/manifest.properties" + local runtime_files="$resource_root/runtime/files" + local template_files="$resource_root/template-pgdata/files" + local -a required_resources=( + "runtime/manifest.properties" + "template-pgdata/manifest.properties" + "static-registry/manifest.properties" + "runtime/files/share/postgresql/postgres.bki" + "runtime/files/share/postgresql/extension/vector.control" + "runtime/files/share/postgresql/extension/pg_trgm.control" + "template-pgdata/files/PG_VERSION" + ) + local relative resource_file extension + : >"$extension_resources_file" + for relative in "${required_resources[@]}"; do + resource_file="$resource_root/$relative" + [ -f "$resource_file" ] || fail "embedded broker extension resource is missing: $relative" + printf '%s\t%s\t%s\n' \ + "$relative" \ + "$(wc -c <"$resource_file" | tr -d '[:space:]')" \ + "$(shasum -a 256 "$resource_file" | awk '{ print $1 }')" \ + >>"$extension_resources_file" + done + grep -Fqx 'selectedExtensions=pg_trgm,vector' "$runtime_manifest" || \ + fail "embedded broker runtime manifest lost the exact extension selection" + grep -Fqx 'brokerDatabaseRole=oliphaunt_broker' \ + "$resource_root/template-pgdata/manifest.properties" || \ + fail "embedded broker template lost its restricted database role" + grep -Fqx 'registeredExtensions=vector,pg_trgm' "$static_manifest" || \ + fail "embedded broker static registry lost the exact extension selection" + [ "$(tr -d '\r\n' <"$template_files/PG_VERSION")" = "18" ] || \ + fail "embedded broker template PGDATA is not PostgreSQL 18" + for extension in vector pg_trgm; do + find "$runtime_files/share/postgresql/extension" -maxdepth 1 -type f \ + -name "$extension--*.sql" -print | LC_ALL=C sort >>"$extension_resources_file" + grep -Fq "/$extension--" "$extension_resources_file" || \ + fail "embedded broker runtime is missing $extension SQL" + done + if find "$resource_root" -type f \( -name '*.dylib' -o -name '*.so' \) -print -quit | grep -q .; then + fail "embedded broker resource tree contains a dynamic extension module" + fi +} + +stop_log_capture() { + [ -n "$log_stream_pid" ] || return 0 + if kill -0 "$log_stream_pid" 2>/dev/null; then + kill -TERM "$log_stream_pid" 2>/dev/null || true + local attempts=20 + while [ "$attempts" -gt 0 ] && kill -0 "$log_stream_pid" 2>/dev/null; do + sleep 0.1 + attempts=$((attempts - 1)) + done + kill -KILL "$log_stream_pid" 2>/dev/null || true + fi + wait "$log_stream_pid" 2>/dev/null || true + log_stream_pid="" +} + +capture_unified_snapshot() { + [ -n "$selected_udid" ] || return 0 + [ -n "$log_predicate" ] || return 0 + [ -n "$log_start_time" ] || return 0 + xcrun simctl spawn "$selected_udid" log show \ + --style compact \ + --start "$log_start_time" \ + --predicate "$log_predicate" \ + >"$unified_snapshot_log" 2>&1 || true +} + +capture_screenshot() { + [ -n "$selected_udid" ] || return 0 + xcrun simctl io "$selected_udid" screenshot "$reports_dir/failure.png" \ + >"$logs_dir/screenshot.log" 2>&1 || true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + set +e + stop_log_capture + if [ "$status" -ne 0 ]; then + capture_unified_snapshot + capture_screenshot + if [ -z "$failure_reason" ]; then + failure_reason="runner command failed with status $status" + fi + printf '%s\n' "$failure_reason" >"$reports_dir/failure.txt" + printf '\nLast simulator/app output:\n' >&2 + tail -120 "$app_stdout_log" "$app_stderr_log" "$unified_stream_log" \ + "$unified_snapshot_log" 2>/dev/null >&2 + fi + if [ -n "$selected_udid" ] && [ "$terminate_after_run" = "YES" ]; then + xcrun simctl terminate "$selected_udid" "$app_bundle_id" >/dev/null 2>&1 || true + fi + if [ -n "$selected_udid" ] && [ "$uninstall_after_run" = "YES" ]; then + xcrun simctl uninstall "$selected_udid" "$app_bundle_id" >/dev/null 2>&1 || true + fi + exit "$status" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +mkdir -p \ + "$build_root" \ + "$derived_data" \ + "$logs_dir" \ + "$reports_dir" \ + "$(dirname "$app_report_path")" \ + "$(dirname "$runner_report_path")" +[ "$app_report_path" != "$runner_report_path" ] || \ + fail "app and runner report paths must be different" +: >"$generator_log" +: >"$build_log" +: >"$boot_log" +: >"$install_log" +: >"$launch_log" +: >"$app_stdout_log" +: >"$app_stderr_log" +: >"$unified_stream_log" +: >"$unified_snapshot_log" +: >"$report_validation_log" +: >"$artifact_preparation_log" +: >"$artifact_validation_file" +: >"$embedded_extensions_file" +: >"$installed_extensions_file" +: >"$host_linkage_file" +: >"$extension_linkage_file" +: >"$extension_symbols_file" +: >"$embedded_native_file" +: >"$extension_resources_file" +: >"$pass_marker_file" +: >"$storage_reset_file" +: >"$app_report_path" +: >"$runner_report_path" +: >"$reports_dir/failure.txt" + +case "$minimum_ios_major" in + ''|*[!0-9]*) fail "OLIPHAUNT_IOS_BROKER_MIN_IOS_MAJOR must be an integer" ;; +esac +[ "$minimum_ios_major" -ge 26 ] || fail "the broker fixture requires iOS 26 or newer" +case "$timeout_seconds" in + ''|*[!0-9]*) fail "OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS must be a positive integer" ;; +esac +[ "$timeout_seconds" -gt 0 ] || fail "OLIPHAUNT_IOS_BROKER_TIMEOUT_SECONDS must be positive" +case "$log_capture_startup_seconds" in + ''|*[!0-9]*) fail "OLIPHAUNT_IOS_BROKER_LOG_CAPTURE_STARTUP_SECONDS must be a nonnegative integer" ;; +esac + +code_signing_allowed="$(normalize_yes_no "$code_signing_allowed" OLIPHAUNT_IOS_BROKER_CODE_SIGNING_ALLOWED)" +terminate_after_run="$(normalize_yes_no "$terminate_after_run" OLIPHAUNT_IOS_BROKER_TERMINATE_AFTER_RUN)" +uninstall_after_run="$(normalize_yes_no "$uninstall_after_run" OLIPHAUNT_IOS_BROKER_UNINSTALL_AFTER_RUN)" +prepare_artifacts="$(normalize_yes_no "$prepare_artifacts" OLIPHAUNT_IOS_BROKER_PREPARE_ARTIFACTS)" +reset_simulator_storage="$(normalize_yes_no "$reset_simulator_storage" OLIPHAUNT_IOS_BROKER_RESET_SIMULATOR_STORAGE)" +case "$fixture_mode" in + semantic|extendedFaults|hang|handshakeNegatives) ;; + *) fail "unsupported OLIPHAUNT_BROKER_FIXTURE_MODE: $fixture_mode" ;; +esac +safe_bundle_identifier "$app_bundle_id" || fail "unsafe host bundle identifier: $app_bundle_id" +safe_bundle_identifier "$extension_bundle_id" || fail "unsafe extension bundle identifier: $extension_bundle_id" +safe_build_name "$scheme" || fail "unsafe Xcode scheme name: $scheme" +safe_build_name "$configuration" || fail "unsafe Xcode configuration name: $configuration" +safe_build_name "$app_product_name" || fail "unsafe host product name: $app_product_name" +safe_build_name "$extension_product_name" || fail "unsafe extension product name: $extension_product_name" + +[ "$(uname -s)" = "Darwin" ] || fail "the iOS simulator runner requires macOS" +for command_name in awk bash cp date dirname find grep kill mv nm otool plutil ruby shasum sleep sort tail tee wc xcodebuild xcrun; do + need_command "$command_name" +done +[ -f "$generator" ] || fail "missing project generator: $generator" +[ -f "$storage_quarantine_helper" ] || \ + fail "missing simulator storage quarantine helper: $storage_quarantine_helper" +[ -d "$fixture_root/Host" ] || fail "missing broker host fixture: $fixture_root/Host" +[ -d "$fixture_root/BrokerAppExtension" ] || fail "missing broker extension fixture: $fixture_root/BrokerAppExtension" + +if ! ruby -e 'require "xcodeproj"' >"$logs_dir/xcodeproj-preflight.log" 2>&1; then + fail "Ruby xcodeproj is required; install the repository's xcodeproj dependency before running the fixture" +fi +xcode_major="$(xcodebuild -version | awk 'NR == 1 { split($2, version, "."); print version[1] }')" +case "$xcode_major" in + ''|*[!0-9]*) fail "could not determine the Xcode major version" ;; +esac +[ "$xcode_major" -ge 26 ] || fail "the ExtensionFoundation fixture requires Xcode 26 or newer" + +if [ "$prepare_artifacts" = "YES" ]; then + [ -x "$artifact_preparer" ] || fail "missing executable broker artifact preparer: $artifact_preparer" + printf 'Preparing native broker XCFramework and runtime resources...\n' + if ! env OLIPHAUNT_IOS_BROKER_ARTIFACT_ROOT="$artifact_root" \ + bash "$artifact_preparer" >"$artifact_preparation_log" 2>&1; then + tail -120 "$artifact_preparation_log" >&2 || true + fail "failed to prepare native broker artifacts" + fi + [ -f "$artifact_environment" ] || \ + fail "broker artifact preparer did not write its environment file: $artifact_environment" + # shellcheck disable=SC1090 + . "$artifact_environment" +elif { [ -z "${OLIPHAUNT_IOS_BROKER_XCFRAMEWORK:-}" ] || \ + [ -z "${OLIPHAUNT_IOS_BROKER_RESOURCES:-}" ]; } && [ -f "$artifact_environment" ]; then + printf 'Using previously prepared native broker artifacts from %s...\n' "$artifact_environment" + # shellcheck disable=SC1090 + . "$artifact_environment" +else + printf 'Using explicitly configured native broker artifacts...\n' +fi +validate_broker_artifacts +# Project generation only includes the native SDK targets when this is exactly +# one. Force it here so the simulator run cannot silently degrade to the +# ExtensionFoundation platform-only probe. +export OLIPHAUNT_BROKER_INCLUDE_SDK=1 + +printf 'Selecting %s on iOS %s+...\n' "$requested_device_name" "$minimum_ios_major" +if ! xcrun simctl list devices available -j >"$simulator_inventory"; then + fail "failed to inventory available iOS simulators" +fi + +simulator_selection="" +if ! simulator_selection="$( + ruby -rjson - "$simulator_inventory" "$requested_udid" "$requested_device_name" \ + "$requested_runtime" "$minimum_ios_major" <<'RUBY' +inventory_path, requested_udid, requested_name, requested_runtime, minimum_major = ARGV +inventory = JSON.parse(File.read(inventory_path)) +minimum_major = Integer(minimum_major, 10) +preferred_numbers = requested_runtime.scan(/\d+/).map(&:to_i) + +candidates = [] +inventory.fetch("devices", {}).each do |runtime_identifier, devices| + match = runtime_identifier.match(/iOS-(\d+)-(\d+)/) + next unless match + major = Integer(match[1], 10) + minor = Integer(match[2], 10) + next if major < minimum_major + unless preferred_numbers.empty? + runtime_matches = preferred_numbers.length == 1 ? major == preferred_numbers[0] : [major, minor] == preferred_numbers.first(2) + next unless runtime_matches + end + + devices.each do |device| + next unless device["isAvailable"] != false + next if !requested_udid.empty? && device["udid"] != requested_udid + next if requested_udid.empty? && device["name"] != requested_name + candidates << [major, minor, device] + end +end + +if candidates.empty? + selector = requested_udid.empty? ? "name=#{requested_name.inspect}" : "udid=#{requested_udid}" + runtime = requested_runtime.empty? ? "iOS #{minimum_major}+" : "iOS #{requested_runtime}" + warn "no available simulator matched #{selector}, runtime=#{runtime}" + exit 1 +end + +candidates.sort_by! { |major, minor, device| [-major, -minor, device.fetch("udid")] } +major, minor, device = candidates.first +puts [device.fetch("udid"), device.fetch("name"), "iOS #{major}.#{minor}", device.fetch("state", "unknown")].join("\t") +RUBY +)"; then + fail "failed to select an iPhone 17 Pro simulator running iOS 26 or newer" +fi + +IFS=$'\t' read -r selected_udid selected_name selected_runtime selected_state <"$generator_log" 2>&1; then + tail -120 "$generator_log" >&2 || true + fail "failed to generate the broker spike Xcode project" +fi +generated_project="$(tail -1 "$generator_log")" +project_path="${OLIPHAUNT_IOS_BROKER_PROJECT_PATH:-$generated_project}" +project_path="$(absolute_path "$project_path")" +[ -d "$project_path" ] || fail "generated Xcode project is missing: $project_path" + +build_result_bundle="$reports_dir/build-$(date -u +%Y%m%dT%H%M%SZ)-$$.xcresult" +printf 'Building %s for %s...\n' "$scheme" "$selected_udid" +if ! xcodebuild \ + -project "$project_path" \ + -scheme "$scheme" \ + -configuration "$configuration" \ + -sdk iphonesimulator \ + -destination "id=$selected_udid" \ + -derivedDataPath "$derived_data" \ + -resultBundlePath "$build_result_bundle" \ + CODE_SIGNING_ALLOWED="$code_signing_allowed" \ + COMPILER_INDEX_STORE_ENABLE=NO \ + clean build 2>&1 | tee "$build_log"; then + fail "xcodebuild failed; see $build_log" +fi + +app_path="${OLIPHAUNT_IOS_BROKER_APP_PATH:-$derived_data/Build/Products/$configuration-iphonesimulator/$app_product_name.app}" +app_path="$(absolute_path "$app_path")" +[ -d "$app_path" ] || fail "built host app is missing: $app_path" +[ -f "$app_path/Info.plist" ] || fail "built host app has no Info.plist: $app_path" +observed_app_bundle_id="$(plutil -extract CFBundleIdentifier raw -o - "$app_path/Info.plist" 2>/dev/null || true)" +[ "$observed_app_bundle_id" = "$app_bundle_id" ] || \ + fail "built host bundle identifier is $observed_app_bundle_id, expected $app_bundle_id" +host_executable="$(plutil -extract CFBundleExecutable raw -o - "$app_path/Info.plist" 2>/dev/null || true)" +safe_process_name "$host_executable" || fail "unsafe or missing host executable name: $host_executable" +[ -x "$app_path/$host_executable" ] || fail "host executable is missing: $app_path/$host_executable" + +extensions_dir="$app_path/Extensions" +extension_path="$extensions_dir/$extension_product_name.appex" +legacy_extension_path="$app_path/PlugIns/$extension_product_name.appex" +[ ! -e "$legacy_extension_path" ] || \ + fail "host app contains stale legacy extension packaging: $legacy_extension_path" +[ -d "$extensions_dir" ] || fail "host app is missing its ExtensionKit Extensions directory: $extensions_dir" +find "$extensions_dir" -mindepth 1 -maxdepth 1 -type d -name '*.appex' -print | \ + LC_ALL=C sort >"$embedded_extensions_file" +[ -d "$extension_path" ] || fail "host app is missing the embedded ExtensionKit extension: $extension_path" +observed_extension_bundle_id="$(plutil -extract CFBundleIdentifier raw -o - "$extension_path/Info.plist" 2>/dev/null || true)" +[ "$observed_extension_bundle_id" = "$extension_bundle_id" ] || \ + fail "embedded extension bundle identifier is $observed_extension_bundle_id, expected $extension_bundle_id" +extension_executable="$(plutil -extract CFBundleExecutable raw -o - "$extension_path/Info.plist" 2>/dev/null || true)" +safe_process_name "$extension_executable" || \ + fail "unsafe or missing extension executable name: $extension_executable" +[ -x "$extension_path/$extension_executable" ] || \ + fail "embedded extension executable is missing: $extension_path/$extension_executable" +validate_built_artifact_isolation + +printf 'Booting simulator...\n' +if [ "$selected_state" != "Booted" ]; then + if ! xcrun simctl boot "$selected_udid" >"$boot_log" 2>&1; then + if ! xcrun simctl list devices | grep -F "$selected_udid" | grep -Fq '(Booted)'; then + tail -80 "$boot_log" >&2 || true + fail "failed to boot simulator $selected_udid" + fi + fi +fi +if ! xcrun simctl bootstatus "$selected_udid" -b 2>&1 | tee -a "$boot_log"; then + fail "simulator did not finish booting: $selected_udid" +fi + +if [ "$reset_simulator_storage" = "YES" ]; then + if ! bash "$storage_quarantine_helper" \ + "$selected_udid" \ + "$app_bundle_id" \ + "$extension_bundle_id" \ + "$app_product_name" \ + "$host_executable" \ + "$extension_product_name" \ + "$extension_executable" \ + "$storage_reset_file"; then + fail "simulator storage quarantine was refused; see $storage_reset_file" + fi +else + xcrun simctl terminate "$selected_udid" "$app_bundle_id" >/dev/null 2>&1 || true + xcrun simctl uninstall "$selected_udid" "$app_bundle_id" >/dev/null 2>&1 || true + printf 'disabled\n' >"$storage_reset_file" +fi +printf 'Installing host app...\n' +if ! xcrun simctl install "$selected_udid" "$app_path" >"$install_log" 2>&1; then + tail -80 "$install_log" >&2 || true + fail "failed to install $app_bundle_id" +fi + +installed_app_path="$(xcrun simctl get_app_container "$selected_udid" "$app_bundle_id" app 2>>"$install_log" || true)" +[ -d "$installed_app_path" ] || fail "installed host app container could not be resolved" +installed_extensions_dir="$installed_app_path/Extensions" +installed_extension_path="$installed_extensions_dir/$extension_product_name.appex" +installed_legacy_extension_path="$installed_app_path/PlugIns/$extension_product_name.appex" +[ ! -e "$installed_legacy_extension_path" ] || \ + fail "installed host app contains stale legacy extension packaging: $installed_legacy_extension_path" +[ -d "$installed_extensions_dir" ] || \ + fail "installed host app is missing its ExtensionKit Extensions directory" +find "$installed_extensions_dir" -mindepth 1 -maxdepth 1 -type d -name '*.appex' -print | \ + LC_ALL=C sort >"$installed_extensions_file" +[ -d "$installed_extension_path" ] || \ + fail "installed host app is missing the embedded ExtensionKit extension: $installed_extension_path" +installed_extension_bundle_id="$(plutil -extract CFBundleIdentifier raw -o - "$installed_extension_path/Info.plist" 2>/dev/null || true)" +[ "$installed_extension_bundle_id" = "$extension_bundle_id" ] || \ + fail "installed extension bundle identifier is $installed_extension_bundle_id, expected $extension_bundle_id" +[ -f "$installed_extension_path/oliphaunt/runtime/files/share/postgresql/postgres.bki" ] || \ + fail "installed broker extension lost its PostgreSQL runtime resources" +[ -f "$installed_extension_path/oliphaunt/template-pgdata/files/PG_VERSION" ] || \ + fail "installed broker extension lost its template PGDATA" +[ -f "$installed_extension_path/oliphaunt/static-registry/manifest.properties" ] || \ + fail "installed broker extension lost its static-extension registry" + +data_container="$(xcrun simctl get_app_container "$selected_udid" "$app_bundle_id" data 2>>"$install_log" || true)" +[ -d "$data_container" ] || fail "installed host data container could not be resolved" +source_app_report="$data_container/Documents/$app_report_name" + +log_predicate="process == '$host_executable' OR process == '$extension_executable' OR eventMessage CONTAINS '$success_marker' OR eventMessage CONTAINS '$failure_marker'" +log_start_time="$(date '+%Y-%m-%d %H:%M:%S')" +xcrun simctl spawn "$selected_udid" log stream \ + --style compact \ + --level debug \ + --predicate "$log_predicate" \ + >"$unified_stream_log" 2>&1 & +log_stream_pid=$! +sleep "$log_capture_startup_seconds" +kill -0 "$log_stream_pid" 2>/dev/null || fail "simulator unified-log capture exited before launch" + +printf 'Launching host app...\n' +if ! SIMCTL_CHILD_NSUnbufferedIO=YES \ + SIMCTL_CHILD_OLIPHAUNT_BROKER_FIXTURE_MODE="$fixture_mode" \ + xcrun simctl launch \ + --terminate-running-process \ + --stdout="$app_stdout_log" \ + --stderr="$app_stderr_log" \ + "$selected_udid" "$app_bundle_id" >"$launch_log" 2>&1; then + tail -80 "$launch_log" >&2 || true + fail "failed to launch $app_bundle_id" +fi + +latest_log_line() { + local marker="$1" + # `log stream` echoes its predicate on startup. Require the separator that + # the app emits after a marker so that diagnostic cannot impersonate PASS + # or FAIL merely by containing the quoted predicate string. + grep -hF "$marker " \ + "$app_stdout_log" "$app_stderr_log" "$unified_stream_log" "$unified_snapshot_log" \ + 2>/dev/null | tail -1 || true +} + +validate_app_report() { + ruby -rjson - "$source_app_report" "$report_validation_log" "$fixture_mode" <<'RUBY' +report_path, validation_path, fixture_mode = ARGV +report = JSON.parse(File.read(report_path)) +raise "app report must be a JSON object" unless report.is_a?(Hash) +error = report["error"] +raise "app reported failure: #{error}" unless error.nil? || error.empty? +result = report["result"] +raise "app report is missing result" unless result.is_a?(Hash) + +host_pid = result["hostPID"] +worker_pid = result["workerPID"] +raise "app report has invalid host PID" unless host_pid.is_a?(Integer) && host_pid.positive? +raise "app report has invalid worker PID" unless worker_pid.is_a?(Integer) && worker_pid.positive? +raise "host and extension PIDs are identical" if host_pid == worker_pid +epoch = result["epoch"] +uuid_pattern = /\A[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\z/ +raise "app report has invalid epoch" unless epoch.is_a?(String) && epoch.match?(uuid_pattern) +checks = result["checks"] +required_checks = if fixture_mode == "extendedFaults" + %w[ + archiveBoundaryRejected + differentRootRejected + closeCancelCompletionRace + beforeDispatchCrash + beforeDispatchNotReplayed + afterResponseChunksCrash + partialStreamOutcomeUnknown + checkpointCrashRecovery + idleAbortRecovery + idleSIGSEGVRecovery + ] +elsif fixture_mode == "hang" + %w[ + hangCapabilityConservative + hangTimeout + mainActorResponsiveDuringHang + oldHangEpochInvalidated + replacementLaunchAttempted + ] +elsif fixture_mode == "handshakeNegatives" + %w[ + incompatibleProtocolRejected + incompatibleABIRejected + runtimeMismatchRejected + rootMismatchRejected + startupConfigurationRejected + validHandshakeAfterRejections + secondActiveDataChannelRejected + ] +else + %w[ + extensionDiscovery + separatePID + xpcSession + fdTransfer + fragmentedFrame + boundedRequestAssembly + realSelect + ddl + write + parameterizedQuery + postgresErrorRecovery + vectorExtension + pgTrgmExtension + multiFrameRequest + streamingResponse + simultaneousHandles + fifoSerialization + referenceCounting + transactionHandlePinning + cancellation + postCancelLiveness + checkpointControl + backgroundLifecycle + sameRootReopen + outcomeUnknown + postCommitAmbiguity + crashRecovery + preCommitRollbackRecovery + noAutomaticReplay + capabilities + pgdataPathConfidentiality + workerDiagnostics + openedIdleMemory + ] +end +raise "app report checks must be an array" unless checks.is_a?(Array) +missing = required_checks - checks +raise "app report is missing checks: #{missing.join(",")}" unless missing.empty? +unexpected = checks - required_checks +raise "app report has unexpected checks: #{unexpected.join(",")}" unless unexpected.empty? +if fixture_mode == "semantic" + observations = result["observations"] + raise "semantic matrix omitted observations" unless observations.is_a?(Hash) + raise "semantic matrix used the wrong database role" unless observations["restrictedDatabaseRole"] == "oliphaunt_broker" + raise "semantic matrix assigned the database to the broker role" unless observations["databaseOwner"] == "postgres" + raise "semantic matrix assigned selected extensions to the broker role" unless observations["selectedExtensionOwners"] == "pg_trgm:postgres,vector:postgres" + raise "semantic matrix omitted the broker-owned working schema" unless observations["brokerSchemaOwner"] == "oliphaunt_broker" + %w[ + dataDirectorySQLState parameterizedDataDirectorySQLState serverFileSQLState + bootstrapEscalationSQLState sessionAuthorizationEscalationSQLState + databaseOwnerEscalationSQLState relationPathSQLState tablespacePathSQLState + listDirectorySQLState statFileSQLState largeObjectImportSQLState + externalCopySQLState externalCopyFromSQLState alterSystemSQLState + createRoleSQLState selfSuperuserEscalationSQLState grantFileRoleSQLState + dropSelectedExtensionSQLState + createTablespaceSQLState createNativeFunctionSQLState loadLibrarySQLState + afterResetDataDirectorySQLState afterDiscardDataDirectorySQLState + ].each do |key| + raise "semantic matrix did not deny #{key}" unless observations[key] == "42501" + end + raise "semantic matrix did not preserve the sanitized backend SQLSTATE" unless observations["sanitizedBackendErrorSQLState"] == "F0000" + raise "semantic matrix did not restore the broker search path after DISCARD ALL" unless observations["afterDiscardSearchPath"] == "{oliphaunt_broker,public}" + raise "semantic matrix exposed data_directory through pg_settings" unless observations["pgSettingsDataDirectoryRows"] == "0" + %w[ + restrictedFunctionExecuteCount restrictedViewSelectCount + pgSettingsSourcePathRows visiblePrivatePathSettingRows + ].each do |key| + raise "semantic matrix exposed private catalog/path evidence through #{key}" unless observations[key] == "0" + end + raise "semantic matrix found a non-default tablespace" unless observations["nonDefaultTablespaceCount"] == "0" +end +if fixture_mode == "extendedFaults" + recovered = result["recoveredEpochs"] + raise "extended fault matrix did not publish five recoveries" unless recovered.is_a?(Array) && recovered.length == 5 && recovered.uniq.length == 5 + observations = result["observations"] + raise "extended fault matrix has no partial response evidence" unless observations.is_a?(Hash) && observations.fetch("partialResponseBytesBeforeCrash", "0").to_i.positive? + raise "extended fault matrix did not prove PostgreSQL cancellation and transport completion after native dispatch" unless observations["closeCancelCompletionTerminal"] == "postgresCanceledCompleted" + raise "extended fault matrix hid an unexpected cancel-control failure" unless %w[acknowledged databaseClosed].include?(observations["closeCancelControlOutcome"]) + raise "extended fault matrix changed its root digest" unless observations["initialManifestDigest"] == observations["finalManifestDigest"] +elsif fixture_mode == "hang" + observations = result["observations"] + raise "hang matrix did not preserve conservative capability" unless observations.is_a?(Hash) && observations["hangRestartableCapability"] == "false" + timeout = observations.fetch("timeout", "") + raise "hang matrix omitted its bounded terminal error" if timeout.empty? + raise "hang matrix terminal was not a deadline/interruption/outcome-unknown result" unless timeout.match?(/deadline|interrupt|outcome.*unknown/i) + raise "hang fault was not acknowledged before the trigger" unless observations["faultAcknowledged"] == "true" + raise "worker was not responsive after the fault acknowledgement" unless observations["postAckWorkerResponsive"] == "true" + raise "post-ack worker PID changed" unless Integer(observations.fetch("postAckWorkerPID"), 10) == worker_pid + raise "post-ack epoch changed" unless observations.fetch("postAckEpoch") == epoch + fresh_process = observations["freshProcessObtained"] + raise "hang matrix did not record fresh-process outcome" unless %w[true false].include?(fresh_process) + initial_attempt_count = Integer(observations.fetch("initialLaunchAttemptCount"), 10) + interrupted_attempt_count = Integer(observations.fetch("interruptedLaunchAttemptCount"), 10) + post_attempt_count = Integer(observations.fetch("postRecoveryLaunchAttemptCount"), 10) + attempt_delta = Integer(observations.fetch("replacementLaunchAttemptDelta"), 10) + initial_launch_count = Integer(observations.fetch("initialLaunchCount"), 10) + interrupted_launch_count = Integer(observations.fetch("interruptedLaunchCount"), 10) + post_launch_count = Integer(observations.fetch("postRecoveryLaunchCount"), 10) + successful_launch_delta = Integer(observations.fetch("successfulLaunchCountDelta"), 10) + raise "hang matrix initial attempt count is invalid" unless initial_attempt_count.positive? + raise "hang matrix initial launch count is invalid" unless initial_launch_count.positive? + raise "hang matrix has fewer attempts than successful launches" unless initial_attempt_count >= initial_launch_count + raise "hang interruption regressed process attempts" unless interrupted_attempt_count >= initial_attempt_count + raise "hang interruption regressed launch count" unless interrupted_launch_count >= initial_launch_count + raise "hang matrix did not prove a replacement process attempt" unless post_attempt_count > interrupted_attempt_count + raise "hang replacement attempt delta is inconsistent" unless attempt_delta == post_attempt_count - interrupted_attempt_count + raise "hang successful launch count regressed" unless post_launch_count >= interrupted_launch_count + raise "hang successful launch delta is inconsistent" unless successful_launch_delta == post_launch_count - interrupted_launch_count + recovered_epochs = result["recoveredEpochs"] + raise "hang matrix recovered epochs must be an array" unless recovered_epochs.is_a?(Array) + if fresh_process == "true" + raise "hang fresh process had no successful replacement launch" unless post_launch_count > interrupted_launch_count + recovered_pid = Integer(observations.fetch("recoveredWorkerPID"), 10) + recovered_epoch = observations.fetch("recoveredEpoch") + raise "hang matrix fresh process reused the stale PID" if recovered_pid == worker_pid + raise "hang matrix fresh process reused the stale epoch" if recovered_epoch == epoch + raise "hang matrix fresh process has an invalid epoch" unless recovered_epoch.is_a?(String) && recovered_epoch.match?(uuid_pattern) + raise "hang matrix fresh recovery list is inconsistent" unless recovered_epochs == [recovered_epoch] + else + raise "hang matrix false recovery omitted its failure" if observations.fetch("recoveryFailure", "").empty? + raise "hang matrix false recovery published recovered epochs" unless recovered_epochs.empty? + if observations.key?("recoveredWorkerPID") || observations.key?("recoveredEpoch") + recovered_pid = Integer(observations.fetch("recoveredWorkerPID"), 10) + recovered_epoch = observations.fetch("recoveredEpoch") + both_fresh = recovered_pid != worker_pid && recovered_epoch != epoch + raise "hang matrix mislabeled a fully fresh process as unavailable" if both_fresh + end + end +end + +File.write(validation_path, JSON.pretty_generate({ + status: "PASS", + hostPID: host_pid, + workerPID: worker_pid, + epoch: epoch, + fixtureMode: fixture_mode, + checks: checks, +}) + "\n") +RUBY +} + +deadline=$((SECONDS + timeout_seconds)) +report_valid=0 +pass_line="" +while [ "$SECONDS" -lt "$deadline" ]; do + failure_line="$(latest_log_line "$failure_marker")" + [ -z "$failure_line" ] || fail "broker spike emitted failure marker: $failure_line" + + if [ "$report_valid" -eq 0 ] && [ -s "$source_app_report" ]; then + if ! validate_app_report; then + cp "$source_app_report" "$app_report_path" 2>/dev/null || true + fail "broker spike wrote an invalid or failed app report" + fi + cp "$source_app_report" "$app_report_path" + report_valid=1 + fi + + pass_line="$(latest_log_line "$success_marker")" + if [ "$report_valid" -eq 1 ] && [ -n "$pass_line" ]; then + break + fi + kill -0 "$log_stream_pid" 2>/dev/null || fail "simulator unified-log capture ended before PASS" + sleep 1 +done + +stop_log_capture +capture_unified_snapshot +if [ -z "$pass_line" ]; then + pass_line="$(latest_log_line "$success_marker")" +fi +if [ "$report_valid" -eq 0 ] && [ -s "$source_app_report" ]; then + if validate_app_report; then + cp "$source_app_report" "$app_report_path" + report_valid=1 + fi +fi +[ "$report_valid" -eq 1 ] || fail "timed out waiting for the broker spike app report" +[ -n "$pass_line" ] || fail "timed out waiting for the authoritative $success_marker marker" +printf '%s\n' "$pass_line" >"$pass_marker_file" + +completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +ruby -rjson - "$runner_report_path" "$app_report_path" \ + "$selected_udid" "$selected_name" "$selected_runtime" "$scheme" "$configuration" \ + "$app_bundle_id" "$extension_bundle_id" "$app_path" "$extension_path" \ + "$OLIPHAUNT_IOS_BROKER_XCFRAMEWORK" "$OLIPHAUNT_IOS_BROKER_RESOURCES" \ + "$embedded_native_library" "$artifact_validation_file" "$host_linkage_file" \ + "$extension_linkage_file" "$extension_symbols_file" "$extension_resources_file" \ + "$artifact_preparation_log" \ + "$build_log" "$app_stdout_log" "$app_stderr_log" "$unified_stream_log" \ + "$unified_snapshot_log" "$build_result_bundle" "$pass_line" "$completed_at" \ + "$fixture_mode" "$storage_reset_file" <<'RUBY' +output, app_report_path, udid, device_name, runtime, scheme, configuration, + app_bundle_id, extension_bundle_id, app_path, extension_path, xcframework, + runtime_resources, embedded_native_library, artifact_validation, host_linkage, + extension_linkage, extension_symbols, extension_resources, + artifact_preparation_log, build_log, + stdout_log, stderr_log, stream_log, snapshot_log, result_bundle, + pass_marker, completed_at, fixture_mode, storage_reset_file = ARGV + +payload = { + schema: "oliphaunt-ios-broker-simulator-run-v1", + status: "PASS", + fixtureMode: fixture_mode, + completedAt: completed_at, + simulator: { udid: udid, name: device_name, runtime: runtime }, + build: { + scheme: scheme, + configuration: configuration, + appPath: app_path, + embeddedExtensionPath: extension_path, + resultBundle: result_bundle, + }, + bundleIdentifiers: { host: app_bundle_id, extension: extension_bundle_id }, + artifacts: { + xcframework: xcframework, + runtimeResources: runtime_resources, + embeddedNativeLibrary: embedded_native_library, + }, + validations: { + artifacts: artifact_validation, + hostLinkage: host_linkage, + extensionLinkage: extension_linkage, + extensionSymbols: extension_symbols, + extensionResources: extension_resources, + simulatorStorageReset: storage_reset_file, + }, + logs: { + artifactPreparation: artifact_preparation_log, + xcodebuild: build_log, + appStdout: stdout_log, + appStderr: stderr_log, + unifiedStream: stream_log, + unifiedSnapshot: snapshot_log, + }, + passMarker: pass_marker, + appReport: JSON.parse(File.read(app_report_path)), +} +File.write(output, JSON.pretty_generate(payload) + "\n") +RUBY + +printf 'OLIPHAUNT_IOS_BROKER_SIMULATOR_PASS report=%s appReport=%s logs=%s\n' \ + "$runner_report_path" "$app_report_path" "$logs_dir" diff --git a/tools/release/render_swiftpm_release_package.mjs b/tools/release/render_swiftpm_release_package.mjs index eab4fdd4..a34fbd13 100755 --- a/tools/release/render_swiftpm_release_package.mjs +++ b/tools/release/render_swiftpm_release_package.mjs @@ -436,7 +436,7 @@ async function resolveChecksum(assetDir, assetBaseUrl, asset, version) { return checksum; } -function renderManifest(assetBaseUrl, liboliphauntVersion, checksum) { +export function renderManifest(assetBaseUrl, liboliphauntVersion, checksum) { const asset = `liboliphaunt-${liboliphauntVersion}-apple-spm-xcframework.zip`; const url = `${assetBaseUrl.replace(/\/+$/u, "")}/${asset}`; return `// swift-tools-version: 6.0 @@ -456,6 +456,10 @@ let package = Package( ], products: [ .library(name: "COliphaunt", targets: ["COliphaunt"]), + .library(name: "OliphauntBrokerProtocol", targets: ["OliphauntBrokerProtocol"]), + .library(name: "OliphauntBrokerXPC", targets: ["OliphauntBrokerXPC"]), + .library(name: "OliphauntIOSBroker", targets: ["OliphauntIOSBroker"]), + .library(name: "OliphauntBrokerExtension", targets: ["OliphauntBrokerExtension"]), .library(name: "Oliphaunt", targets: ["Oliphaunt"]), .library(name: "OliphauntExtensionSupport", targets: ["OliphauntExtensionSupport"]), .library(name: "OliphauntICU", targets: ["OliphauntICU"]) @@ -472,11 +476,30 @@ let package = Package( path: "src/sdks/swift/Sources/COliphaunt", publicHeadersPath: "include" ), + .target( + name: "OliphauntBrokerProtocol", + path: "src/sdks/swift/Sources/OliphauntBrokerProtocol" + ), + .target( + name: "OliphauntBrokerXPC", + dependencies: ["OliphauntBrokerProtocol"], + path: "src/sdks/swift/Sources/OliphauntBrokerXPC" + ), .target( name: "Oliphaunt", dependencies: ["COliphaunt"], path: "src/sdks/swift/Sources/Oliphaunt" ), + .target( + name: "OliphauntIOSBroker", + dependencies: ["Oliphaunt", "OliphauntBrokerProtocol", "OliphauntBrokerXPC"], + path: "src/sdks/swift/Sources/OliphauntIOSBroker" + ), + .target( + name: "OliphauntBrokerExtension", + dependencies: ["COliphaunt", "Oliphaunt", "OliphauntBrokerProtocol"], + path: "src/sdks/swift/Sources/OliphauntBrokerExtension" + ), .target( name: "OliphauntExtensionSupport", dependencies: ["COliphaunt", "Oliphaunt"], diff --git a/tools/release/render_swiftpm_release_package.test.mjs b/tools/release/render_swiftpm_release_package.test.mjs index adbe9047..89fd8715 100644 --- a/tools/release/render_swiftpm_release_package.test.mjs +++ b/tools/release/render_swiftpm_release_package.test.mjs @@ -1,6 +1,50 @@ import { describe, expect, test } from "bun:test"; -import { fetchText, missingRequiredAppleArm64Slices } from "./render_swiftpm_release_package.mjs"; +import { + fetchText, + missingRequiredAppleArm64Slices, + renderManifest, +} from "./render_swiftpm_release_package.mjs"; + +describe("SwiftPM release broker package graph", () => { + test("renders every public broker product with the source-package module boundaries", () => { + const manifest = renderManifest( + "https://github.example/releases/download/liboliphaunt-native-v1.2.3", + "1.2.3", + "a".repeat(64), + ); + + for (const product of [ + "OliphauntBrokerProtocol", + "OliphauntBrokerXPC", + "OliphauntIOSBroker", + "OliphauntBrokerExtension", + ]) { + expect(manifest).toContain( + `.library(name: "${product}", targets: ["${product}"])`, + ); + } + expect(manifest).toContain(` .target( + name: "OliphauntBrokerProtocol", + path: "src/sdks/swift/Sources/OliphauntBrokerProtocol" + )`); + expect(manifest).toContain(` .target( + name: "OliphauntBrokerXPC", + dependencies: ["OliphauntBrokerProtocol"], + path: "src/sdks/swift/Sources/OliphauntBrokerXPC" + )`); + expect(manifest).toContain(` .target( + name: "OliphauntIOSBroker", + dependencies: ["Oliphaunt", "OliphauntBrokerProtocol", "OliphauntBrokerXPC"], + path: "src/sdks/swift/Sources/OliphauntIOSBroker" + )`); + expect(manifest).toContain(` .target( + name: "OliphauntBrokerExtension", + dependencies: ["COliphaunt", "Oliphaunt", "OliphauntBrokerProtocol"], + path: "src/sdks/swift/Sources/OliphauntBrokerExtension" + )`); + }); +}); describe("SwiftPM Apple carrier architecture contract", () => { test("accepts the three published arm64 slices", () => {