From 32a7248bfc0e289bba6fa1c5343ca3a783068565 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 20:23:39 +0200 Subject: [PATCH 01/11] doc --- tools/simulator_manager/Sources/README.md | 141 ++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tools/simulator_manager/Sources/README.md diff --git a/tools/simulator_manager/Sources/README.md b/tools/simulator_manager/Sources/README.md new file mode 100644 index 0000000..3d9eda7 --- /dev/null +++ b/tools/simulator_manager/Sources/README.md @@ -0,0 +1,141 @@ +# Mac simulator manager + +A long-running daemon that manages the lifecycle of iOS Simulator devices and +hands them out, over HTTP, as leased resources for tests running on a Mac +worker. It exists so that concurrent test runs share a small pool of +simulators instead of each provisioning and tearing down its own, which is +slow and resource-hungry. + +The daemon speaks HTTP over a Unix domain socket rather than a TCP port, +since it is only ever talked to by processes on the same machine. + +## Running it + +`Main.swift` is a `swift-argument-parser` command. Notable options: + +- `--pid-path` / `--unix-socket-path` — where the daemon writes its PID file + and creates the listening socket. +- `--delete-idle-after` / `--delete-recently-used-idle-after` — how long an + unleased simulator is kept around before deletion, depending on whether its + config was recently leased (see [Idle deletion](#idle-deletion)). +- `--recently-used-capacity` — how many distinct simulator configs count as + "recently used" at once. +- `--startup-process` (repeatable) — extra processes to launch alongside the + daemon; see [Child processes](#child-processes). +- `--post-boot` — a script run against every freshly booted clone (see + [`SimulatorControl.swift`](SimulatorControl.swift)). +- `--lease-path` — where leases are mirrored to disk so a replacement daemon + can adopt them; see [Restarts and lease persistence](#restarts-and-lease-persistence). + +The `--version` flag is not read from source; it is passed in externally +(e.g. by a wrapper script) so the manager doesn't need to be recompiled just +to change how it reports its own version. + +## HTTP API + +Requests are parsed by `SimulatorManagerHTTPHandler.swift` and routed by +`HTTPServer.swift` on the first path component: + +| Method | Path | Query params | Description | +|--------|---------------------|--------------------------------------------|--------------| +| `POST` | `/simulator/` | `exclusive`, `deviceType`, `os`, `version` | Lease a simulator matching the given config to the process ``. Returns the simulator's UDID. | +| `DELETE` | `/simulator/` | | Release the simulator leased to ``. | +| `GET` | `/version` | | The daemon's version string. | +| `GET` | `/leases` | | The number of leases whose leasing process is still alive; for diagnostics. | +| `POST` | `/shutdown` | | Begin a graceful shutdown. | + +`exclusive=1` requests a simulator that no other lease may share; otherwise +the manager may hand out a simulator that is already leased non-exclusively +for the same config. Request parsing and response mapping live in +[`SimulatorRequestHandler.swift`](SimulatorRequestHandler.swift). + +## Architecture + +``` +HTTPServer (SwiftNIO, Unix domain socket) + └─ AccumulatedHTTPHandler buffers HTTP head/body/end into one FullHTTPRequest + └─ SimulatorManagerHTTPHandler parses method/path/query into a SimulatorManagerRequest + └─ SimulatorRequestHandler maps HTTP requests to SimulatorManager calls + └─ SimulatorManager (actor) lease/slot/reference-count bookkeeping + └─ SimulatorControl wraps `xcrun simctl` (create/clone/boot/delete) +``` + +### `SimulatorManager` + +[`SimulatorManager.swift`](SimulatorManager.swift) is the core state machine, +implemented as an actor so its bookkeeping is safe under concurrent leases. +For each `SimulatorConfig` (device type, OS, version) it keeps an array of +slots, each of which is `empty`, `pendingCreation`, `active`, `pendingDeletion`, +or `deleting`. Leasing a config walks the slots in a fixed preference order +(reuse an active non-exclusive simulator, then a pending deletion, then an +empty slot, then a pending creation) and either reuses what's there or clones +a fresh device from a per-config base simulator (created lazily and cached in +`getBaseSimulatorTasks`). + +Every active simulator has a reference count. Leasing increments it; releasing +decrements it, and it is only queued for deletion once it drops to zero, so a +non-exclusive simulator with multiple leasers survives until all of them +release it. + +### Idle deletion + +Once a simulator's reference count hits zero it isn't deleted immediately — +`pendingDeletion` schedules a delayed delete so a device can be reused by the +next lease for the same config. The wait is either `deleteIdleAfter` or the +longer `deleteRecentlyUsedIdleAfter`, chosen by whether that config appears in +`recentlyLeased`, an `LRUSet` (see [`LRUSet.swift`](LRUSet.swift)) capped at +`recentlyUsedCapacity` distinct configs. This keeps simulators for +in-demand configs around longer while letting one-off configs get cleaned up +quickly. + +### Restarts and lease persistence + +A new daemon version replaces the running one (the caller kills the old +process and starts the new one), which would otherwise lose track of leases +held by tests that are still running. `LeaseStore.swift` mirrors every lease +change to a JSON file; on startup, `restoreLeases()` reads it back and rebuilds +enough state (the lease, the slot holding the device, the reference count, and +an exit listener) for `release` to work normally. A lease is only adopted if +its leasing process is still running under the *same* start time (`kinfo_proc` +via `processStartTime`, not just the same PID — PIDs get recycled) and doesn't +conflict with another lease already restored; otherwise it's dropped, and its +device is picked up again automatically the next time something leases that +config. + +A leaser's exit is also watched for directly, via +`DispatchSource.makeProcessSource`, so a lease is released automatically if +its process dies without calling `DELETE /simulator/`. + +### `SimulatorControl` + +[`SimulatorControl.swift`](SimulatorControl.swift) wraps `xcrun simctl` calls +(`create`, `clone`, `bootstatus`, `shutdown`, `delete`, `list devices`) behind +a protocol, so `SimulatorManager` can be tested against a fake. Concurrent +calls for the same base/clone name are coalesced onto a single in-flight +`Task`, and deletion/existence checks for a given name are serialized through +`SimulatorDeleteOrExistenceMutex` so a `clone` can't observe a device that's +mid-deletion. + +### Child processes + +The daemon can launch extra long-lived processes alongside itself +(`--startup-process`). Each one's stdout/stderr is captured through a +`PTY` (see [`PTY.swift`](PTY.swift), needed because plain pipes make some +tools line-buffer differently) and logged line-by-line via `os.Logger`. +These processes are not restarted if they exit. + +## Files + +| File | Purpose | +|------|---------| +| `Main.swift` | CLI entry point; wires flags into a `SimulatorManager` and `HTTPServer`. | +| `HTTPServer.swift` | SwiftNIO server bound to a Unix domain socket; top-level request routing. | +| `AccumulatedHTTPHandler.swift` | Buffers streamed HTTP request parts into one in-memory request/response. | +| `SimulatorManagerHTTPHandler.swift` | Parses the HTTP request into method/path/query; serializes responses. | +| `SimulatorRequestHandler.swift` | Translates `/simulator` requests into `SimulatorManager` lease/release calls. | +| `SimulatorManager.swift` | Core actor: slots, reference counts, leases, idle deletion, child processes. | +| `SimulatorControl.swift` | `simctl`-backed implementation of creating, cloning, booting, and deleting simulators. | +| `LeaseStore.swift` | Persists leases to disk so a replacement daemon can adopt them. | +| `LRUSet.swift` | Fixed-capacity, least-recently-used set used to track recently leased configs. | +| `PTY.swift` | Minimal pseudo-terminal wrapper used to capture child process output. | +| `Logger.swift` | Shared `os.Logger` subsystem/category helper. | From d9636b3da9a7edbb8aabeb6b257ea7da7f003260 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 20:26:59 +0200 Subject: [PATCH 02/11] fix bugs found by claude --- tools/simulator_manager/Sources/LRUSet.swift | 28 ++++++++-------- .../Sources/SimulatorManager.swift | 32 ++++++++++++------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/tools/simulator_manager/Sources/LRUSet.swift b/tools/simulator_manager/Sources/LRUSet.swift index 972c08c..1f903fd 100644 --- a/tools/simulator_manager/Sources/LRUSet.swift +++ b/tools/simulator_manager/Sources/LRUSet.swift @@ -14,28 +14,28 @@ struct LRUSet { self.capacity = capacity } - // Returns an element that was evicted from the set. + // Returns an element that was evicted from the set, or nil if nothing was evicted + // (including when `element` was already present and just moved to most-recently-used). mutating func insert(_ element: Element) -> Element? { - let evicted: Element? if storage.contains(element) { if let index = order.firstIndex(of: element) { - evicted = order.remove(at: index) - } else { - evicted = nil + order.remove(at: index) } order.append(element) - } else { - if order.count >= capacity, let oldest = order.first { - order.removeFirst() - evicted = storage.remove(oldest) - } else { - evicted = nil - } + return nil + } - order.append(element) - storage.insert(element) + let evicted: Element? + if order.count >= capacity, let oldest = order.first { + order.removeFirst() + evicted = storage.remove(oldest) + } else { + evicted = nil } + + order.append(element) + storage.insert(element) return evicted } diff --git a/tools/simulator_manager/Sources/SimulatorManager.swift b/tools/simulator_manager/Sources/SimulatorManager.swift index cff32d1..f6142c6 100644 --- a/tools/simulator_manager/Sources/SimulatorManager.swift +++ b/tools/simulator_manager/Sources/SimulatorManager.swift @@ -51,19 +51,21 @@ private enum SimulatorSlot { extension SimulatorSlot { var sortOrder: Int { switch self { - // Try to use active or pending creation simulators first (should only be - // one of either for non-exclusive) + // Try to use an active simulator first (should only be one for non-exclusive) case .active: return 0 - case .pendingCreation: - return 1 - // Use a pending deletion before any empty slots + // A pending deletion is already created and booted, so reuse it before waiting + // on a pending creation or starting a fresh one. case .pendingDeletion: - return 2 + return 1 - // Finally use empty slots + // Use an empty slot before waiting on a pending creation, so a lease doesn't + // block on someone else's in-flight clone when a fresh slot is free to start. case .empty: + return 2 + + case .pendingCreation: return 3 // Deleting simulator can't be used, so put it at the end @@ -646,12 +648,18 @@ actor SimulatorManager { let processSource = DispatchSource.makeProcessSource(identifier: leaser, eventMask: .exit, queue: .main) + // Avoid double handling of exit in case the process exits between + // `processSource.resume()` and the check with `kill` below. The event handler runs + // on `.main`, while the liveness check below runs on whatever thread calls this + // method, so the flag guarding against a double call must itself be synchronized. + let handledExitLock = NSLock() var handledExit = false let onExitHandler: () -> Void = { [weak self] in - // Avoid double handling of exit in case the process exits between - // `processSource.resume()` and the check with `kill` - guard !handledExit else { return } + handledExitLock.lock() + let alreadyHandled = handledExit handledExit = true + handledExitLock.unlock() + guard !alreadyHandled else { return } Task { guard let self else { return } @@ -665,8 +673,8 @@ actor SimulatorManager { processSource.setEventHandler { onExitHandler() } processSource.resume() - // Check to see if the process is already dead and cancel the source if it is, which will - // trigger `setCancelHandler`, which releases the simulator + // Check to see if the process is already dead. There is no `setCancelHandler`, so + // handle the exit directly here rather than relying on cancellation to do it. guard processIsRunning(leaser) else { processSource.cancel() onExitHandler() From 52a246b83d3f893f66d796416911b531d586be21 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 20:59:57 +0200 Subject: [PATCH 03/11] more --- .../Sources/LEASE_LIFECYCLE.md | 257 ++++++++++++++++++ tools/simulator_manager/Sources/Main.swift | 11 + .../Sources/SimulatorControl.swift | 87 +++++- .../Sources/SimulatorManager.swift | 84 ++++++ 4 files changed, 426 insertions(+), 13 deletions(-) create mode 100644 tools/simulator_manager/Sources/LEASE_LIFECYCLE.md diff --git a/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md b/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md new file mode 100644 index 0000000..8b11869 --- /dev/null +++ b/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md @@ -0,0 +1,257 @@ +# The lifecycle of a simulator lease + +This is a reasoning walkthrough of everything that happens to one lease, from +`POST /simulator/` to the device eventually being deleted (or handed to +someone else). It's meant to make the state machine in `SimulatorManager.swift` +easy to hold in your head, not to restate the architecture already covered in +`README.md`. + +## The actors + +- **A slot** (`SimulatorSlot`) is one device-shaped bucket for a `SimulatorConfig`. + It's `empty`, `pendingCreation`, `active`, `pendingDeletion`, or `deleting`. +- **A lease** (`SimulatorLease` in memory, `PersistedLease` on disk) is the + daemon's record that PID `p` is holding device `udid` in slot `i`. +- **A reference count** (`referenceCount[udid]`) is how many leases currently + point at that device. Non-exclusive leases for the same config can share one + device, so this can be > 1; exclusive leases never share, so it's always + exactly 1 while active. + +A lease and a slot are related but not the same thing: the slot is about the +*device*, the lease is about *who's holding it*. A non-exclusive device can +outlive any single lease on it (reused by the next leaser), and a lease always +maps to exactly one slot at a time. + +## Phase 1 — request arrives + +`SimulatorRequestHandler` parses the HTTP request and calls +`SimulatorManager.lease(to:exclusive:config:)`. First check, before anything +else: does this PID already have a lease? (`leases[leaser]` in +`SimulatorManager.swift:305`). One process, one simulator — this isn't a +counter, it's a hard rule enforced synchronously (no `await` before the +check), so two concurrent requests from the same PID can't both slip through. + +## Phase 2 — is the leaser even still there? + +Before doing anything expensive, the manager checks `processIsRunning(leaser)` +(only when `deleteOnPIDExit` is set — otherwise the caller owns the PID's +lifetime and it need not be a real process at all, e.g. in tests). This looks +redundant — surely the caller wouldn't ask for a lease if it's already dead — +but the point isn't "is it dead right now," it's "is it dead *before we commit +to a multi-minute operation*." Provisioning is the expensive part; checking +liveness is nearly free. This is the cheap half of a two-part guard; the +expensive half is Phase 4. + +## Phase 3 — finding a device: the slot state machine + +This is the core of `getSimulator()`. Slots for the requested config are +sorted by `SimulatorSlot.sortOrder` and walked in order, taking the first one +that matches. The order (after the fix earlier in this session) is: + +1. **`active`, non-exclusive, and the request is non-exclusive** — reuse it + directly. This is the fast path: no provisioning at all, just + `ensureBooted` as a sanity check. +2. **`pendingDeletion`** — a device that's fully created and booted, just + waiting out its idle timer. Reusing it is just as cheap as case 1; the + *only* reason it wasn't in case 1 is that its reference count already hit + zero once. Grabbing it cancels the scheduled deletion task and reassigns + its exclusivity to whatever the new lease wants — which is fine, since + "pending deletion" means nobody currently holds it, so there's no + conflicting owner to worry about. +3. **`empty`** — nobody's even started building a device for this slot index. + Start a fresh clone. +4. **`pendingCreation`, non-exclusive, and the request is non-exclusive** — + someone else is already cloning a device for this config; wait on their + task and count as a second (or third...) leaser of it once it's ready. + +The interesting design choice is putting `empty` *ahead of* `pendingCreation`. +It means a new non-exclusive request would rather kick off a second, +independent clone than wait behind someone else's in-flight one, even though +the in-flight one might finish sooner. My read: this trades a bit of +redundant provisioning work for not making a request's latency depend on +someone else's request. It also means the pool tends to grow to "however many +concurrent first-requests there were for a config" before it starts +consolidating onto reuse — which matches the doc's framing of `pendingCreation` +sharing as a bonus for latecomers, not the primary mechanism. + +Exclusive requests only ever match case 3 (fresh empty slot) or fall through +to appending a brand new slot — they never share `active`, `pendingCreation`, +or (implicitly, since case 2 always wins first if reached) another lease's +slot. `pendingDeletion` is the one case exclusive requests *do* match, and +that's safe precisely because "pending deletion" implies zero current owners. + +If nothing matches, a new slot is appended at the end of the array — the pool +for a config only grows on demand, never pre-allocated. + +## Phase 4 — provisioning, and when the reference count actually increments + +This part is subtle because of the actor-isolation rule spelled out in a +comment above `getSimulator`: slot state must be updated *before* the first +`await`, or a concurrent call could observe a stale slot. That rule shapes +where reference counting happens too — it can't just happen "whenever," it has +to happen at the exact point where the device becomes claimed, before any +suspension point could let the device get swept into deletion by someone else +finishing first. Three different code paths each own this responsibility: + +- **Reuse (`reuseSimulator`)**: increments immediately, *then* awaits + `ensureBooted`. If boot fails with "invalid device" (exit 148), it deletes + the corpse and clones a replacement — the increment from the original + attempt is moot because `delete()` wipes the reference count entry outright, + and the replacement gets its own fresh increment via `createCloneTask`. +- **Fresh clone (`createCloneTask`)**: increments right after the clone + finishes, before returning, on the theory (per the comment in the code) + that whoever created the device should be the one to count it, and anyone + who later shares that same task must increment separately. +- **Joining an in-flight `pendingCreation`**: increments after awaiting the + shared task's result — this is the "anyone who shares it counts separately" + half of the previous point. + +So for a config with 3 non-exclusive leases sharing one device, there are 3 +separate `incrementReferenceCount` calls across possibly all three code paths, +never a batch increment. That's what makes "decrement to zero → pendingDeletion" +in Phase 6 a safe signal: it really does mean the last leaser let go. + +Once a device is in hand, the lease is recorded in `leases[leaser]` and +persisted to disk (`persistLeases()`) *before* the second liveness check. Then +liveness is checked again — the expensive half of the Phase 2 guard. Cloning +and booting can take a real amount of wall-clock time (comment says +"minutes"), long enough for a test harness's timeout to have killed the leaser +already. If that happened, the manager doesn't hand back a UDID nobody will +ever release — it calls `release(for: leaser)` on itself immediately and +reports `leaserExited`. Recording the lease first (even though it's about to +be released) is what lets `release()` work at all; it operates purely on +`leases[leaser]`, it has no other way to find the device. + +Only after surviving both liveness checks does the manager register a process- +exit watcher (`registerReleaseOnExit`) — no point watching a PID you're not +going to hand anything to. + +## Phase 5 — active + +The device sits in `leases[leaser]` and as `.active(udid, exclusive)` in its +slot, reference-counted. Nothing happens to it on its own; it just waits for +one of the release triggers below. Multiple non-exclusive leases can be +pointing at the same UDID simultaneously, each with their own entry in +`leases`, all decrementing the same shared `referenceCount[udid]` independently +whenever *they* release. + +## Phase 6 — release: three ways in + +1. **Explicit `DELETE /simulator/`** → `SimulatorRequestHandler` → + `SimulatorManager.release(for:)`. +2. **The leasing process exits** without ever calling DELETE — caught by the + `DispatchSourceProcess` registered in Phase 4, which calls `release(for:)` + itself. This is what makes leases safe against a build tool that + `kill -9`s a test that forgot to clean up. +3. **The daemon is replaced and the leaser didn't survive the handover** — + handled entirely in `restoreLeases()` at startup, not in `release()` at all; + see Phase 8. The lease is just never adopted, so no explicit release call + ever happens for it, and the device is later picked up implicitly by name. + +`release()` itself is small and ordered deliberately: + +1. Pop the lease out of `leases` (also the point where a nonexistent lease + surfaces as `SimulatorManagerError.noLease` — expected after a daemon + restart dropped a lease the caller doesn't know is gone). +2. Persist the *now-shorter* lease list — before touching the device, so a + daemon replaced mid-release doesn't try to re-adopt a lease whose device is + about to be torn down. +3. Cancel the exit watcher (no longer relevant). +4. Clean temp files on the device. +5. Decrement the reference count, which is where the real branch happens. + +## Phase 7 — idle, and the reference count reaching zero + +`decrementReferenceCount` only does something interesting when the count hits +zero: it hands off to `pendingDeletion()`. If both idle timers are configured +as zero, deletion is immediate — no grace period at all. Otherwise a task is +scheduled that polls once a second, and *at every tick* re-decides which +deadline applies by checking `recentlyLeased.contains(config)` fresh. That's a +live check, not a snapshot taken when the task started — a config that gets +leased again elsewhere while this device is winding down flips it from the +short deadline to the long one (or the reverse, if it falls out of the +LRU set's capacity) mid-wait. It's a genuinely dynamic decision, not "pick a +deadline once." + +Two ways out of `pendingDeletion`: + +- **Resurrection**: a new lease for the same config finds this slot before the + timer expires (Phase 3, case 2) and turns it back into `active`. The + scheduled task gets cancelled, and because `Task.sleep` throws on + cancellation, the task body simply unwinds without ever calling `delete()`. +- **Timeout**: the deadline is reached, the task double-checks the slot still + holds *this exact device* (guarding against a resurrection racing the + cancellation), and calls `delete()` for real — `simctl delete`, slot to + `.deleting` then `.empty`, and (if `cleanUpSlots: true`) trims trailing empty + slots off the end of the array so the pool doesn't grow unboundedly with + dead slot indices. + +## Phase 8 — surviving a daemon restart + +This is the one part of the lifecycle that doesn't originate from an HTTP +request at all — it happens once, at startup, before the server even binds. +`start.sh` replaces the daemon on every new version, which would otherwise +silently orphan every in-flight lease. The fix is that every lease mutation in +Phases 4, 6, and 7 already persists the *entire* current lease set to disk +(atomically), so `restoreLeases()` just has to read that file back and decide, +per lease, whether to trust it: + +1. **Did the leasing process survive?** Not just "is this PID alive" — PIDs + recycle, so it also compares the process's actual start time + (`kinfo_proc`) against what was recorded at lease time. A live PID with a + *different* start time means a different process entirely; treated as + exited. +2. **Does adopting it contradict something already restored?** Two leases + can't share one exclusive device, and one slot can't hold two devices. A + file that claims otherwise is treated as corrupt rather than trusted. + +A lease that passes both checks gets its slot rebuilt as `.active`, its +reference count bumped, and (if configured) a fresh exit watcher — from that +point on it's indistinguishable from a lease that was granted moments ago by +this same daemon. A lease that fails either check is simply dropped — not +released, not cleaned up, just forgotten. Its device isn't touched at all. +`createBase`/`clone` look up devices by name before creating new ones, so an +orphaned device gets rediscovered and correctly reference-counted the next +time something happens to lease that same config — but if nothing ever leases +that exact config again, this lazy reconciliation never fires. See Phase 9 +for the backstop that catches that case. + +## Phase 9 — the orphan reaper + +Every cleanup path above is triggered by an event: a release call, a PID +exit, an idle timer, or "the same config got leased again." Each of those can +miss a device — Phase 8 above is one way (a lease that never gets re-leased), +and a `simctl delete` call inside `delete()` failing silently (it's called as +`try? await delete(...)`, and the slot is reset to `.empty` in a `defer` +regardless of whether the delete actually succeeded) is another. Either way, +nothing else ever looks for that device again. + +`startReaper(interval:)` runs a sweep on a timer, independent of any lease +event, that reconciles against reality instead of the event stream: it lists +every simulator named with the manager's clone prefix +(`SimulatorConfig.cloneDeviceName`, so base simulators and anything not +created by this manager are never touched) and deletes any whose UDID has no +entry in `referenceCount` — the same dictionary key presence check works here +as everywhere else, since an entry exists from the moment a device is claimed +until `delete()` removes it, including through the idle-timer grace period. + +To avoid deleting a clone that's mid-creation (it exists on disk for a moment +before `createCloneTask` resumes and records it in `referenceCount`), a +device must show up as unknown on two consecutive sweeps before it's reaped. +Deletion goes straight through `simulatorControl`, not through `delete()`: +an orphan has no slot pointing at it, so there's nothing in `simulatorSlots` +for `delete()`'s bookkeeping to update. + +## Where this got subtle + +- The "mutate slot before the first `await`" rule is easy to state and easy to + violate by accident; it's the reason reference-count bookkeeping is smeared + across three different functions instead of living in one place. +- `pendingDeletion` slots deliberately don't remember their old exclusivity + flag — and don't need to, because reaching that state already implies zero + current owners. +- The dynamic (not snapshotted) recheck of `recentlyLeased` inside the idle- + deletion poll loop means the *duration* of a device's grace period can + change while it's already ticking. +- Persistence is a side effect of every state change, not a periodic snapshot + or a shutdown hook — which is what makes it safe against `kill -9`. diff --git a/tools/simulator_manager/Sources/Main.swift b/tools/simulator_manager/Sources/Main.swift index e3e5dc5..add0094 100644 --- a/tools/simulator_manager/Sources/Main.swift +++ b/tools/simulator_manager/Sources/Main.swift @@ -52,6 +52,15 @@ struct Main: AsyncParsableCommand { ) var leasePath: String? + @Option( + help: """ + Seconds between sweeps of the orphan reaper, which deletes clone simulators the \ + manager has lost track of (e.g. left behind by a prior daemon's restart, or a \ + failed deletion). 0 disables the reaper. + """ + ) + var reapIntervalSeconds: UInt16 = 300 + func validate() throws { guard recentlyUsedCapacity > 0 else { throw ValidationError( @@ -82,6 +91,8 @@ struct Main: AsyncParsableCommand { // daemon inherited. await simulatorManager.restoreLeases() + await simulatorManager.startReaper(interval: .seconds(Int(reapIntervalSeconds))) + try await simulatorManager.startChildProcesses() try await HTTPServer( diff --git a/tools/simulator_manager/Sources/SimulatorControl.swift b/tools/simulator_manager/Sources/SimulatorControl.swift index bd8288a..29d8897 100644 --- a/tools/simulator_manager/Sources/SimulatorControl.swift +++ b/tools/simulator_manager/Sources/SimulatorControl.swift @@ -105,6 +105,13 @@ protocol SimulatorControl: Actor { runtimeIdentifier: String, context: @escaping @autoclosure () -> String? ) async throws -> String? + + // Every clone-named simulator that currently exists, across all runtimes. + // + // Used by the orphan reaper to reconcile the manager's bookkeeping against reality. + // Deliberately excludes base simulators, which are meant to be long-lived and are + // never reference-counted. + func listManagedClones() async throws -> [SimCtlDevice] } actor RealSimulatorControl: SimulatorControl { @@ -363,6 +370,38 @@ actor RealSimulatorControl: SimulatorControl { } } + func listManagedClones() async throws -> [SimCtlDevice] { + let output = try await simctl(["list", "devices", "-j"], context: "listManagedClones") + + guard let jsonData = output.data(using: .utf8) else { + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to convert output to data"] + ) + } + + let devicesByRuntime: [String: [SimCtlDevice]] + do { + devicesByRuntime = try JSONDecoder().decode(SimCtlDevices.self, from: jsonData).devices + } catch { + let json = String(data: jsonData, encoding: .utf8) ?? "" + Logger.simulatorControl.error( + """ + ❌ Failed to decode 'simctl list devices -j': \(error, privacy: .public). + Output: \(json, privacy: .public) + """ + ) + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to decode output: \(error) - \(json)"] + ) + } + + return devicesByRuntime.values.flatMap { $0 }.filter { $0.name.hasPrefix(managedCloneNamePrefix) } + } + func ensureBooted(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { for retriesLeft in (0...1).reversed() { do { @@ -544,20 +583,37 @@ actor SimulatorDeleteOrExistenceMutex { ) async throws { Logger.simulatorControl.info("🗑️ Deleting simulator \(simulator, privacy: .public)") - do { - _ = try await simctl(["delete", simulator], context: context()) - } catch { - Logger.simulatorControl.error( - """ - ❌ Failed to delete simulator \(simulator, privacy: .public): \ - \(error, privacy: .public) - """ - ) + // `simctl delete` can fail transiently (CoreSimulator daemon busy, a lingering + // child process, disk I/O). Callers treat a thrown error here as "give up until + // the next event," so retry a few times before surfacing failure -- it's much + // cheaper than leaving the simulator running until the orphan reaper's next + // sweep catches it. + let maxAttempts = 3 + for attempt in 1...maxAttempts { + do { + _ = try await simctl(["delete", simulator], context: context()) + Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") + return + } catch { + guard attempt < maxAttempts else { + Logger.simulatorControl.error( + """ + ❌ Failed to delete simulator \(simulator, privacy: .public) after \ + \(maxAttempts, privacy: .public) attempts: \(error, privacy: .public) + """ + ) + throw error + } - throw error + Logger.simulatorControl.warning( + """ + ⚠️ Delete attempt \(attempt, privacy: .public)/\(maxAttempts, privacy: .public) for \ + simulator \(simulator, privacy: .public) failed, retrying: \(error, privacy: .public) + """ + ) + try? await Task.sleep(for: .seconds(2)) + } } - - Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") } } @@ -612,13 +668,18 @@ private func syncSubprocess( } } +/// Prefix shared by every simulator this manager creates for cloning (see +/// `SimulatorConfig.cloneDeviceName`). Used to scope the orphan reaper to devices +/// it manages, so it never touches a developer's own simulators or base templates. +let managedCloneNamePrefix = "EXAMPLE_BAZEL_CLONE_" + extension SimulatorConfig { func baseDeviceName() -> String { return "EXAMPLE_BAZEL_BASE_\(deviceType)_\(version)" } func cloneDeviceName(index: Int) -> String { - return "EXAMPLE_BAZEL_CLONE_\(deviceType)_\(version)_\(index)" + return "\(managedCloneNamePrefix)\(deviceType)_\(version)_\(index)" } func runtimeIdentifier() -> String { diff --git a/tools/simulator_manager/Sources/SimulatorManager.swift b/tools/simulator_manager/Sources/SimulatorManager.swift index f6142c6..d07944c 100644 --- a/tools/simulator_manager/Sources/SimulatorManager.swift +++ b/tools/simulator_manager/Sources/SimulatorManager.swift @@ -106,6 +106,13 @@ actor SimulatorManager { private var childProcessTasks: [Task] = [] private var childProcesses: [String: (Process, DispatchSourceRead, DispatchSourceRead)] = [:] + private var reaperTask: Task? + /// Clone UDIDs that looked orphaned on the *previous* sweep. A device must appear + /// unknown on two consecutive sweeps before the reaper deletes it, so a clone that's + /// mid-creation (it exists on disk before `createCloneTask` resumes and records it in + /// `referenceCount`) never gets caught by a single unlucky sweep. + private var previousOrphanCandidates: Set = [] + init( simulatorControl: SimulatorControl, deleteRecentlyUsedIdleAfter: UInt16, @@ -131,6 +138,8 @@ actor SimulatorManager { } deinit { + reaperTask?.cancel() + for task in childProcessTasks { task.cancel() } @@ -148,6 +157,81 @@ actor SimulatorManager { } } + /// Starts the periodic sweep that deletes clone simulators the manager has lost + /// track of. + /// + /// Every other cleanup path is event-driven: an explicit release, a PID-exit + /// watcher, an idle timer, or "rediscovered by name on the next lease of the same + /// config." Each of those can miss a device -- a daemon restart drops a lease + /// whose device is never leased again (see `restoreLeases`), or a `simctl delete` + /// call fails and is swallowed (see `delete`) -- and nothing else ever looks for + /// it again. This sweep is the backstop: it reconciles against what CoreSimulator + /// actually has running, independent of how a device became untracked. + /// + /// `interval <= .zero` disables it, matching how `deleteIdleAfter` of 0 means + /// "immediately" elsewhere in this file rather than "never." + func startReaper(interval: Duration) { + guard interval > .zero else { return } + + reaperTask = Task { + while !Task.isCancelled { + try? await Task.sleep(for: interval) + guard !Task.isCancelled else { break } + await reapOrphanedSimulators() + } + } + } + + /// One sweep of the orphan reaper: list every clone simulator that exists, and + /// delete the ones with no entry in `referenceCount`. + /// + /// `referenceCount` is the right ground truth to check against, not `leases` or + /// `simulatorSlots`: a device gets an entry in it the instant it's claimed (before + /// any `await`, per the invariant on `getSimulator`), and the entry is removed + /// only in `delete()` -- including while the device sits in the idle-timer grace + /// period, where the count is `0` but the key stays. So "no entry" reliably means + /// "the manager has no idea this exists," not merely "nothing is leasing it right + /// now." + /// + /// Bypasses `delete()` deliberately: that function updates a slot in + /// `simulatorSlots`, but an orphan by definition has no slot pointing at it, so + /// there is nothing there to update. This calls `simulatorControl` directly, the + /// same lower-level operation `delete()` itself wraps. + private func reapOrphanedSimulators() async { + let known = Set(referenceCount.keys) + + let managedClones: [SimCtlDevice] + do { + managedClones = try await simulatorControl.listManagedClones() + } catch { + Logger.simulatorManager.error( + "❌ Orphan reaper failed to list simulators, skipping this sweep: \(error, privacy: .public)" + ) + return + } + + let currentOrphanCandidates = Set(managedClones.map(\.udid)).subtracting(known) + let confirmedOrphans = currentOrphanCandidates.intersection(previousOrphanCandidates) + previousOrphanCandidates = currentOrphanCandidates + + guard !confirmedOrphans.isEmpty else { return } + + for device in managedClones where confirmedOrphans.contains(device.udid) { + Logger.simulatorManager.warning( + """ + 🧹 Reaping orphaned simulator \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)); the manager has no lease or reference to it + """ + ) + + try? await simulatorControl.delete( + device.udid, + name: device.name, + context: "orphan reaper" + ) + } + } + /// Adopts the leases a previous daemon left behind. /// /// Leases used to live only in this actor's memory, so replacing the daemon -- From fdcc048ddadebb939d8b0def33d8afb41fd70877 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 21:06:33 +0200 Subject: [PATCH 04/11] more --- .../Sources/ORPHAN_SIMULATOR_FIX.md | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md diff --git a/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md new file mode 100644 index 0000000..53b3afa --- /dev/null +++ b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md @@ -0,0 +1,140 @@ +# Orphaned simulator fix + +Customers reported resource contention caused by `launchd_sim` processes that +had been running for days, tied to CoreSimulator devices the daemon no longer +knew about: + +```text +PID PPID RSS ELAPSED COMMAND +78247 1 9728 05-23:05:41 launchd_sim .../Devices/B9A13CD5-.../data/var/run/launchd_bootstrap.plist +78293 1 11600 05-23:05:40 launchd_sim .../Devices/88362C6D-.../... +78337 1 9968 05-23:05:40 launchd_sim .../Devices/B8CD72C0-.../... +``` + +This document explains the two root causes and the fix. See +[`LEASE_LIFECYCLE.md`](LEASE_LIFECYCLE.md) for the full lease state machine +this fix adds to (Phase 9). + +## Root causes + +Every existing cleanup path in `SimulatorManager.swift` is event-driven: an +explicit `DELETE /simulator/`, a PID-exit watcher, an idle timer, or a +device getting rediscovered by name the next time its config is leased +again. There was no path that checked the manager's bookkeeping against what +CoreSimulator actually had running. Two specific gaps let devices escape all +of those events at once. + +### 1. Daemon restarts drop leases without touching their devices + +`start.sh` replaces the daemon on every version change. On startup, +`restoreLeases()` (`SimulatorManager.swift:166`) re-adopts a persisted lease +only if its PID is still alive *and* its recorded process start time matches +(to rule out PID reuse). A lease that fails either check is dropped — +intentionally, per the existing code comment: "not released, not cleaned up, +just forgotten." The device itself is never shut down or deleted. + +The intended recovery path was that `createBase`/`clone` look up devices by +name, so an orphaned device would get picked up and reference-counted again +the next time something leased that same `SimulatorConfig`. If that exact +config was never requested again, nothing ever looked for the device again — +it, and its `launchd_sim`, ran forever. + +### 2. Failed `simctl delete` calls were swallowed silently + +In `delete()` (`SimulatorManager.swift:758-792`), the slot was reset to +`.empty` and the reference-count entry removed in a `defer` — regardless of +whether the `simctl delete` call inside it actually succeeded. Both call +sites invoked this as `try? await delete(...)`, discarding any error with no +retry. If `simctl delete` failed for any reason (CoreSimulator daemon busy, a +lingering child process, disk I/O), the manager's bookkeeping said "deleted" +while the real device kept running — and since its slot was now empty, the +manager would never look at that UDID again. + +Both gaps produce the same outcome: a real, running simulator with zero +corresponding state in the manager. + +## The fix + +### Retry `simctl delete` before giving up + +`SimulatorControl.swift`, `SimulatorDeleteOrExistenceMutex.unlockedDelete` +now retries the underlying `simctl delete` up to 3 times, 2 seconds apart, +before rethrowing. This doesn't touch `SimulatorManager`'s actor-isolated +slot state machine, so it carries no risk to the "mutate slot before the +first `await`" invariant documented above `getSimulator()`. It simply makes +the existing call sites succeed more often instead of falling straight +through to `try?`. + +### Periodic orphan reaper + +The real fix: a sweep that reconciles the manager's state against reality, +independent of any lease event, so it catches a device regardless of *how* +it became untracked. + +- **`SimulatorControl.listManagedClones()`** (new protocol method) lists + every simulator whose name starts with `EXAMPLE_BAZEL_CLONE_` — the prefix + this manager already uses for every clone it creates + (`SimulatorConfig.cloneDeviceName`), now hoisted into the shared constant + `managedCloneNamePrefix`. Base simulators (`EXAMPLE_BAZEL_BASE_...`) are + excluded — they're intentionally long-lived templates and are never + reference-counted, so they must never be touched by the reaper. Anything + without this prefix (a developer's own simulator, Xcode's own devices) is + untouched. +- **`SimulatorManager.reapOrphanedSimulators()`** (new, private) computes + `known = Set(referenceCount.keys)` and treats any listed clone UDID not in + that set as an orphan candidate. `referenceCount` is the right thing to + check: a device gets an entry the instant it's claimed, before any + `await`, and the entry is removed only in `delete()` — including while the + device sits in its idle-timer grace period, where the count is `0` but the + key stays. So "no entry" reliably means "the manager has no idea this + exists." +- **Two-sweep confirmation.** A clone that's mid-creation exists on disk (via + `simctl clone`) for a moment before `createCloneTask` resumes and records + it in `referenceCount`. To avoid mistaking that window for an orphan, a + device must show up as unknown on two consecutive sweeps + (`previousOrphanCandidates` intersected with the current sweep's + candidates) before it's deleted. +- **Deletes bypass `delete()`.** An orphan has no slot pointing at it in + `simulatorSlots`, so there's nothing for `delete()`'s slot bookkeeping to + update. The reaper calls `simulatorControl.delete()` directly instead — the + same lower-level operation `delete()` itself wraps. +- **`SimulatorManager.startReaper(interval:)`** (new, public) runs the sweep + on a loop and is started from `Main.swift` right after `restoreLeases()`. + The interval is configurable via the new `--reap-interval-seconds` flag + (default 300; `0` disables the reaper), and the task is cancelled in + `deinit` alongside the other background tasks. + +## Files changed + +| File | Change | +|:-----|:-------| +| `SimulatorControl.swift` | Retry logic in `unlockedDelete`; new `listManagedClones()`; hoisted `managedCloneNamePrefix` constant | +| `SimulatorManager.swift` | New `startReaper(interval:)` and `reapOrphanedSimulators()`; new `reaperTask` and `previousOrphanCandidates` state | +| `Main.swift` | New `--reap-interval-seconds` flag; starts the reaper after `restoreLeases()` | +| `LEASE_LIFECYCLE.md` | Corrected Phase 8's claim that nothing hunts down orphans; added Phase 9 describing the reaper | + +## What this does not fix + +- The slot state machine itself (`SimulatorSlot` cases, `getSimulator`, the + "mutate before first `await`" rule) is unchanged. The reaper works + alongside it by going straight to `simulatorControl`, so the delicate + actor-isolation invariant didn't need to be touched. +- The Go port at `experiments/yannic/macsimulatormanager/go` was out of + scope for this fix — the bug report and investigation were both scoped to + the Swift implementation. + +## Verification status + +There is no `BUILD` file or `Package.swift` for this target yet, so it isn't +wired into Bazel and can't be built with `swift build` either. This change +has been verified by code review against the existing invariants (see +`LEASE_LIFECYCLE.md`), not by compiling or running it. Before relying on +this in production: + +1. Set up a build target (Bazel or an ad hoc `Package.swift`) so this code + compiles and can be typechecked — it currently depends on `ShellOut`, + `ArgumentParser`, and SwiftNIO, none of which are vendored here. +2. Manually exercise the restart-orphan path: lease a config, `kill -9` the + daemon, start a new instance with a short `--reap-interval-seconds`, and + confirm the orphaned clone disappears from `xcrun simctl list devices` + within two sweep intervals without touching unrelated simulators. From 3c218f85f9e8105bf0386bd24e94dc4d36350bbd Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 21:13:25 +0200 Subject: [PATCH 05/11] more --- .../Sources/LEASE_LIFECYCLE.md | 22 +++++ .../Sources/ORPHAN_SIMULATOR_FIX.md | 65 ++++++++++++- .../Sources/SimulatorControl.swift | 93 +++++++++++++++---- .../Sources/SimulatorManager.swift | 81 ++++++++++++++-- 4 files changed, 231 insertions(+), 30 deletions(-) diff --git a/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md b/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md index 8b11869..4747b3a 100644 --- a/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md +++ b/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md @@ -226,6 +226,14 @@ and a `simctl delete` call inside `delete()` failing silently (it's called as regardless of whether the delete actually succeeded) is another. Either way, nothing else ever looks for that device again. +`delete()` failing isn't necessarily transient, either: `simctl delete` isn't +reliable against a still-booted device, and nothing in the original lifecycle +ever shuts a clone down before deleting it (a clone goes straight from +"active" to "delete this" — there's no shutdown step in between). A device +that's genuinely wedged (unresponsive, the same profile as a `launchd_sim` +that's been running for days) will fail both `shutdown` and `delete` for the +same reason every time, not just once. + `startReaper(interval:)` runs a sweep on a timer, independent of any lease event, that reconciles against reality instead of the event stream: it lists every simulator named with the manager's clone prefix @@ -242,6 +250,20 @@ Deletion goes straight through `simulatorControl`, not through `delete()`: an orphan has no slot pointing at it, so there's nothing in `simulatorSlots` for `delete()`'s bookkeeping to update. +`SimulatorControl.delete()` itself now shuts the device down before each of +its own delete attempts (see `shutdownSimulator` in `SimulatorControl.swift`), +so most orphans clean up on the reaper's first attempt. For the wedged case +where that still isn't enough, the reaper tracks how many sweeps in a row a +given UDID has failed to delete (`orphanDeleteFailureCounts`). After three +consecutive failures, `reapOrphan` treats the device as stuck rather than +unlucky and calls `SimulatorControl.forceKillLaunchdSim(for:)`, which finds +the device's `launchd_sim` process by `pgrep -f ` (the process's own +command line embeds its data path, so the UDID is a safe, specific pattern), +double-checks the process name before signaling it, and sends it `SIGKILL` +directly — bypassing `simctl` entirely. One more delete attempt follows; if +that still fails, the device is logged loudly as needing manual cleanup +rather than retried silently forever. + ## Where this got subtle - The "mutate slot before the first `await`" rule is easy to state and easy to diff --git a/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md index 53b3afa..45d9398 100644 --- a/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md +++ b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md @@ -104,14 +104,61 @@ it became untracked. (default 300; `0` disables the reaper), and the task is cancelled in `deinit` alongside the other background tasks. +## Follow-up: the report didn't go away + +After the fix above, the same customer report came back. Investigating that +turned up two separate things: + +**The fix likely never shipped.** The commit was still local to this branch +(`git status` showed it 1 commit ahead of `origin/yannic-simulator-manager`, +never pushed), this package has no `BUILD` file or `Package.swift` — nothing +wires it into a build or deploy pipeline — and this branch's history includes +an earlier commit titled `DO NOT MERGE: Simulator manager`. Nothing here +indicates a build customers actually run has changed at all. + +**Independent of that, a real gap in the reaper itself.** Both the original +`delete()` and the new reaper's delete path called `simctl delete` without +ever calling `shutdown` first, and both discarded failures with `try?` and no +escalation. That's survivable for a healthy device, but a device that's +genuinely wedged — exactly the profile of a `launchd_sim` that's been running +for days — will fail `shutdown` and `delete` for the same underlying reason +every time. The reaper would correctly identify such a device as orphaned +after two sweeps, then retry a delete that fails identically every 5 minutes, +forever, silently. Retrying a deterministically broken operation isn't a fix. +See the next section for what addresses this. + +## Additional fix: shut down before delete, escalate when delete keeps failing + +- **`shutdownSimulator`** (new private free function in `SimulatorControl.swift`) + factors out the shutdown-with-"already shut down"-handling logic that + previously only backed the public `shutdown()` method. `unlockedDelete` now + calls it before each of its (still up to 3) delete attempts, best-effort + (`try?` — a shutdown failure for any other reason shouldn't block trying + delete anyway, since delete is what actually matters). +- **`SimulatorControl.forceKillLaunchdSim(for:)`** (new protocol method) is the + last resort for a device that keeps failing to delete. It finds the + device's `launchd_sim` process with `pgrep -f ` (the process's own + command line embeds its data path, so the UDID is a safe, specific + pattern), double-checks the process name before signaling it (`ps -p + -o comm=`, must end in `launchd_sim`), and sends `SIGKILL` directly, + bypassing `simctl` entirely. +- **`SimulatorManager` tracks consecutive reap failures per UDID** + (`orphanDeleteFailureCounts`). `reapOrphan` (new, factored out of + `reapOrphanedSimulators`) deletes normally on each sweep; after 3 + consecutive failures for the same UDID, it calls `forceKillLaunchdSim` and + makes one more delete attempt. If that still fails, it logs loudly ("needs + manual cleanup") instead of retrying the same failing call forever. The + failure count resets whenever a delete succeeds or the UDID stops being an + orphan at all. + ## Files changed | File | Change | |:-----|:-------| -| `SimulatorControl.swift` | Retry logic in `unlockedDelete`; new `listManagedClones()`; hoisted `managedCloneNamePrefix` constant | -| `SimulatorManager.swift` | New `startReaper(interval:)` and `reapOrphanedSimulators()`; new `reaperTask` and `previousOrphanCandidates` state | +| `SimulatorControl.swift` | Retry + shutdown-first logic in `unlockedDelete`; new `listManagedClones()` and `forceKillLaunchdSim(for:)`; hoisted `managedCloneNamePrefix` constant; factored-out `shutdownSimulator` | +| `SimulatorManager.swift` | New `startReaper(interval:)`, `reapOrphanedSimulators()`, and `reapOrphan(_:)`; new `reaperTask`, `previousOrphanCandidates`, and `orphanDeleteFailureCounts` state | | `Main.swift` | New `--reap-interval-seconds` flag; starts the reaper after `restoreLeases()` | -| `LEASE_LIFECYCLE.md` | Corrected Phase 8's claim that nothing hunts down orphans; added Phase 9 describing the reaper | +| `LEASE_LIFECYCLE.md` | Corrected Phase 8's claim that nothing hunts down orphans; added Phase 9 describing the reaper and its escalation path | ## What this does not fix @@ -131,10 +178,18 @@ has been verified by code review against the existing invariants (see `LEASE_LIFECYCLE.md`), not by compiling or running it. Before relying on this in production: -1. Set up a build target (Bazel or an ad hoc `Package.swift`) so this code +1. Confirm this branch is actually the source for whatever customers run — + given the missing build target and the `DO NOT MERGE` history noted + above, that's not yet established. If it is, push this branch and get it + merged; if there's a separate deploy path, find it. +2. Set up a build target (Bazel or an ad hoc `Package.swift`) so this code compiles and can be typechecked — it currently depends on `ShellOut`, `ArgumentParser`, and SwiftNIO, none of which are vendored here. -2. Manually exercise the restart-orphan path: lease a config, `kill -9` the +3. Manually exercise the restart-orphan path: lease a config, `kill -9` the daemon, start a new instance with a short `--reap-interval-seconds`, and confirm the orphaned clone disappears from `xcrun simctl list devices` within two sweep intervals without touching unrelated simulators. +4. Manually exercise the wedged-device escalation path: lease a config, then + independently hang or otherwise make its clone's `simctl shutdown`/`delete` + fail out-of-band, and confirm the reaper force-kills its `launchd_sim` + after 3 failed sweeps rather than retrying forever. diff --git a/tools/simulator_manager/Sources/SimulatorControl.swift b/tools/simulator_manager/Sources/SimulatorControl.swift index 29d8897..b4b46c1 100644 --- a/tools/simulator_manager/Sources/SimulatorControl.swift +++ b/tools/simulator_manager/Sources/SimulatorControl.swift @@ -112,6 +112,13 @@ protocol SimulatorControl: Actor { // Deliberately excludes base simulators, which are meant to be long-lived and are // never reference-counted. func listManagedClones() async throws -> [SimCtlDevice] + + // Kills the `launchd_sim` process for `simulator` directly, bypassing `simctl` + // entirely. Best-effort and never throws: this is the last resort for a device + // that keeps failing `delete()` through normal means (a wedged simulator can fail + // both `shutdown` and `delete` indefinitely), used by the orphan reaper's + // escalation path. + func forceKillLaunchdSim(for simulator: SimulatorUDID) async } actor RealSimulatorControl: SimulatorControl { @@ -310,20 +317,7 @@ actor RealSimulatorControl: SimulatorControl { } func shutdown(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { - do { - _ = try await simctl(["shutdown", simulator], context: context()) - } catch let error as ProcessError { - // Exit code 149 is related to the simulator already being shut down - guard error.exitCode == 149 else { - throw error - } - - Logger.simulatorControl.warning( - """ - ⚠️ Shutdown failed, but probably \"already shut down\": \(error, privacy: .public) - """ - ) - } + try await shutdownSimulator(simulator, context: context()) } func cleanTempFiles(in simulator: SimulatorUDID) { @@ -402,6 +396,40 @@ actor RealSimulatorControl: SimulatorControl { return devicesByRuntime.values.flatMap { $0 }.filter { $0.name.hasPrefix(managedCloneNamePrefix) } } + func forceKillLaunchdSim(for simulator: SimulatorUDID) async { + // `launchd_sim`'s command line embeds the device's own data path (see the ps + // output that motivated this: ".../Devices//data/var/run/..."), so the + // UDID is a safe, specific `pgrep -f` pattern -- it can only match that one + // device's process tree. + let pids: [Int32] + do { + let output = try await subprocess("/usr/bin/pgrep", ["-f", simulator]) + pids = output.split(separator: "\n").compactMap { Int32($0.trimmingCharacters(in: .whitespaces)) } + } catch { + // `pgrep` exits non-zero (surfaced here as a thrown `ProcessError`) when nothing + // matches -- there's nothing left to kill. + return + } + + for pid in pids { + // Double check this is actually `launchd_sim` before signaling it. `pgrep -f` + // matches the UDID anywhere in the command line; being wrong here would kill an + // unrelated process that merely mentioned this device (e.g. in a log path). + guard let comm = try? await subprocess("/bin/ps", ["-p", "\(pid)", "-o", "comm="]), + comm.trimmingCharacters(in: .whitespacesAndNewlines).hasSuffix("launchd_sim") else { + continue + } + + Logger.simulatorControl.warning( + """ + 🔨 Force-killing launchd_sim (pid \(pid, privacy: .public)) for stuck simulator \ + \(simulator, privacy: .public) + """ + ) + kill(pid, SIGKILL) + } + } + func ensureBooted(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { for retriesLeft in (0...1).reversed() { do { @@ -584,12 +612,19 @@ actor SimulatorDeleteOrExistenceMutex { Logger.simulatorControl.info("🗑️ Deleting simulator \(simulator, privacy: .public)") // `simctl delete` can fail transiently (CoreSimulator daemon busy, a lingering - // child process, disk I/O). Callers treat a thrown error here as "give up until - // the next event," so retry a few times before surfacing failure -- it's much - // cheaper than leaving the simulator running until the orphan reaper's next - // sweep catches it. + // child process, disk I/O) or because the device is still booted -- `delete` + // does not reliably shut a booted device down on its own. Callers treat a thrown + // error here as "give up until the next event," so shut down and retry a few + // times before surfacing failure -- it's much cheaper than leaving the simulator + // running until the orphan reaper's next sweep catches it. let maxAttempts = 3 for attempt in 1...maxAttempts { + // Best-effort: proceed to the delete attempt regardless of whether this + // succeeds. A shutdown failure for a reason other than "already shut down" + // (already handled inside `shutdownSimulator`) shouldn't block trying delete + // anyway, since delete is the operation that actually matters here. + try? await shutdownSimulator(simulator, context: context()) + do { _ = try await simctl(["delete", simulator], context: context()) Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") @@ -617,6 +652,28 @@ actor SimulatorDeleteOrExistenceMutex { } } +/// Shared by `RealSimulatorControl.shutdown` and `unlockedDelete`, since delete needs +/// to shut the device down first and shouldn't reimplement this. +private func shutdownSimulator( + _ simulator: SimulatorUDID, + context: @escaping @autoclosure () -> String? +) async throws { + do { + _ = try await simctl(["shutdown", simulator], context: context()) + } catch let error as ProcessError { + // Exit code 149 is related to the simulator already being shut down + guard error.exitCode == 149 else { + throw error + } + + Logger.simulatorControl.warning( + """ + ⚠️ Shutdown failed, but probably \"already shut down\": \(error, privacy: .public) + """ + ) + } +} + private func simctl( _ args: [String], context: @escaping @autoclosure () -> String? = nil diff --git a/tools/simulator_manager/Sources/SimulatorManager.swift b/tools/simulator_manager/Sources/SimulatorManager.swift index d07944c..a488b5e 100644 --- a/tools/simulator_manager/Sources/SimulatorManager.swift +++ b/tools/simulator_manager/Sources/SimulatorManager.swift @@ -112,6 +112,15 @@ actor SimulatorManager { /// mid-creation (it exists on disk before `createCloneTask` resumes and records it in /// `referenceCount`) never gets caught by a single unlucky sweep. private var previousOrphanCandidates: Set = [] + /// How many sweeps in a row the reaper has failed to delete a confirmed orphan. + /// Reset once a delete succeeds, or once the UDID stops being an orphan at all + /// (claimed by a new lease, or already cleaned up some other way). + private var orphanDeleteFailureCounts: [SimulatorUDID: Int] = [:] + /// After this many consecutive failed deletes, a device is treated as wedged + /// rather than merely unlucky: normal `simctl shutdown`/`delete` retries have + /// already been exhausted inside `SimulatorControl.delete()` every sweep, so more + /// of the same is unlikely to help. Escalate to killing its `launchd_sim` directly. + private let forceKillAfterFailedReapAttempts = 3 init( simulatorControl: SimulatorControl, @@ -214,21 +223,79 @@ actor SimulatorManager { let confirmedOrphans = currentOrphanCandidates.intersection(previousOrphanCandidates) previousOrphanCandidates = currentOrphanCandidates + // Forget the failure count for anything that isn't a confirmed orphan any more + // (claimed by a new lease, or already cleaned up), so a UDID that's reused later + // starts with a clean slate rather than inheriting an old device's history. + orphanDeleteFailureCounts = orphanDeleteFailureCounts.filter { confirmedOrphans.contains($0.key) } + guard !confirmedOrphans.isEmpty else { return } for device in managedClones where confirmedOrphans.contains(device.udid) { - Logger.simulatorManager.warning( + await reapOrphan(device) + } + } + + /// Deletes one confirmed-orphaned device, escalating to a direct kill of its + /// `launchd_sim` if it has already failed to delete + /// `forceKillAfterFailedReapAttempts` sweeps in a row. + /// + /// `SimulatorControl.delete()` already shuts the device down and retries a few + /// times internally before throwing, so a failure reaching here means those + /// retries were exhausted -- consistent with a genuinely wedged device (the same + /// profile as the multi-day-old `launchd_sim` processes that motivated this), not + /// a one-off transient error. Retrying the same call every 5-minute sweep forever + /// would just fail the same way forever, silently; escalating is what makes this a + /// backstop instead of another silent no-op. + private func reapOrphan(_ device: SimCtlDevice) async { + Logger.simulatorManager.warning( + """ + 🧹 Reaping orphaned simulator \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)); the manager has no lease or reference to it + """ + ) + + do { + try await simulatorControl.delete(device.udid, name: device.name, context: "orphan reaper") + orphanDeleteFailureCounts.removeValue(forKey: device.udid) + return + } catch { + let failures = (orphanDeleteFailureCounts[device.udid] ?? 0) + 1 + orphanDeleteFailureCounts[device.udid] = failures + + Logger.simulatorManager.error( """ - 🧹 Reaping orphaned simulator \(device.udid, privacy: .public) \ - (\(device.name, privacy: .public)); the manager has no lease or reference to it + ❌ Orphan reaper failed to delete \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)), attempt \(failures, privacy: .public): \ + \(error, privacy: .public) """ ) - try? await simulatorControl.delete( - device.udid, - name: device.name, - context: "orphan reaper" + guard failures >= forceKillAfterFailedReapAttempts else { return } + + Logger.simulatorManager.error( + """ + 🔨 \(device.udid, privacy: .public) (\(device.name, privacy: .public)) has failed to \ + delete \(failures, privacy: .public) sweeps in a row; force-killing its launchd_sim + """ ) + + await simulatorControl.forceKillLaunchdSim(for: device.udid) + + do { + try await simulatorControl.delete( + device.udid, + name: device.name, + context: "orphan reaper, post force-kill" + ) + orphanDeleteFailureCounts.removeValue(forKey: device.udid) + } catch { + Logger.simulatorManager.error( + """ + ❌ \(device.udid, privacy: .public) (\(device.name, privacy: .public)) still failed to \ + delete after force-killing launchd_sim: \(error, privacy: .public). Needs manual cleanup. + """ + ) + } } } From 51f43bccb13107a11bc75a81f7c0c32266fb589b Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 21:24:34 +0200 Subject: [PATCH 06/11] more --- #/AccumulatedHTTPHandler.swift | 80 ++ #/HTTPServer.swift | 170 +++++ #/LEASE_LIFECYCLE.md | 279 +++++++ #/LRUSet.swift | 49 ++ #/LeaseStore.swift | 112 +++ #/Logger.swift | 10 + #/Main.swift | 105 +++ #/ORPHAN_SIMULATOR_FIX.md | 195 +++++ #/PTY.swift | 38 + #/Package.resolved | 176 +++++ #/Package.swift | 16 + #/README.md | 141 ++++ #/SimulatorControl.swift | 746 +++++++++++++++++++ #/SimulatorManager.swift | 1058 +++++++++++++++++++++++++++ #/SimulatorManagerHTTPHandler.swift | 92 +++ #/SimulatorRequestHandler.swift | 104 +++ 16 files changed, 3371 insertions(+) create mode 100644 #/AccumulatedHTTPHandler.swift create mode 100644 #/HTTPServer.swift create mode 100644 #/LEASE_LIFECYCLE.md create mode 100644 #/LRUSet.swift create mode 100644 #/LeaseStore.swift create mode 100644 #/Logger.swift create mode 100644 #/Main.swift create mode 100644 #/ORPHAN_SIMULATOR_FIX.md create mode 100644 #/PTY.swift create mode 100644 #/Package.resolved create mode 100644 #/Package.swift create mode 100644 #/README.md create mode 100644 #/SimulatorControl.swift create mode 100644 #/SimulatorManager.swift create mode 100644 #/SimulatorManagerHTTPHandler.swift create mode 100644 #/SimulatorRequestHandler.swift diff --git a/#/AccumulatedHTTPHandler.swift b/#/AccumulatedHTTPHandler.swift new file mode 100644 index 0000000..0595e09 --- /dev/null +++ b/#/AccumulatedHTTPHandler.swift @@ -0,0 +1,80 @@ +import NIO +import NIOHTTP1 +import os.log + +extension Logger { + static let accumulatedHTTP = simulatorManager(category: "accumulated_http") +} + +struct FullHTTPRequest { + let head: HTTPRequestHead + var body: ByteBuffer +} + +struct FullHTTPResponse { + let head: HTTPResponseHead + var body: ByteBuffer +} + +final class AccumulatedHTTPHandler: ChannelInboundHandler, ChannelOutboundHandler { + typealias InboundIn = HTTPServerRequestPart + typealias InboundOut = FullHTTPRequest + + typealias OutboundIn = FullHTTPResponse + typealias OutboundOut = HTTPServerResponsePart + + private var requestHead: HTTPRequestHead? + private var bodyBuffer: ByteBuffer? + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + let part = self.unwrapInboundIn(data) + + switch part { + case .head(let head): + self.requestHead = head + self.bodyBuffer = context.channel.allocator.buffer(capacity: 0) + + case .body(var chunk): + self.bodyBuffer?.writeBuffer(&chunk) + + case .end: + if let head = requestHead, let body = bodyBuffer { + Logger.accumulatedHTTP.info( + """ + ▶️ Received \(head.method.rawValue, privacy: .public) request for \ + \(head.uri, privacy: .public) + """ + ) + + let fullRequest = FullHTTPRequest(head: head, body: body) + context.fireChannelRead(self.wrapInboundOut(fullRequest)) + } + + self.requestHead = nil + self.bodyBuffer = nil + } + } + + func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise?) { + let fullResponse = unwrapOutboundIn(data) + + Logger.accumulatedHTTP.info( + "◀️ Sending \(fullResponse.head.status, privacy: .public) response" + ) + + context.write(wrapOutboundOut(.head(fullResponse.head)), promise: nil) + + if fullResponse.body.readableBytes > 0 { + context.write(wrapOutboundOut(.body(.byteBuffer(fullResponse.body))), promise: nil) + } + + context.write(wrapOutboundOut(.end(nil)), promise: promise) + } + + func errorCaught(context: ChannelHandlerContext, error: Error) { + Logger.accumulatedHTTP.error( + "❌ \(error.localizedDescription, privacy: .public)" + ) + context.close(promise: nil) + } +} diff --git a/#/HTTPServer.swift b/#/HTTPServer.swift new file mode 100644 index 0000000..ad14e58 --- /dev/null +++ b/#/HTTPServer.swift @@ -0,0 +1,170 @@ +import Foundation +import NIO +import NIOExtras +import NIOHTTP1 +import NIOPosix +import os.log + +extension Logger { + static let httpServer = simulatorManager(category: "server") +} + +final class HTTPServer { + private let simulatorRequestHandler: SimulatorRequestHandler + + private let version: String + + private var serverShutdownHandler: (() -> Void)? + + init(simulatorRequestHandler: SimulatorRequestHandler, version: String) { + self.simulatorRequestHandler = simulatorRequestHandler + self.version = version + } + + func run(pidPath: String, unixSocketPath: String) async throws { + let socketURL = URL(fileURLWithPath: unixSocketPath) + let pidURL = URL(fileURLWithPath: pidPath) + + // Remove existing files if they exist + try? FileManager.default.removeItem(at: socketURL) + try? FileManager.default.removeItem(at: pidURL) + + try String(ProcessInfo.processInfo.processIdentifier).write( + to: pidURL, + atomically: true, + encoding: .utf8 + ) + + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) + + do { + // This nested block is necessary to ensure that all the destructors for objects defined + // inside are called before the final call to `eventLoopGroup.syncShutdownGracefully()`. A + // possible side effect of not doing this is a run-time error "Cannot schedule tasks on an + // EventLoop that has already shut down". + let quiesce = ServerQuiescingHelper(group: eventLoopGroup) + let fullyShutdownPromise: EventLoopPromise = eventLoopGroup.next().makePromise() + serverShutdownHandler = { + Logger.httpServer.info("⚠️ Shutting down server") + quiesce.initiateShutdown(promise: fullyShutdownPromise) + } + + do { + let serverChannel = try await ServerBootstrap(group: eventLoopGroup) + .serverChannelOption(ChannelOptions.backlog, value: 256) + .serverChannelInitializer { channel in + return channel.eventLoop.makeCompletedFuture { + try channel.pipeline.syncOperations.addHandler( + quiesce.makeServerChannelHandler(channel: channel) + ) + } + } + .bind(unixDomainSocketPath: unixSocketPath, childChannelInitializer: { childChannel in + return childChannel.eventLoop.makeCompletedFuture { + try childChannel.pipeline.syncOperations.addHandlers([ + HTTPResponseEncoder(), + ByteToMessageHandler(HTTPRequestDecoder()), + AccumulatedHTTPHandler(), + SimulatorManagerHTTPHandler(), + ]) + + return try NIOAsyncChannel( + wrappingChannelSynchronously: childChannel, + configuration: .init() + ) + } + }) + + Logger.httpServer.info("🔌 Server running on UDS at \(unixSocketPath, privacy: .public)") + + try await withThrowingDiscardingTaskGroup { group in + try await serverChannel.executeThenClose { inbound in + for try await connectionChannel in inbound { + group.addTask { + do { + try await self.handleConnection( + channel: connectionChannel + ) + } catch { + // We don't throw here, as it locks up the whole server + Logger.httpServer.error( + """ + ❌ Caught connection error: \(error, privacy: .public) + """ + ) + } + } + } + } + } + } catch { + Logger.httpServer.error("❌ Caught top-level error: \(error, privacy: .public)") + try await eventLoopGroup.shutdownGracefully() + throw error + } + + try await fullyShutdownPromise.futureResult.get() + } + + try await eventLoopGroup.shutdownGracefully() + Logger.httpServer.info("✅ Server shut down") + + // Cleanup files + try? FileManager.default.removeItem(at: socketURL) + try? FileManager.default.removeItem(at: pidURL) + } + + private func handleConnection( + channel: NIOAsyncChannel + ) async throws { + try await channel.executeThenClose { inbound, outbound in + for try await request in inbound { + try await outbound.write(handleRequest(request)) + } + } + } + + private func handleRequest( + _ request: SimulatorManagerRequest + ) async -> SimulatorManagerResponse { + switch request.path { + case "simulator": + do { + return try await simulatorRequestHandler.handleRequest( + method: request.method, + pathComponents: request.pathComponents, + queryParameters: request.queryParameters + ) + } catch { + Logger.httpServer.error( + "❌ simulatorRequestHandler.handleRequest error: \(error, privacy: .public)" + ) + + return .init( + status: .internalServerError, + message: "Internal server error: \(error)" + ) + } + + case "version": + return .init( + status: .ok, + message: version + ) + + case "leases": + let count = await simulatorRequestHandler.liveLeaseCount() + return .init(status: .ok, message: String(count)) + + case "shutdown": + // Shutting down with leases outstanding is safe: they are mirrored to disk, + // and the replacement daemon adopts the ones whose process is still running. + Logger.httpServer.info("⚠️ Shutdown request received") + serverShutdownHandler?() + return .init(status: .ok, message: "Server shutting down") + + default: + return .init(status: .badRequest, message: "Unknown method: \(request.path)") + } + } +} diff --git a/#/LEASE_LIFECYCLE.md b/#/LEASE_LIFECYCLE.md new file mode 100644 index 0000000..4747b3a --- /dev/null +++ b/#/LEASE_LIFECYCLE.md @@ -0,0 +1,279 @@ +# The lifecycle of a simulator lease + +This is a reasoning walkthrough of everything that happens to one lease, from +`POST /simulator/` to the device eventually being deleted (or handed to +someone else). It's meant to make the state machine in `SimulatorManager.swift` +easy to hold in your head, not to restate the architecture already covered in +`README.md`. + +## The actors + +- **A slot** (`SimulatorSlot`) is one device-shaped bucket for a `SimulatorConfig`. + It's `empty`, `pendingCreation`, `active`, `pendingDeletion`, or `deleting`. +- **A lease** (`SimulatorLease` in memory, `PersistedLease` on disk) is the + daemon's record that PID `p` is holding device `udid` in slot `i`. +- **A reference count** (`referenceCount[udid]`) is how many leases currently + point at that device. Non-exclusive leases for the same config can share one + device, so this can be > 1; exclusive leases never share, so it's always + exactly 1 while active. + +A lease and a slot are related but not the same thing: the slot is about the +*device*, the lease is about *who's holding it*. A non-exclusive device can +outlive any single lease on it (reused by the next leaser), and a lease always +maps to exactly one slot at a time. + +## Phase 1 — request arrives + +`SimulatorRequestHandler` parses the HTTP request and calls +`SimulatorManager.lease(to:exclusive:config:)`. First check, before anything +else: does this PID already have a lease? (`leases[leaser]` in +`SimulatorManager.swift:305`). One process, one simulator — this isn't a +counter, it's a hard rule enforced synchronously (no `await` before the +check), so two concurrent requests from the same PID can't both slip through. + +## Phase 2 — is the leaser even still there? + +Before doing anything expensive, the manager checks `processIsRunning(leaser)` +(only when `deleteOnPIDExit` is set — otherwise the caller owns the PID's +lifetime and it need not be a real process at all, e.g. in tests). This looks +redundant — surely the caller wouldn't ask for a lease if it's already dead — +but the point isn't "is it dead right now," it's "is it dead *before we commit +to a multi-minute operation*." Provisioning is the expensive part; checking +liveness is nearly free. This is the cheap half of a two-part guard; the +expensive half is Phase 4. + +## Phase 3 — finding a device: the slot state machine + +This is the core of `getSimulator()`. Slots for the requested config are +sorted by `SimulatorSlot.sortOrder` and walked in order, taking the first one +that matches. The order (after the fix earlier in this session) is: + +1. **`active`, non-exclusive, and the request is non-exclusive** — reuse it + directly. This is the fast path: no provisioning at all, just + `ensureBooted` as a sanity check. +2. **`pendingDeletion`** — a device that's fully created and booted, just + waiting out its idle timer. Reusing it is just as cheap as case 1; the + *only* reason it wasn't in case 1 is that its reference count already hit + zero once. Grabbing it cancels the scheduled deletion task and reassigns + its exclusivity to whatever the new lease wants — which is fine, since + "pending deletion" means nobody currently holds it, so there's no + conflicting owner to worry about. +3. **`empty`** — nobody's even started building a device for this slot index. + Start a fresh clone. +4. **`pendingCreation`, non-exclusive, and the request is non-exclusive** — + someone else is already cloning a device for this config; wait on their + task and count as a second (or third...) leaser of it once it's ready. + +The interesting design choice is putting `empty` *ahead of* `pendingCreation`. +It means a new non-exclusive request would rather kick off a second, +independent clone than wait behind someone else's in-flight one, even though +the in-flight one might finish sooner. My read: this trades a bit of +redundant provisioning work for not making a request's latency depend on +someone else's request. It also means the pool tends to grow to "however many +concurrent first-requests there were for a config" before it starts +consolidating onto reuse — which matches the doc's framing of `pendingCreation` +sharing as a bonus for latecomers, not the primary mechanism. + +Exclusive requests only ever match case 3 (fresh empty slot) or fall through +to appending a brand new slot — they never share `active`, `pendingCreation`, +or (implicitly, since case 2 always wins first if reached) another lease's +slot. `pendingDeletion` is the one case exclusive requests *do* match, and +that's safe precisely because "pending deletion" implies zero current owners. + +If nothing matches, a new slot is appended at the end of the array — the pool +for a config only grows on demand, never pre-allocated. + +## Phase 4 — provisioning, and when the reference count actually increments + +This part is subtle because of the actor-isolation rule spelled out in a +comment above `getSimulator`: slot state must be updated *before* the first +`await`, or a concurrent call could observe a stale slot. That rule shapes +where reference counting happens too — it can't just happen "whenever," it has +to happen at the exact point where the device becomes claimed, before any +suspension point could let the device get swept into deletion by someone else +finishing first. Three different code paths each own this responsibility: + +- **Reuse (`reuseSimulator`)**: increments immediately, *then* awaits + `ensureBooted`. If boot fails with "invalid device" (exit 148), it deletes + the corpse and clones a replacement — the increment from the original + attempt is moot because `delete()` wipes the reference count entry outright, + and the replacement gets its own fresh increment via `createCloneTask`. +- **Fresh clone (`createCloneTask`)**: increments right after the clone + finishes, before returning, on the theory (per the comment in the code) + that whoever created the device should be the one to count it, and anyone + who later shares that same task must increment separately. +- **Joining an in-flight `pendingCreation`**: increments after awaiting the + shared task's result — this is the "anyone who shares it counts separately" + half of the previous point. + +So for a config with 3 non-exclusive leases sharing one device, there are 3 +separate `incrementReferenceCount` calls across possibly all three code paths, +never a batch increment. That's what makes "decrement to zero → pendingDeletion" +in Phase 6 a safe signal: it really does mean the last leaser let go. + +Once a device is in hand, the lease is recorded in `leases[leaser]` and +persisted to disk (`persistLeases()`) *before* the second liveness check. Then +liveness is checked again — the expensive half of the Phase 2 guard. Cloning +and booting can take a real amount of wall-clock time (comment says +"minutes"), long enough for a test harness's timeout to have killed the leaser +already. If that happened, the manager doesn't hand back a UDID nobody will +ever release — it calls `release(for: leaser)` on itself immediately and +reports `leaserExited`. Recording the lease first (even though it's about to +be released) is what lets `release()` work at all; it operates purely on +`leases[leaser]`, it has no other way to find the device. + +Only after surviving both liveness checks does the manager register a process- +exit watcher (`registerReleaseOnExit`) — no point watching a PID you're not +going to hand anything to. + +## Phase 5 — active + +The device sits in `leases[leaser]` and as `.active(udid, exclusive)` in its +slot, reference-counted. Nothing happens to it on its own; it just waits for +one of the release triggers below. Multiple non-exclusive leases can be +pointing at the same UDID simultaneously, each with their own entry in +`leases`, all decrementing the same shared `referenceCount[udid]` independently +whenever *they* release. + +## Phase 6 — release: three ways in + +1. **Explicit `DELETE /simulator/`** → `SimulatorRequestHandler` → + `SimulatorManager.release(for:)`. +2. **The leasing process exits** without ever calling DELETE — caught by the + `DispatchSourceProcess` registered in Phase 4, which calls `release(for:)` + itself. This is what makes leases safe against a build tool that + `kill -9`s a test that forgot to clean up. +3. **The daemon is replaced and the leaser didn't survive the handover** — + handled entirely in `restoreLeases()` at startup, not in `release()` at all; + see Phase 8. The lease is just never adopted, so no explicit release call + ever happens for it, and the device is later picked up implicitly by name. + +`release()` itself is small and ordered deliberately: + +1. Pop the lease out of `leases` (also the point where a nonexistent lease + surfaces as `SimulatorManagerError.noLease` — expected after a daemon + restart dropped a lease the caller doesn't know is gone). +2. Persist the *now-shorter* lease list — before touching the device, so a + daemon replaced mid-release doesn't try to re-adopt a lease whose device is + about to be torn down. +3. Cancel the exit watcher (no longer relevant). +4. Clean temp files on the device. +5. Decrement the reference count, which is where the real branch happens. + +## Phase 7 — idle, and the reference count reaching zero + +`decrementReferenceCount` only does something interesting when the count hits +zero: it hands off to `pendingDeletion()`. If both idle timers are configured +as zero, deletion is immediate — no grace period at all. Otherwise a task is +scheduled that polls once a second, and *at every tick* re-decides which +deadline applies by checking `recentlyLeased.contains(config)` fresh. That's a +live check, not a snapshot taken when the task started — a config that gets +leased again elsewhere while this device is winding down flips it from the +short deadline to the long one (or the reverse, if it falls out of the +LRU set's capacity) mid-wait. It's a genuinely dynamic decision, not "pick a +deadline once." + +Two ways out of `pendingDeletion`: + +- **Resurrection**: a new lease for the same config finds this slot before the + timer expires (Phase 3, case 2) and turns it back into `active`. The + scheduled task gets cancelled, and because `Task.sleep` throws on + cancellation, the task body simply unwinds without ever calling `delete()`. +- **Timeout**: the deadline is reached, the task double-checks the slot still + holds *this exact device* (guarding against a resurrection racing the + cancellation), and calls `delete()` for real — `simctl delete`, slot to + `.deleting` then `.empty`, and (if `cleanUpSlots: true`) trims trailing empty + slots off the end of the array so the pool doesn't grow unboundedly with + dead slot indices. + +## Phase 8 — surviving a daemon restart + +This is the one part of the lifecycle that doesn't originate from an HTTP +request at all — it happens once, at startup, before the server even binds. +`start.sh` replaces the daemon on every new version, which would otherwise +silently orphan every in-flight lease. The fix is that every lease mutation in +Phases 4, 6, and 7 already persists the *entire* current lease set to disk +(atomically), so `restoreLeases()` just has to read that file back and decide, +per lease, whether to trust it: + +1. **Did the leasing process survive?** Not just "is this PID alive" — PIDs + recycle, so it also compares the process's actual start time + (`kinfo_proc`) against what was recorded at lease time. A live PID with a + *different* start time means a different process entirely; treated as + exited. +2. **Does adopting it contradict something already restored?** Two leases + can't share one exclusive device, and one slot can't hold two devices. A + file that claims otherwise is treated as corrupt rather than trusted. + +A lease that passes both checks gets its slot rebuilt as `.active`, its +reference count bumped, and (if configured) a fresh exit watcher — from that +point on it's indistinguishable from a lease that was granted moments ago by +this same daemon. A lease that fails either check is simply dropped — not +released, not cleaned up, just forgotten. Its device isn't touched at all. +`createBase`/`clone` look up devices by name before creating new ones, so an +orphaned device gets rediscovered and correctly reference-counted the next +time something happens to lease that same config — but if nothing ever leases +that exact config again, this lazy reconciliation never fires. See Phase 9 +for the backstop that catches that case. + +## Phase 9 — the orphan reaper + +Every cleanup path above is triggered by an event: a release call, a PID +exit, an idle timer, or "the same config got leased again." Each of those can +miss a device — Phase 8 above is one way (a lease that never gets re-leased), +and a `simctl delete` call inside `delete()` failing silently (it's called as +`try? await delete(...)`, and the slot is reset to `.empty` in a `defer` +regardless of whether the delete actually succeeded) is another. Either way, +nothing else ever looks for that device again. + +`delete()` failing isn't necessarily transient, either: `simctl delete` isn't +reliable against a still-booted device, and nothing in the original lifecycle +ever shuts a clone down before deleting it (a clone goes straight from +"active" to "delete this" — there's no shutdown step in between). A device +that's genuinely wedged (unresponsive, the same profile as a `launchd_sim` +that's been running for days) will fail both `shutdown` and `delete` for the +same reason every time, not just once. + +`startReaper(interval:)` runs a sweep on a timer, independent of any lease +event, that reconciles against reality instead of the event stream: it lists +every simulator named with the manager's clone prefix +(`SimulatorConfig.cloneDeviceName`, so base simulators and anything not +created by this manager are never touched) and deletes any whose UDID has no +entry in `referenceCount` — the same dictionary key presence check works here +as everywhere else, since an entry exists from the moment a device is claimed +until `delete()` removes it, including through the idle-timer grace period. + +To avoid deleting a clone that's mid-creation (it exists on disk for a moment +before `createCloneTask` resumes and records it in `referenceCount`), a +device must show up as unknown on two consecutive sweeps before it's reaped. +Deletion goes straight through `simulatorControl`, not through `delete()`: +an orphan has no slot pointing at it, so there's nothing in `simulatorSlots` +for `delete()`'s bookkeeping to update. + +`SimulatorControl.delete()` itself now shuts the device down before each of +its own delete attempts (see `shutdownSimulator` in `SimulatorControl.swift`), +so most orphans clean up on the reaper's first attempt. For the wedged case +where that still isn't enough, the reaper tracks how many sweeps in a row a +given UDID has failed to delete (`orphanDeleteFailureCounts`). After three +consecutive failures, `reapOrphan` treats the device as stuck rather than +unlucky and calls `SimulatorControl.forceKillLaunchdSim(for:)`, which finds +the device's `launchd_sim` process by `pgrep -f ` (the process's own +command line embeds its data path, so the UDID is a safe, specific pattern), +double-checks the process name before signaling it, and sends it `SIGKILL` +directly — bypassing `simctl` entirely. One more delete attempt follows; if +that still fails, the device is logged loudly as needing manual cleanup +rather than retried silently forever. + +## Where this got subtle + +- The "mutate slot before the first `await`" rule is easy to state and easy to + violate by accident; it's the reason reference-count bookkeeping is smeared + across three different functions instead of living in one place. +- `pendingDeletion` slots deliberately don't remember their old exclusivity + flag — and don't need to, because reaching that state already implies zero + current owners. +- The dynamic (not snapshotted) recheck of `recentlyLeased` inside the idle- + deletion poll loop means the *duration* of a device's grace period can + change while it's already ticking. +- Persistence is a side effect of every state change, not a periodic snapshot + or a shutdown hook — which is what makes it safe against `kill -9`. diff --git a/#/LRUSet.swift b/#/LRUSet.swift new file mode 100644 index 0000000..1f903fd --- /dev/null +++ b/#/LRUSet.swift @@ -0,0 +1,49 @@ +/// A `Set` that has a maximum capacity and evicts the least recently used item +// when full. +struct LRUSet { + private let capacity: Int + // An array to keep track of the order in which elements were inserted. + // The first element in the array is the least recently used. + private var order: [Element] = [] + + // A Set to enable fast O(1) membership tests. + private var storage: Set = [] + + init(capacity: Int) { + precondition(capacity > 0, "Capacity must be greater than zero.") + self.capacity = capacity + } + + // Returns an element that was evicted from the set, or nil if nothing was evicted + // (including when `element` was already present and just moved to most-recently-used). + mutating func insert(_ element: Element) -> Element? { + if storage.contains(element) { + if let index = order.firstIndex(of: element) { + order.remove(at: index) + } + + order.append(element) + return nil + } + + let evicted: Element? + if order.count >= capacity, let oldest = order.first { + order.removeFirst() + evicted = storage.remove(oldest) + } else { + evicted = nil + } + + order.append(element) + storage.insert(element) + return evicted + } + + func contains(_ element: Element) -> Bool { + return storage.contains(element) + } + + var elements: [Element] { + return order + } +} diff --git a/#/LeaseStore.swift b/#/LeaseStore.swift new file mode 100644 index 0000000..4c4cebd --- /dev/null +++ b/#/LeaseStore.swift @@ -0,0 +1,112 @@ +import Foundation +import os + +extension Logger { + static let leaseStore = simulatorManager(category: "manager.lease-store") +} + +/// A lease, in the form it takes on disk. +/// +/// Holds everything needed to rebuild the daemon's in-memory bookkeeping for one +/// lease: which device, under which configuration, in which slot, and whether it +/// was owned exclusively. Nothing here has to be re-derived by asking `simctl`. +struct PersistedLease: Codable, Equatable { + let pid: PID + /// The leaser's start time, used to tell "PID 500 is still running" apart from + /// "PID 500 exited and something unrelated is now PID 500". + /// + /// Optional so a record written by a build that could not read the start time + /// still loads; such a record is restored on liveness alone. + let leaserStartTime: UInt64? + let udid: SimulatorUDID + let config: SimulatorConfig + let exclusive: Bool + let slotIndex: Int +} + +/// Where the daemon keeps its leases so a successor can pick them up. +protocol LeaseStore: Sendable { + /// Replaces the stored set with `leases`. Failures are logged, not thrown: + /// losing persistence degrades a restart, but failing the lease that triggered + /// the write would break a test that is otherwise fine. + func save(_ leases: [PersistedLease]) + + /// The stored set, or empty if there is nothing readable to restore. + func load() -> [PersistedLease] +} + +/// A `LeaseStore` backed by a JSON file. +/// +/// Written atomically, so a reader never sees a half-written set and a crash +/// mid-write leaves the previous contents intact. Combined with writing on every +/// change, this means the file is always current -- there is no flush-on-shutdown +/// step for a `kill -9` to skip. +struct FileLeaseStore: LeaseStore { + let path: String + + func save(_ leases: [PersistedLease]) { + let url = URL(fileURLWithPath: path) + do { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(leases) + + // `.atomic` writes an auxiliary file alongside the destination and renames + // it over the top, which is the whole reason this is durable. Doing that by + // hand would only duplicate it, and would leave a stray file of our own + // naming behind if we crashed between the write and the rename. + try data.write(to: url, options: .atomic) + } catch { + Logger.leaseStore.error( + """ + ❌ Failed to persist \(leases.count, privacy: .public) lease(s) to \ + \(path, privacy: .public): \(error, privacy: .public) + """ + ) + } + } + + func load() -> [PersistedLease] { + let url = URL(fileURLWithPath: path) + guard FileManager.default.fileExists(atPath: path) else { + return [] + } + + do { + let data = try Data(contentsOf: url) + return try JSONDecoder().decode([PersistedLease].self, from: data) + } catch { + // A corrupt or stale-format file must not stop the daemon from starting. + // The cost of ignoring it is the old behavior: leases predating the restart + // are unknown, and their releases report no lease. + Logger.leaseStore.error( + """ + ❌ Failed to read leases from \(path, privacy: .public); continuing with \ + none: \(error, privacy: .public) + """ + ) + return [] + } + } +} + +/// When `pid` started, in microseconds since the epoch, or nil if it cannot be +/// determined (typically because the process is gone). +/// +/// A PID alone does not identify a process across a daemon restart: PIDs are +/// recycled, so a lease for a dead PID could otherwise be restored onto whatever +/// unrelated process now holds that number, tying up a device until it exits. +/// Start time distinguishes them -- the kernel assigns it at fork, so a recycled +/// PID has a different one. +func processStartTime(_ pid: PID) -> UInt64? { + var info = kinfo_proc() + var size = MemoryLayout.stride + var mib: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, pid] + + guard sysctl(&mib, UInt32(mib.count), &info, &size, nil, 0) == 0, size > 0 else { + return nil + } + + let startTime = info.kp_proc.p_starttime + return UInt64(startTime.tv_sec) * 1_000_000 + UInt64(startTime.tv_usec) +} diff --git a/#/Logger.swift b/#/Logger.swift new file mode 100644 index 0000000..39507ba --- /dev/null +++ b/#/Logger.swift @@ -0,0 +1,10 @@ +import os.log + +extension Logger { + static func simulatorManager(category: String) -> Logger { + Logger( + subsystem: "com.example.tools.simulator_manager", + category: category + ) + } +} diff --git a/#/Main.swift b/#/Main.swift new file mode 100644 index 0000000..add0094 --- /dev/null +++ b/#/Main.swift @@ -0,0 +1,105 @@ +import ArgumentParser + +@main +struct Main: AsyncParsableCommand { + // This is set externally to prevent having to recompile the manager just for `start.sh` changes + @Option(help: "Version of the simulator manager") + var version: String + + @Option(help: "Path to where the pid should be written") + var pidPath: String + + @Option(help: "Path to where the unix domain socket should be created") + var unixSocketPath: String + + @Option(help: "Number of seconds to wait before deleting a recently used idle simulator") + var deleteRecentlyUsedIdleAfter: UInt16 + + @Option(help: "Number of seconds to wait before deleting a non-recently used idle simulator") + var deleteIdleAfter: UInt16 + + @Option( + help: """ + The number of simulators to keep in the recently used list; affects wether \ + 'delete-recently-used-idle-after' or 'delete-idle-after' is used when determining when to \ + delete an unused simulator + """ + ) + var recentlyUsedCapacity = 1 + + @Option( + name: .customLong("startup-process"), + help: """ + The path to a startup process that will be run when the simulator manager is started. This \ + process will not be relaunched if it exits. + + To pass custom arguments to the process you should wrap it in a script. + + Setting this flag multiple times will result in multiple startup process being launched. + """ + ) + var startupProcesses: [String] = [] + + @Option(help: "Path to an executable that will run after a simulator clone is booted") + var postBoot: String + + @Option( + help: """ + Path to a file where leases are mirrored, so that a simulator manager started \ + to replace this one adopts the leases of tests that are still running. Omit to \ + keep leases only in memory, in which case a restart loses them. + """ + ) + var leasePath: String? + + @Option( + help: """ + Seconds between sweeps of the orphan reaper, which deletes clone simulators the \ + manager has lost track of (e.g. left behind by a prior daemon's restart, or a \ + failed deletion). 0 disables the reaper. + """ + ) + var reapIntervalSeconds: UInt16 = 300 + + func validate() throws { + guard recentlyUsedCapacity > 0 else { + throw ValidationError( + """ + 'recently-used-capacity' must be greater than 0. + """ + ) + } + + guard Set(startupProcesses).count == startupProcesses.count else { + throw ValidationError("'startup-process' must be unique.") + } + } + + func run() async throws { + let simulatorManager = SimulatorManager( + simulatorControl: RealSimulatorControl(), + deleteRecentlyUsedIdleAfter: deleteRecentlyUsedIdleAfter, + deleteIdleAfter: deleteIdleAfter, + recentlyUsedCapacity: recentlyUsedCapacity, + deleteOnPIDExit: true, + startupProcesses: startupProcesses, + postBoot: postBoot, + leaseStore: leasePath.map { FileLeaseStore(path: $0) } + ) + + // Before serving, so the first release to arrive already sees the leases this + // daemon inherited. + await simulatorManager.restoreLeases() + + await simulatorManager.startReaper(interval: .seconds(Int(reapIntervalSeconds))) + + try await simulatorManager.startChildProcesses() + + try await HTTPServer( + simulatorRequestHandler: SimulatorRequestHandler( + simulatorManager: simulatorManager + ), + version: version + ).run(pidPath: pidPath, unixSocketPath: unixSocketPath) + } +} diff --git a/#/ORPHAN_SIMULATOR_FIX.md b/#/ORPHAN_SIMULATOR_FIX.md new file mode 100644 index 0000000..45d9398 --- /dev/null +++ b/#/ORPHAN_SIMULATOR_FIX.md @@ -0,0 +1,195 @@ +# Orphaned simulator fix + +Customers reported resource contention caused by `launchd_sim` processes that +had been running for days, tied to CoreSimulator devices the daemon no longer +knew about: + +```text +PID PPID RSS ELAPSED COMMAND +78247 1 9728 05-23:05:41 launchd_sim .../Devices/B9A13CD5-.../data/var/run/launchd_bootstrap.plist +78293 1 11600 05-23:05:40 launchd_sim .../Devices/88362C6D-.../... +78337 1 9968 05-23:05:40 launchd_sim .../Devices/B8CD72C0-.../... +``` + +This document explains the two root causes and the fix. See +[`LEASE_LIFECYCLE.md`](LEASE_LIFECYCLE.md) for the full lease state machine +this fix adds to (Phase 9). + +## Root causes + +Every existing cleanup path in `SimulatorManager.swift` is event-driven: an +explicit `DELETE /simulator/`, a PID-exit watcher, an idle timer, or a +device getting rediscovered by name the next time its config is leased +again. There was no path that checked the manager's bookkeeping against what +CoreSimulator actually had running. Two specific gaps let devices escape all +of those events at once. + +### 1. Daemon restarts drop leases without touching their devices + +`start.sh` replaces the daemon on every version change. On startup, +`restoreLeases()` (`SimulatorManager.swift:166`) re-adopts a persisted lease +only if its PID is still alive *and* its recorded process start time matches +(to rule out PID reuse). A lease that fails either check is dropped — +intentionally, per the existing code comment: "not released, not cleaned up, +just forgotten." The device itself is never shut down or deleted. + +The intended recovery path was that `createBase`/`clone` look up devices by +name, so an orphaned device would get picked up and reference-counted again +the next time something leased that same `SimulatorConfig`. If that exact +config was never requested again, nothing ever looked for the device again — +it, and its `launchd_sim`, ran forever. + +### 2. Failed `simctl delete` calls were swallowed silently + +In `delete()` (`SimulatorManager.swift:758-792`), the slot was reset to +`.empty` and the reference-count entry removed in a `defer` — regardless of +whether the `simctl delete` call inside it actually succeeded. Both call +sites invoked this as `try? await delete(...)`, discarding any error with no +retry. If `simctl delete` failed for any reason (CoreSimulator daemon busy, a +lingering child process, disk I/O), the manager's bookkeeping said "deleted" +while the real device kept running — and since its slot was now empty, the +manager would never look at that UDID again. + +Both gaps produce the same outcome: a real, running simulator with zero +corresponding state in the manager. + +## The fix + +### Retry `simctl delete` before giving up + +`SimulatorControl.swift`, `SimulatorDeleteOrExistenceMutex.unlockedDelete` +now retries the underlying `simctl delete` up to 3 times, 2 seconds apart, +before rethrowing. This doesn't touch `SimulatorManager`'s actor-isolated +slot state machine, so it carries no risk to the "mutate slot before the +first `await`" invariant documented above `getSimulator()`. It simply makes +the existing call sites succeed more often instead of falling straight +through to `try?`. + +### Periodic orphan reaper + +The real fix: a sweep that reconciles the manager's state against reality, +independent of any lease event, so it catches a device regardless of *how* +it became untracked. + +- **`SimulatorControl.listManagedClones()`** (new protocol method) lists + every simulator whose name starts with `EXAMPLE_BAZEL_CLONE_` — the prefix + this manager already uses for every clone it creates + (`SimulatorConfig.cloneDeviceName`), now hoisted into the shared constant + `managedCloneNamePrefix`. Base simulators (`EXAMPLE_BAZEL_BASE_...`) are + excluded — they're intentionally long-lived templates and are never + reference-counted, so they must never be touched by the reaper. Anything + without this prefix (a developer's own simulator, Xcode's own devices) is + untouched. +- **`SimulatorManager.reapOrphanedSimulators()`** (new, private) computes + `known = Set(referenceCount.keys)` and treats any listed clone UDID not in + that set as an orphan candidate. `referenceCount` is the right thing to + check: a device gets an entry the instant it's claimed, before any + `await`, and the entry is removed only in `delete()` — including while the + device sits in its idle-timer grace period, where the count is `0` but the + key stays. So "no entry" reliably means "the manager has no idea this + exists." +- **Two-sweep confirmation.** A clone that's mid-creation exists on disk (via + `simctl clone`) for a moment before `createCloneTask` resumes and records + it in `referenceCount`. To avoid mistaking that window for an orphan, a + device must show up as unknown on two consecutive sweeps + (`previousOrphanCandidates` intersected with the current sweep's + candidates) before it's deleted. +- **Deletes bypass `delete()`.** An orphan has no slot pointing at it in + `simulatorSlots`, so there's nothing for `delete()`'s slot bookkeeping to + update. The reaper calls `simulatorControl.delete()` directly instead — the + same lower-level operation `delete()` itself wraps. +- **`SimulatorManager.startReaper(interval:)`** (new, public) runs the sweep + on a loop and is started from `Main.swift` right after `restoreLeases()`. + The interval is configurable via the new `--reap-interval-seconds` flag + (default 300; `0` disables the reaper), and the task is cancelled in + `deinit` alongside the other background tasks. + +## Follow-up: the report didn't go away + +After the fix above, the same customer report came back. Investigating that +turned up two separate things: + +**The fix likely never shipped.** The commit was still local to this branch +(`git status` showed it 1 commit ahead of `origin/yannic-simulator-manager`, +never pushed), this package has no `BUILD` file or `Package.swift` — nothing +wires it into a build or deploy pipeline — and this branch's history includes +an earlier commit titled `DO NOT MERGE: Simulator manager`. Nothing here +indicates a build customers actually run has changed at all. + +**Independent of that, a real gap in the reaper itself.** Both the original +`delete()` and the new reaper's delete path called `simctl delete` without +ever calling `shutdown` first, and both discarded failures with `try?` and no +escalation. That's survivable for a healthy device, but a device that's +genuinely wedged — exactly the profile of a `launchd_sim` that's been running +for days — will fail `shutdown` and `delete` for the same underlying reason +every time. The reaper would correctly identify such a device as orphaned +after two sweeps, then retry a delete that fails identically every 5 minutes, +forever, silently. Retrying a deterministically broken operation isn't a fix. +See the next section for what addresses this. + +## Additional fix: shut down before delete, escalate when delete keeps failing + +- **`shutdownSimulator`** (new private free function in `SimulatorControl.swift`) + factors out the shutdown-with-"already shut down"-handling logic that + previously only backed the public `shutdown()` method. `unlockedDelete` now + calls it before each of its (still up to 3) delete attempts, best-effort + (`try?` — a shutdown failure for any other reason shouldn't block trying + delete anyway, since delete is what actually matters). +- **`SimulatorControl.forceKillLaunchdSim(for:)`** (new protocol method) is the + last resort for a device that keeps failing to delete. It finds the + device's `launchd_sim` process with `pgrep -f ` (the process's own + command line embeds its data path, so the UDID is a safe, specific + pattern), double-checks the process name before signaling it (`ps -p + -o comm=`, must end in `launchd_sim`), and sends `SIGKILL` directly, + bypassing `simctl` entirely. +- **`SimulatorManager` tracks consecutive reap failures per UDID** + (`orphanDeleteFailureCounts`). `reapOrphan` (new, factored out of + `reapOrphanedSimulators`) deletes normally on each sweep; after 3 + consecutive failures for the same UDID, it calls `forceKillLaunchdSim` and + makes one more delete attempt. If that still fails, it logs loudly ("needs + manual cleanup") instead of retrying the same failing call forever. The + failure count resets whenever a delete succeeds or the UDID stops being an + orphan at all. + +## Files changed + +| File | Change | +|:-----|:-------| +| `SimulatorControl.swift` | Retry + shutdown-first logic in `unlockedDelete`; new `listManagedClones()` and `forceKillLaunchdSim(for:)`; hoisted `managedCloneNamePrefix` constant; factored-out `shutdownSimulator` | +| `SimulatorManager.swift` | New `startReaper(interval:)`, `reapOrphanedSimulators()`, and `reapOrphan(_:)`; new `reaperTask`, `previousOrphanCandidates`, and `orphanDeleteFailureCounts` state | +| `Main.swift` | New `--reap-interval-seconds` flag; starts the reaper after `restoreLeases()` | +| `LEASE_LIFECYCLE.md` | Corrected Phase 8's claim that nothing hunts down orphans; added Phase 9 describing the reaper and its escalation path | + +## What this does not fix + +- The slot state machine itself (`SimulatorSlot` cases, `getSimulator`, the + "mutate before first `await`" rule) is unchanged. The reaper works + alongside it by going straight to `simulatorControl`, so the delicate + actor-isolation invariant didn't need to be touched. +- The Go port at `experiments/yannic/macsimulatormanager/go` was out of + scope for this fix — the bug report and investigation were both scoped to + the Swift implementation. + +## Verification status + +There is no `BUILD` file or `Package.swift` for this target yet, so it isn't +wired into Bazel and can't be built with `swift build` either. This change +has been verified by code review against the existing invariants (see +`LEASE_LIFECYCLE.md`), not by compiling or running it. Before relying on +this in production: + +1. Confirm this branch is actually the source for whatever customers run — + given the missing build target and the `DO NOT MERGE` history noted + above, that's not yet established. If it is, push this branch and get it + merged; if there's a separate deploy path, find it. +2. Set up a build target (Bazel or an ad hoc `Package.swift`) so this code + compiles and can be typechecked — it currently depends on `ShellOut`, + `ArgumentParser`, and SwiftNIO, none of which are vendored here. +3. Manually exercise the restart-orphan path: lease a config, `kill -9` the + daemon, start a new instance with a short `--reap-interval-seconds`, and + confirm the orphaned clone disappears from `xcrun simctl list devices` + within two sweep intervals without touching unrelated simulators. +4. Manually exercise the wedged-device escalation path: lease a config, then + independently hang or otherwise make its clone's `simctl shutdown`/`delete` + fail out-of-band, and confirm the reaper force-kills its `launchd_sim` + after 3 failed sweeps rather than retrying forever. diff --git a/#/PTY.swift b/#/PTY.swift new file mode 100644 index 0000000..6f11ed4 --- /dev/null +++ b/#/PTY.swift @@ -0,0 +1,38 @@ +import Darwin + +struct PTY { + let parent: Int32 + let child: Int32 + + init() throws { + var parentFd: Int32 = 0 + var childFd: Int32 = 0 + + // NULL for name/pw/termios/winsize = defaults + let result = openpty(&parentFd, &childFd, nil, nil, nil) + guard result == 0 else { + throw Errno(rawValue: errno) + } + + self.parent = parentFd + self.child = childFd + } +} + +/// Simple POSIX errno wrapper. +struct Errno: Error, RawRepresentable { + /// The raw POSIX error number. + let rawValue: Int32 + + init(rawValue: Int32) { + self.rawValue = rawValue + } +} + +extension Errno: CustomStringConvertible { + var description: String { + var buf = [CChar](repeating: 0, count: 256) + strerror_r(rawValue, &buf, buf.count) + return String(cString: buf) + } +} diff --git a/#/Package.resolved b/#/Package.resolved new file mode 100644 index 0000000..0691b04 --- /dev/null +++ b/#/Package.resolved @@ -0,0 +1,176 @@ +{ + "pins" : [ + { + "identity" : "shellout", + "kind" : "remoteSourceControl", + "location" : "https://github.com/JohnSundell/ShellOut", + "state" : { + "revision" : "e1577acf2b6e90086d01a6d5e2b8efdaae033568", + "version" : "2.3.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", + "version" : "1.8.2" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", + "version" : "1.19.4" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", + "version" : "4.5.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", + "version" : "1.7.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "db774a277f60063a32d854f2980299caf06da041", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", + "version" : "2.101.3" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras", + "state" : { + "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", + "version" : "1.34.3" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", + "version" : "1.45.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", + "version" : "2.37.2" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", + "version" : "2.12.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", + "version" : "1.8.1" + } + } + ], + "version" : 2 +} diff --git a/#/Package.swift b/#/Package.swift new file mode 100644 index 0000000..e2830e1 --- /dev/null +++ b/#/Package.swift @@ -0,0 +1,16 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +// This manifest exists only so `rules_swift_package_manager` can resolve and vendor +// this daemon's third-party dependencies for Bazel (see the BUILD file in this +// directory). It declares no targets/products of its own -- the daemon is built as +// a `swift_binary` directly from these sources, not as a Swift package. +let package = Package( + name: "macsimulatormanager", + dependencies: [ + .package(url: "https://github.com/JohnSundell/ShellOut", from: "2.3.0"), + .package(url: "https://github.com/apple/swift-argument-parser", from: "1.8.2"), + .package(url: "https://github.com/apple/swift-nio-extras", from: "1.34.3"), + ] +) diff --git a/#/README.md b/#/README.md new file mode 100644 index 0000000..3d9eda7 --- /dev/null +++ b/#/README.md @@ -0,0 +1,141 @@ +# Mac simulator manager + +A long-running daemon that manages the lifecycle of iOS Simulator devices and +hands them out, over HTTP, as leased resources for tests running on a Mac +worker. It exists so that concurrent test runs share a small pool of +simulators instead of each provisioning and tearing down its own, which is +slow and resource-hungry. + +The daemon speaks HTTP over a Unix domain socket rather than a TCP port, +since it is only ever talked to by processes on the same machine. + +## Running it + +`Main.swift` is a `swift-argument-parser` command. Notable options: + +- `--pid-path` / `--unix-socket-path` — where the daemon writes its PID file + and creates the listening socket. +- `--delete-idle-after` / `--delete-recently-used-idle-after` — how long an + unleased simulator is kept around before deletion, depending on whether its + config was recently leased (see [Idle deletion](#idle-deletion)). +- `--recently-used-capacity` — how many distinct simulator configs count as + "recently used" at once. +- `--startup-process` (repeatable) — extra processes to launch alongside the + daemon; see [Child processes](#child-processes). +- `--post-boot` — a script run against every freshly booted clone (see + [`SimulatorControl.swift`](SimulatorControl.swift)). +- `--lease-path` — where leases are mirrored to disk so a replacement daemon + can adopt them; see [Restarts and lease persistence](#restarts-and-lease-persistence). + +The `--version` flag is not read from source; it is passed in externally +(e.g. by a wrapper script) so the manager doesn't need to be recompiled just +to change how it reports its own version. + +## HTTP API + +Requests are parsed by `SimulatorManagerHTTPHandler.swift` and routed by +`HTTPServer.swift` on the first path component: + +| Method | Path | Query params | Description | +|--------|---------------------|--------------------------------------------|--------------| +| `POST` | `/simulator/` | `exclusive`, `deviceType`, `os`, `version` | Lease a simulator matching the given config to the process ``. Returns the simulator's UDID. | +| `DELETE` | `/simulator/` | | Release the simulator leased to ``. | +| `GET` | `/version` | | The daemon's version string. | +| `GET` | `/leases` | | The number of leases whose leasing process is still alive; for diagnostics. | +| `POST` | `/shutdown` | | Begin a graceful shutdown. | + +`exclusive=1` requests a simulator that no other lease may share; otherwise +the manager may hand out a simulator that is already leased non-exclusively +for the same config. Request parsing and response mapping live in +[`SimulatorRequestHandler.swift`](SimulatorRequestHandler.swift). + +## Architecture + +``` +HTTPServer (SwiftNIO, Unix domain socket) + └─ AccumulatedHTTPHandler buffers HTTP head/body/end into one FullHTTPRequest + └─ SimulatorManagerHTTPHandler parses method/path/query into a SimulatorManagerRequest + └─ SimulatorRequestHandler maps HTTP requests to SimulatorManager calls + └─ SimulatorManager (actor) lease/slot/reference-count bookkeeping + └─ SimulatorControl wraps `xcrun simctl` (create/clone/boot/delete) +``` + +### `SimulatorManager` + +[`SimulatorManager.swift`](SimulatorManager.swift) is the core state machine, +implemented as an actor so its bookkeeping is safe under concurrent leases. +For each `SimulatorConfig` (device type, OS, version) it keeps an array of +slots, each of which is `empty`, `pendingCreation`, `active`, `pendingDeletion`, +or `deleting`. Leasing a config walks the slots in a fixed preference order +(reuse an active non-exclusive simulator, then a pending deletion, then an +empty slot, then a pending creation) and either reuses what's there or clones +a fresh device from a per-config base simulator (created lazily and cached in +`getBaseSimulatorTasks`). + +Every active simulator has a reference count. Leasing increments it; releasing +decrements it, and it is only queued for deletion once it drops to zero, so a +non-exclusive simulator with multiple leasers survives until all of them +release it. + +### Idle deletion + +Once a simulator's reference count hits zero it isn't deleted immediately — +`pendingDeletion` schedules a delayed delete so a device can be reused by the +next lease for the same config. The wait is either `deleteIdleAfter` or the +longer `deleteRecentlyUsedIdleAfter`, chosen by whether that config appears in +`recentlyLeased`, an `LRUSet` (see [`LRUSet.swift`](LRUSet.swift)) capped at +`recentlyUsedCapacity` distinct configs. This keeps simulators for +in-demand configs around longer while letting one-off configs get cleaned up +quickly. + +### Restarts and lease persistence + +A new daemon version replaces the running one (the caller kills the old +process and starts the new one), which would otherwise lose track of leases +held by tests that are still running. `LeaseStore.swift` mirrors every lease +change to a JSON file; on startup, `restoreLeases()` reads it back and rebuilds +enough state (the lease, the slot holding the device, the reference count, and +an exit listener) for `release` to work normally. A lease is only adopted if +its leasing process is still running under the *same* start time (`kinfo_proc` +via `processStartTime`, not just the same PID — PIDs get recycled) and doesn't +conflict with another lease already restored; otherwise it's dropped, and its +device is picked up again automatically the next time something leases that +config. + +A leaser's exit is also watched for directly, via +`DispatchSource.makeProcessSource`, so a lease is released automatically if +its process dies without calling `DELETE /simulator/`. + +### `SimulatorControl` + +[`SimulatorControl.swift`](SimulatorControl.swift) wraps `xcrun simctl` calls +(`create`, `clone`, `bootstatus`, `shutdown`, `delete`, `list devices`) behind +a protocol, so `SimulatorManager` can be tested against a fake. Concurrent +calls for the same base/clone name are coalesced onto a single in-flight +`Task`, and deletion/existence checks for a given name are serialized through +`SimulatorDeleteOrExistenceMutex` so a `clone` can't observe a device that's +mid-deletion. + +### Child processes + +The daemon can launch extra long-lived processes alongside itself +(`--startup-process`). Each one's stdout/stderr is captured through a +`PTY` (see [`PTY.swift`](PTY.swift), needed because plain pipes make some +tools line-buffer differently) and logged line-by-line via `os.Logger`. +These processes are not restarted if they exit. + +## Files + +| File | Purpose | +|------|---------| +| `Main.swift` | CLI entry point; wires flags into a `SimulatorManager` and `HTTPServer`. | +| `HTTPServer.swift` | SwiftNIO server bound to a Unix domain socket; top-level request routing. | +| `AccumulatedHTTPHandler.swift` | Buffers streamed HTTP request parts into one in-memory request/response. | +| `SimulatorManagerHTTPHandler.swift` | Parses the HTTP request into method/path/query; serializes responses. | +| `SimulatorRequestHandler.swift` | Translates `/simulator` requests into `SimulatorManager` lease/release calls. | +| `SimulatorManager.swift` | Core actor: slots, reference counts, leases, idle deletion, child processes. | +| `SimulatorControl.swift` | `simctl`-backed implementation of creating, cloning, booting, and deleting simulators. | +| `LeaseStore.swift` | Persists leases to disk so a replacement daemon can adopt them. | +| `LRUSet.swift` | Fixed-capacity, least-recently-used set used to track recently leased configs. | +| `PTY.swift` | Minimal pseudo-terminal wrapper used to capture child process output. | +| `Logger.swift` | Shared `os.Logger` subsystem/category helper. | diff --git a/#/SimulatorControl.swift b/#/SimulatorControl.swift new file mode 100644 index 0000000..b4b46c1 --- /dev/null +++ b/#/SimulatorControl.swift @@ -0,0 +1,746 @@ +import Foundation +import os.log +import ShellOut + +typealias SimulatorUDID = String + +extension Logger { + static let simulatorControl = simulatorManager(category: "control") +} + +struct SimulatorConfig: Hashable, Equatable, Codable { + let deviceType: String + let os: String + let version: String +} + +extension SimulatorConfig: CustomStringConvertible { + var description: String { + return "\(deviceType) (\(os) \(version))" + } +} + +struct SimCtlDevices: Decodable { + let devices: [String: [SimCtlDevice]] +} + +struct SimCtlDevice: Decodable { + let name: String + let udid: String +} + +struct ProcessError: Error { + let command: String + let context: String? + let exitCode: Int32 + let stdOut: String + let stdErr: String +} + +extension ProcessError: CustomStringConvertible { + var description: String { + let contextStr: String + if let context { + contextStr = " (\(context))" + } else { + contextStr = "" + } + + return """ + "\(command)"\(contextStr) failed with exit code \(exitCode): + \(stdOut)\(stdErr) + """ + } +} + +extension ProcessError: LocalizedError { + var errorDescription: String? { + return description + } +} + +protocol SimulatorControl: Actor { + // Creates a base simulator with the given config. + // + // It also boots and shuts down the simulator, making it ready for cloning. + // + // If an existing simulator with the same name already exists, that is returned instead of + // creating a new one. This is to support the simulator manager being restarted and losing state. + func createBase( + name: String, + with config: SimulatorConfig, + runtimeIdentifier: String + ) async throws -> SimulatorUDID + + // Clones a base simulator. + // + // It also boots the cloned simulator, making it ready for use. + // + // If an existing simulator with the same name already exists, that is returned instead of + // creating a new one. This is to support the simulator manager being restarted and losing state. + func clone( + _ baseSimulator: SimulatorUDID, + name: String, + deviceType: String, + runtimeIdentifier: String, + postBoot: String? + ) async throws -> SimulatorUDID + + func ensureBooted( + _ simulator: SimulatorUDID, + context: @escaping @autoclosure () -> String? + ) async throws + + func cleanTempFiles(in simulator: SimulatorUDID) + + func delete( + _ simulator: SimulatorUDID, + name: String, + context: @escaping @autoclosure () -> String? + ) async throws + + func getExisting( + name: String, + deviceType: String, + runtimeIdentifier: String, + context: @escaping @autoclosure () -> String? + ) async throws -> String? + + // Every clone-named simulator that currently exists, across all runtimes. + // + // Used by the orphan reaper to reconcile the manager's bookkeeping against reality. + // Deliberately excludes base simulators, which are meant to be long-lived and are + // never reference-counted. + func listManagedClones() async throws -> [SimCtlDevice] + + // Kills the `launchd_sim` process for `simulator` directly, bypassing `simctl` + // entirely. Best-effort and never throws: this is the last resort for a device + // that keeps failing `delete()` through normal means (a wedged simulator can fail + // both `shutdown` and `delete` indefinitely), used by the orphan reaper's + // escalation path. + func forceKillLaunchdSim(for simulator: SimulatorUDID) async +} + +actor RealSimulatorControl: SimulatorControl { + private var createBaseTasks: [String: Task] = [:] + private var cloneTasks: [String: Task] = [:] + + private var deleteAndExistenceMutexes: [String: (SimulatorDeleteOrExistenceMutex, Int)] = [:] + + func createBase( + name: String, + with config: SimulatorConfig, + runtimeIdentifier: String + ) async throws -> SimulatorUDID { + if let existingTask = createBaseTasks[name] { + return try await existingTask.value + } + + // We use a task to prevent data races that can occur when the `await` on `simctl` blocks. This + // ensures that multiple callers trying create a base simulator will all wait for the same + // simulator to be returned. + let task = Task { + defer { + createBaseTasks.removeValue(forKey: name) + } + + if let existingUDID = try await getExisting( + name: name, + deviceType: config.deviceType, + runtimeIdentifier: runtimeIdentifier, + context: "createBase" + ) { + Logger.simulatorControl.info( + """ + 📱 Base simulator "\(name, privacy: .public)" already exists, skipping creation: \ + \(existingUDID, privacy: .public) + """ + ) + + do { + // Under weird circumstances, the base simulator might be booted. This could happen if + // the simulator manager is killed in the process of creating a new base. Always call + // shutdown just in case. + try await shutdown(existingUDID, context: "createBase existing: \(name)") + } catch { + // If we fail to do what we need to, then we need to delete the faulty base simulator + Logger.simulatorControl.error( + """ + 📱 Failed to set up base simulator "\(name)" \(existingUDID, privacy: .public); deleting + """ + ) + + // If we fail to delete, don't throw _that_ error, throw the original error + try? await delete(existingUDID, name: name, context: "createBase existing: \(name)") + + throw error + } + + return existingUDID + } + + Logger.simulatorControl.info( + #"📱 Creating \#(config, privacy: .public) base simulator "\#(name, privacy: .public)""# + ) + + let udid = try await simctl( + ["create", name, config.deviceType, runtimeIdentifier] + ).trimmingCharacters(in: .whitespacesAndNewlines) + + do { + try await ensureBooted(udid, context: "createBase new: \(name)") + + // FIXME: Find a better way to know the simulator is ready + // Give the simulator some time to do some post-boot processing + try await Task.sleep(for: .seconds(5)) + + try await shutdown(udid, context: "createBase new: \(name)") + } catch { + // If we fail to do what we need to, then we need to delete the faulty base simulator + Logger.simulatorControl.error( + #""📱 Failed to set up base simulator "\#(name)" \#(udid, privacy: .public); deleting"# + ) + + // If we fail to delete, don't throw _that_ error, throw the original error + try? await delete(udid, name: name, context: "createBase new: \(name)") + + throw error + } + + Logger.simulatorControl.info( + """ + 📱 Created \(config, privacy: .public) base simulator \ + "\(name, privacy: .public)": \(udid, privacy: .public) + """ + ) + + return udid + } + + createBaseTasks[name] = task + + return try await task.value + } + + func clone( + _ baseSimulator: SimulatorUDID, + name: String, + deviceType: String, + runtimeIdentifier: String, + postBoot: String? = nil + ) async throws -> SimulatorUDID { + if let existingTask = cloneTasks[name] { + return try await existingTask.value + } + + // We use a task to prevent data races that can occur when the `await` on `simctl` blocks. This + // ensures that multiple callers trying create a base simulator will all wait for the same + // simulator to be returned. + let task = Task { + defer { + cloneTasks.removeValue(forKey: name) + } + + let udid: String + let isExisting: Bool + if let existingUDID = try await getExisting( + name: name, + deviceType: deviceType, + runtimeIdentifier: runtimeIdentifier, + context: "clone" + ) { + udid = existingUDID + isExisting = true + + // An existing simulator can be found if a previous simulator manager was killed before the + // clone was deleted. No tests _should_ be actively leasing the simulator. + Logger.simulatorControl.info( + """ + 📱 Cloned simulator "\(name, privacy: .public)" already exists, skipping creation: \ + \(udid, privacy: .public) + """ + ) + + // Wait for it to boot. This shouldn't be necessary, but sometimes the simulator will + // reboot because of a migration. + try await ensureBooted(udid, context: "clone, existing: \(name)") + } else { + isExisting = false + + Logger.simulatorControl.info( + """ + 📱 Cloning base simulator \(baseSimulator, privacy: .public) as \ + "\(name, privacy: .public)" + """ + ) + + udid = try await simctl( + ["clone", baseSimulator, name] + ).trimmingCharacters(in: .whitespacesAndNewlines) + + Logger.simulatorControl.info( + """ + 📱 Cloned base simulator \(baseSimulator, privacy: .public) as \ + "\(name, privacy: .public)": \(udid, privacy: .public) + """ + ) + + try await ensureBooted(udid, context: "clone, new: \(name)") + } + + if let postBoot { + Logger.simulatorControl.info( + """ + 📱 Running post-boot script "\(postBoot, privacy: .public)" on \ + \(udid, privacy: .public) + """ + ) + + do { + _ = try await subprocess(postBoot, env: ["SIMULATOR_UDID": udid]) + } catch { + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: + [NSLocalizedDescriptionKey: "postBoot failed (isExisting: \(isExisting)): \(error)"] + ) + } + } + + return udid + } + + cloneTasks[name] = task + + return try await task.value + } + + func shutdown(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { + try await shutdownSimulator(simulator, context: context()) + } + + func cleanTempFiles(in simulator: SimulatorUDID) { + let fileManager = FileManager.default + + // Remove all files and directories under + // `data/Library/Caches/com.apple.containermanagerd/Dead/`, ignoring errors. There seems to be + // a bug where the simulator moves files here but never cleans them up. Maybe it's waiting for + // a reboot or something, which we never do. + let deadCachesPath = + "\(NSHomeDirectory())/Library/Developer/CoreSimulator/Devices/\(simulator)/data/Library/Caches/com.apple.containermanagerd/Dead" + guard let contents = try? fileManager.contentsOfDirectory(atPath: deadCachesPath) else { + return + } + for item in contents { + let itemPath = "\(deadCachesPath)/\(item)" + try? fileManager.removeItem(atPath: itemPath) + } + } + + func delete( + _ simulator: SimulatorUDID, + name: String, + context: @escaping @autoclosure () -> String? + ) async throws { + try await deleteAndExistenceMutex(name: name) { mutex in + try await mutex.unlockedDelete(simulator, context: context()) + } + } + + func getExisting( + name: String, + deviceType: String, + runtimeIdentifier: String, + context: @escaping @autoclosure () -> String? + ) async throws -> String? { + return try await deleteAndExistenceMutex(name: name) { mutex in + return try await mutex.unlockedGetExisting( + name: name, + deviceType: deviceType, + runtimeIdentifier: runtimeIdentifier, + context: context() + ) + } + } + + func listManagedClones() async throws -> [SimCtlDevice] { + let output = try await simctl(["list", "devices", "-j"], context: "listManagedClones") + + guard let jsonData = output.data(using: .utf8) else { + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to convert output to data"] + ) + } + + let devicesByRuntime: [String: [SimCtlDevice]] + do { + devicesByRuntime = try JSONDecoder().decode(SimCtlDevices.self, from: jsonData).devices + } catch { + let json = String(data: jsonData, encoding: .utf8) ?? "" + Logger.simulatorControl.error( + """ + ❌ Failed to decode 'simctl list devices -j': \(error, privacy: .public). + Output: \(json, privacy: .public) + """ + ) + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to decode output: \(error) - \(json)"] + ) + } + + return devicesByRuntime.values.flatMap { $0 }.filter { $0.name.hasPrefix(managedCloneNamePrefix) } + } + + func forceKillLaunchdSim(for simulator: SimulatorUDID) async { + // `launchd_sim`'s command line embeds the device's own data path (see the ps + // output that motivated this: ".../Devices//data/var/run/..."), so the + // UDID is a safe, specific `pgrep -f` pattern -- it can only match that one + // device's process tree. + let pids: [Int32] + do { + let output = try await subprocess("/usr/bin/pgrep", ["-f", simulator]) + pids = output.split(separator: "\n").compactMap { Int32($0.trimmingCharacters(in: .whitespaces)) } + } catch { + // `pgrep` exits non-zero (surfaced here as a thrown `ProcessError`) when nothing + // matches -- there's nothing left to kill. + return + } + + for pid in pids { + // Double check this is actually `launchd_sim` before signaling it. `pgrep -f` + // matches the UDID anywhere in the command line; being wrong here would kill an + // unrelated process that merely mentioned this device (e.g. in a log path). + guard let comm = try? await subprocess("/bin/ps", ["-p", "\(pid)", "-o", "comm="]), + comm.trimmingCharacters(in: .whitespacesAndNewlines).hasSuffix("launchd_sim") else { + continue + } + + Logger.simulatorControl.warning( + """ + 🔨 Force-killing launchd_sim (pid \(pid, privacy: .public)) for stuck simulator \ + \(simulator, privacy: .public) + """ + ) + kill(pid, SIGKILL) + } + } + + func ensureBooted(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { + for retriesLeft in (0...1).reversed() { + do { + // This private command boots the simulator if it isn't already, and waits for the + // appropriate amount of time until we can actually run tests + _ = try await simctl(["bootstatus", simulator, "-b"], context: context()) + break + } catch let error as ProcessError { + // Exit code 149 is related to the simulator already being booted + guard error.exitCode == 149 && retriesLeft > 0 else { + throw error + } + + // This is a known error that happens when the simulator is already booted. A retry + // should succeed. + Logger.simulatorControl.warning( + """ + ⚠️ Boot of simulator \(simulator, privacy: .public) failed, but probably \"already \ + booted\": \(error, privacy: .public) + """ + ) + } + } + } + + private func deleteAndExistenceMutex( + name: String, + _ call: (_ mutex: SimulatorDeleteOrExistenceMutex) async throws -> T + ) async throws -> T { + var (mutex, referenceCount) = + deleteAndExistenceMutexes[name] ?? (SimulatorDeleteOrExistenceMutex(), 0) + referenceCount += 1 + deleteAndExistenceMutexes[name] = (mutex, referenceCount) + + defer { + guard let mutexAndRef = deleteAndExistenceMutexes[name] else { + preconditionFailure( + """ + State of `deleteAndExistenceMutexes` changed unexpectedly. Expected value for "\(name)". + """ + ) + } + let mutex = mutexAndRef.0 + var referenceCount = mutexAndRef.1 + referenceCount -= 1 + + if referenceCount == 0 { + deleteAndExistenceMutexes.removeValue(forKey: name) + } else { + deleteAndExistenceMutexes[name] = (mutex, referenceCount) + } + } + + return try await mutex.withLock { + try await call(mutex) + } + } +} + +// An instance of this actor is created for each simulator name that is being checked for existence +// or being deleted. The actor is only called through `withLock()`, which will suspend on multiple +// calls to ensure that these operations are serialized. Without this, someone could try to call +// `clone()` while a deletion is pending, which will call `getExisting()`, and it can return the +// simulator that is in the process of being deleted. +actor SimulatorDeleteOrExistenceMutex { + private var isLocked = false + private var waiters: [CheckedContinuation] = [] + + /// Acquires, runs the work, and then releases the lock. + func withLock(_ work: () async throws -> T) async throws -> T { + await lock() + defer { unlock() } + return try await work() + } + + /// Acquires the lock. If already locked, will suspend until unlocked. + private func lock() async { + if !isLocked { + isLocked = true + } else { + await withCheckedContinuation { cont in + waiters.append(cont) + } + } + } + + /// Releases the lock and wakes one waiter (if any). + private func unlock() { + if !waiters.isEmpty { + let cont = waiters.removeFirst() + cont.resume() + } else { + isLocked = false + } + } + + func unlockedGetExisting( + name: String, + deviceType: String, + runtimeIdentifier: String, + context: @escaping @autoclosure () -> String? + ) async throws -> String? { + Logger.simulatorControl.debug( + #"🔍 Trying to find existing simulator "\#(name, privacy: .public)""# + ) + + let output = try await simctl(["list", "devices", "-j", deviceType], context: context()) + + guard let jsonData = output.data(using: .utf8) else { + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to convert output to data"] + ) + } + + let jsonDecoder = JSONDecoder() + + let devicesByRuntime: [String: [SimCtlDevice]] + do { + devicesByRuntime = try jsonDecoder.decode(SimCtlDevices.self, from: jsonData).devices + } catch { + let json = String(data: jsonData, encoding: .utf8) ?? "" + Logger.simulatorControl.error( + """ + ❌ Failed to decode 'simctl list devices -j': \(error, privacy: .public). + Output: \(json, privacy: .public) + """ + ) + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to decode output: \(error) - \(json)"] + ) + } + + if let devices = devicesByRuntime[runtimeIdentifier] { + for device in devices { + if device.name == name { + let udid = device.udid + + Logger.simulatorControl.debug( + #"🔍 Found existing simulator "\#(name, privacy: .public)": \#(udid, privacy: .public)"# + ) + + // Sometimes the simulator is not actually on disk, but it is in the list. If this + // happens, "delete" it so simctl stops reporting it as existing. + if !FileManager.default.fileExists( + atPath: + "\(NSHomeDirectory())/Library/Developer/CoreSimulator/Devices/\(udid)" + ) { + Logger.simulatorControl.debug( + """ + ⚠️ Simulator \(udid, privacy: .public) doesn't actually exist on disk; "deleting" + """ + ) + + // If we fail to delete, don't throw an error + try? await unlockedDelete(udid, context: context()) + + return nil + } + + return udid + } + } + } + + Logger.simulatorControl.debug( + #"🔍 No existing simulator "\#(name, privacy: .public)" found"# + ) + + return nil + } + + func unlockedDelete( + _ simulator: SimulatorUDID, + context: @escaping @autoclosure () -> String? + ) async throws { + Logger.simulatorControl.info("🗑️ Deleting simulator \(simulator, privacy: .public)") + + // `simctl delete` can fail transiently (CoreSimulator daemon busy, a lingering + // child process, disk I/O) or because the device is still booted -- `delete` + // does not reliably shut a booted device down on its own. Callers treat a thrown + // error here as "give up until the next event," so shut down and retry a few + // times before surfacing failure -- it's much cheaper than leaving the simulator + // running until the orphan reaper's next sweep catches it. + let maxAttempts = 3 + for attempt in 1...maxAttempts { + // Best-effort: proceed to the delete attempt regardless of whether this + // succeeds. A shutdown failure for a reason other than "already shut down" + // (already handled inside `shutdownSimulator`) shouldn't block trying delete + // anyway, since delete is the operation that actually matters here. + try? await shutdownSimulator(simulator, context: context()) + + do { + _ = try await simctl(["delete", simulator], context: context()) + Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") + return + } catch { + guard attempt < maxAttempts else { + Logger.simulatorControl.error( + """ + ❌ Failed to delete simulator \(simulator, privacy: .public) after \ + \(maxAttempts, privacy: .public) attempts: \(error, privacy: .public) + """ + ) + throw error + } + + Logger.simulatorControl.warning( + """ + ⚠️ Delete attempt \(attempt, privacy: .public)/\(maxAttempts, privacy: .public) for \ + simulator \(simulator, privacy: .public) failed, retrying: \(error, privacy: .public) + """ + ) + try? await Task.sleep(for: .seconds(2)) + } + } + } +} + +/// Shared by `RealSimulatorControl.shutdown` and `unlockedDelete`, since delete needs +/// to shut the device down first and shouldn't reimplement this. +private func shutdownSimulator( + _ simulator: SimulatorUDID, + context: @escaping @autoclosure () -> String? +) async throws { + do { + _ = try await simctl(["shutdown", simulator], context: context()) + } catch let error as ProcessError { + // Exit code 149 is related to the simulator already being shut down + guard error.exitCode == 149 else { + throw error + } + + Logger.simulatorControl.warning( + """ + ⚠️ Shutdown failed, but probably \"already shut down\": \(error, privacy: .public) + """ + ) + } +} + +private func simctl( + _ args: [String], + context: @escaping @autoclosure () -> String? = nil +) async throws -> String { + return try await subprocess("/usr/bin/xcrun", ["simctl"] + args, context: context()) +} + +private func subprocess( + _ executable: String, + _ args: [String] = [], + env: [String: String] = [:], + context: @escaping @autoclosure () -> String? = nil +) async throws -> String { + return try await Task { try syncSubprocess(executable, args, env: env, context: context()) }.value +} + +private func syncSubprocess( + _ executable: String, + _ args: [String] = [], + env: [String: String] = [:], + context: @escaping @autoclosure () -> String? = nil +) throws -> String { + let quotedArgs = args.map { "'\($0)'" } + + var newEnv = ProcessInfo.processInfo.environment.merging(env) { _, new in new } + newEnv["PWD"] = FileManager.default.currentDirectoryPath + + let process = Process() + process.environment = ProcessInfo.processInfo.environment.merging(env) { _, new in new } + + let command = "\(executable) \(quotedArgs.joined(separator: " "))" + + Logger.simulatorControl.debug(#"🛠️ Running "\#(command, privacy: .public)""#) + + do { + return try shellOut( + to: executable, + arguments: quotedArgs, + process: process + ) + } catch let error as ShellOutError { + throw ProcessError( + command: command, + context: context(), + exitCode: error.terminationStatus, + stdOut: error.output, + stdErr: error.message + ) + } +} + +/// Prefix shared by every simulator this manager creates for cloning (see +/// `SimulatorConfig.cloneDeviceName`). Used to scope the orphan reaper to devices +/// it manages, so it never touches a developer's own simulators or base templates. +let managedCloneNamePrefix = "EXAMPLE_BAZEL_CLONE_" + +extension SimulatorConfig { + func baseDeviceName() -> String { + return "EXAMPLE_BAZEL_BASE_\(deviceType)_\(version)" + } + + func cloneDeviceName(index: Int) -> String { + return "\(managedCloneNamePrefix)\(deviceType)_\(version)_\(index)" + } + + func runtimeIdentifier() -> String { + let runtimeVersion = version.replacingOccurrences(of: ".", with: "-") + return "com.apple.CoreSimulator.SimRuntime.\(os)-\(runtimeVersion)" + } +} diff --git a/#/SimulatorManager.swift b/#/SimulatorManager.swift new file mode 100644 index 0000000..a488b5e --- /dev/null +++ b/#/SimulatorManager.swift @@ -0,0 +1,1058 @@ +import Foundation +import os +import ShellOut + +typealias PID = pid_t + +extension Logger { + static let simulatorManager = simulatorManager(category: "manager") + static let childProcess = simulatorManager(category: "manager.child-process") +} + +enum SimulatorManagerError: Error { + case alreadyLeased(udid: SimulatorUDID) + case noLease + case leaserExited +} + +private struct SimulatorLease { + let udid: SimulatorUDID + let config: SimulatorConfig + let exclusive: Bool + let slotIndex: Int + /// When the leasing process started, captured at lease time. + /// + /// Persisted so a successor daemon can tell a still-running leaser from a + /// recycled PID. Nil when it could not be read. + let leaserStartTime: UInt64? +} + +/// Whether `pid` is still running. +/// +/// `kill(pid, 0)` reports failure for two unrelated reasons, and only one of them +/// means the process is gone: `ESRCH` (no such process) versus `EPERM` (it exists +/// but we may not signal it). Treating `EPERM` as death would release a live +/// leaser's device out from under it, so only `ESRCH` counts. +func processIsRunning(_ pid: PID) -> Bool { + if kill(pid, 0) == 0 { + return true + } + return errno != ESRCH +} + +private enum SimulatorSlot { + case empty + case pendingCreation(Task, exclusive: Bool) + case active(SimulatorUDID, exclusive: Bool) + case pendingDeletion(SimulatorUDID, Task) + case deleting(SimulatorUDID) +} + +extension SimulatorSlot { + var sortOrder: Int { + switch self { + // Try to use an active simulator first (should only be one for non-exclusive) + case .active: + return 0 + + // A pending deletion is already created and booted, so reuse it before waiting + // on a pending creation or starting a fresh one. + case .pendingDeletion: + return 1 + + // Use an empty slot before waiting on a pending creation, so a lease doesn't + // block on someone else's in-flight clone when a fresh slot is free to start. + case .empty: + return 2 + + case .pendingCreation: + return 3 + + // Deleting simulator can't be used, so put it at the end + case .deleting: + return 4 + } + } +} + +private enum SimulatorSlotResult { + case active(SimulatorUDID, slotIndex: Int) + case pending(Task, slotIndex: Int) +} + +actor SimulatorManager { + private let simulatorControl: SimulatorControl + + private var simulatorSlots: [SimulatorConfig: [SimulatorSlot]] = [:] + private var referenceCount: [SimulatorUDID: Int] = [:] + private var leases: [PID: SimulatorLease] = [:] + + private var leaserExitListeners: [PID: DispatchSourceProcess] = [:] + + private var getBaseSimulatorTasks: [SimulatorConfig: Task] = [:] + + private let deleteIdleAfter: UInt16 + private let deleteRecentlyUsedIdleAfter: UInt16 + private let deleteOnPIDExit: Bool + + /// Where leases are mirrored so a successor daemon can adopt them. Nil disables + /// persistence, which is the default in tests that do not care about it. + private let leaseStore: LeaseStore? + + private var recentlyLeased: LRUSet + + private var startupProcessPaths: [String] + private var postBoot: String? + private var childProcessTasks: [Task] = [] + private var childProcesses: [String: (Process, DispatchSourceRead, DispatchSourceRead)] = [:] + + private var reaperTask: Task? + /// Clone UDIDs that looked orphaned on the *previous* sweep. A device must appear + /// unknown on two consecutive sweeps before the reaper deletes it, so a clone that's + /// mid-creation (it exists on disk before `createCloneTask` resumes and records it in + /// `referenceCount`) never gets caught by a single unlucky sweep. + private var previousOrphanCandidates: Set = [] + /// How many sweeps in a row the reaper has failed to delete a confirmed orphan. + /// Reset once a delete succeeds, or once the UDID stops being an orphan at all + /// (claimed by a new lease, or already cleaned up some other way). + private var orphanDeleteFailureCounts: [SimulatorUDID: Int] = [:] + /// After this many consecutive failed deletes, a device is treated as wedged + /// rather than merely unlucky: normal `simctl shutdown`/`delete` retries have + /// already been exhausted inside `SimulatorControl.delete()` every sweep, so more + /// of the same is unlikely to help. Escalate to killing its `launchd_sim` directly. + private let forceKillAfterFailedReapAttempts = 3 + + init( + simulatorControl: SimulatorControl, + deleteRecentlyUsedIdleAfter: UInt16, + deleteIdleAfter: UInt16, + recentlyUsedCapacity: Int, + deleteOnPIDExit: Bool, + startupProcesses: [String] = [], + postBoot: String? = nil, + leaseStore: LeaseStore? = nil + ) { + self.simulatorControl = simulatorControl + self.deleteIdleAfter = deleteIdleAfter + self.deleteRecentlyUsedIdleAfter = deleteRecentlyUsedIdleAfter + self.deleteOnPIDExit = deleteOnPIDExit + self.recentlyLeased = LRUSet(capacity: recentlyUsedCapacity) + self.startupProcessPaths = startupProcesses + self.postBoot = postBoot + self.leaseStore = leaseStore + + // Change the working directory to some place stable, since on RBE the runfiles directory can + // get cleaned up + FileManager.default.changeCurrentDirectoryPath("/tmp") + } + + deinit { + reaperTask?.cancel() + + for task in childProcessTasks { + task.cancel() + } + + for (process, outWatcher, errWatcher) in childProcesses.values { + process.terminate() + outWatcher.cancel() + errWatcher.cancel() + } + } + + func startChildProcesses() throws { + for path in startupProcessPaths { + childProcessTasks.append(createStartChildProcessTask(path: path)) + } + } + + /// Starts the periodic sweep that deletes clone simulators the manager has lost + /// track of. + /// + /// Every other cleanup path is event-driven: an explicit release, a PID-exit + /// watcher, an idle timer, or "rediscovered by name on the next lease of the same + /// config." Each of those can miss a device -- a daemon restart drops a lease + /// whose device is never leased again (see `restoreLeases`), or a `simctl delete` + /// call fails and is swallowed (see `delete`) -- and nothing else ever looks for + /// it again. This sweep is the backstop: it reconciles against what CoreSimulator + /// actually has running, independent of how a device became untracked. + /// + /// `interval <= .zero` disables it, matching how `deleteIdleAfter` of 0 means + /// "immediately" elsewhere in this file rather than "never." + func startReaper(interval: Duration) { + guard interval > .zero else { return } + + reaperTask = Task { + while !Task.isCancelled { + try? await Task.sleep(for: interval) + guard !Task.isCancelled else { break } + await reapOrphanedSimulators() + } + } + } + + /// One sweep of the orphan reaper: list every clone simulator that exists, and + /// delete the ones with no entry in `referenceCount`. + /// + /// `referenceCount` is the right ground truth to check against, not `leases` or + /// `simulatorSlots`: a device gets an entry in it the instant it's claimed (before + /// any `await`, per the invariant on `getSimulator`), and the entry is removed + /// only in `delete()` -- including while the device sits in the idle-timer grace + /// period, where the count is `0` but the key stays. So "no entry" reliably means + /// "the manager has no idea this exists," not merely "nothing is leasing it right + /// now." + /// + /// Bypasses `delete()` deliberately: that function updates a slot in + /// `simulatorSlots`, but an orphan by definition has no slot pointing at it, so + /// there is nothing there to update. This calls `simulatorControl` directly, the + /// same lower-level operation `delete()` itself wraps. + private func reapOrphanedSimulators() async { + let known = Set(referenceCount.keys) + + let managedClones: [SimCtlDevice] + do { + managedClones = try await simulatorControl.listManagedClones() + } catch { + Logger.simulatorManager.error( + "❌ Orphan reaper failed to list simulators, skipping this sweep: \(error, privacy: .public)" + ) + return + } + + let currentOrphanCandidates = Set(managedClones.map(\.udid)).subtracting(known) + let confirmedOrphans = currentOrphanCandidates.intersection(previousOrphanCandidates) + previousOrphanCandidates = currentOrphanCandidates + + // Forget the failure count for anything that isn't a confirmed orphan any more + // (claimed by a new lease, or already cleaned up), so a UDID that's reused later + // starts with a clean slate rather than inheriting an old device's history. + orphanDeleteFailureCounts = orphanDeleteFailureCounts.filter { confirmedOrphans.contains($0.key) } + + guard !confirmedOrphans.isEmpty else { return } + + for device in managedClones where confirmedOrphans.contains(device.udid) { + await reapOrphan(device) + } + } + + /// Deletes one confirmed-orphaned device, escalating to a direct kill of its + /// `launchd_sim` if it has already failed to delete + /// `forceKillAfterFailedReapAttempts` sweeps in a row. + /// + /// `SimulatorControl.delete()` already shuts the device down and retries a few + /// times internally before throwing, so a failure reaching here means those + /// retries were exhausted -- consistent with a genuinely wedged device (the same + /// profile as the multi-day-old `launchd_sim` processes that motivated this), not + /// a one-off transient error. Retrying the same call every 5-minute sweep forever + /// would just fail the same way forever, silently; escalating is what makes this a + /// backstop instead of another silent no-op. + private func reapOrphan(_ device: SimCtlDevice) async { + Logger.simulatorManager.warning( + """ + 🧹 Reaping orphaned simulator \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)); the manager has no lease or reference to it + """ + ) + + do { + try await simulatorControl.delete(device.udid, name: device.name, context: "orphan reaper") + orphanDeleteFailureCounts.removeValue(forKey: device.udid) + return + } catch { + let failures = (orphanDeleteFailureCounts[device.udid] ?? 0) + 1 + orphanDeleteFailureCounts[device.udid] = failures + + Logger.simulatorManager.error( + """ + ❌ Orphan reaper failed to delete \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)), attempt \(failures, privacy: .public): \ + \(error, privacy: .public) + """ + ) + + guard failures >= forceKillAfterFailedReapAttempts else { return } + + Logger.simulatorManager.error( + """ + 🔨 \(device.udid, privacy: .public) (\(device.name, privacy: .public)) has failed to \ + delete \(failures, privacy: .public) sweeps in a row; force-killing its launchd_sim + """ + ) + + await simulatorControl.forceKillLaunchdSim(for: device.udid) + + do { + try await simulatorControl.delete( + device.udid, + name: device.name, + context: "orphan reaper, post force-kill" + ) + orphanDeleteFailureCounts.removeValue(forKey: device.udid) + } catch { + Logger.simulatorManager.error( + """ + ❌ \(device.udid, privacy: .public) (\(device.name, privacy: .public)) still failed to \ + delete after force-killing launchd_sim: \(error, privacy: .public). Needs manual cleanup. + """ + ) + } + } + } + + /// Adopts the leases a previous daemon left behind. + /// + /// Leases used to live only in this actor's memory, so replacing the daemon -- + /// which `start.sh` does on any version change, on every lease -- lost the + /// bookkeeping for tests that were still running. Their release calls then + /// reached a daemon that had never heard of them and reported "doesn't have a + /// simulator leased". + /// + /// Restoring rebuilds enough state for `release` to work: the lease itself, the + /// slot holding the device, a reference count, and an exit listener so a leaser + /// that dies during the handover is still cleaned up. + /// + /// Leases whose process is gone are dropped rather than adopted. The device they + /// held is left alone: `createBase`/`clone` find existing devices by name, so it + /// gets picked up and reference counted by the next lease of that config. + func restoreLeases() { + guard let leaseStore else { return } + + let persisted = leaseStore.load() + guard !persisted.isEmpty else { return } + + var adopted = 0 + var dropped = 0 + + for lease in persisted { + guard leaserSurvived(lease) else { + dropped += 1 + continue + } + + // Two leases cannot share an exclusive device, and a slot cannot hold two + // different devices. A file that says otherwise is inconsistent -- possibly + // hand-edited, or written by a version with different slot semantics -- so + // prefer dropping the lease over corrupting the slot bookkeeping. + guard canAdopt(lease) else { + Logger.simulatorManager.error( + """ + ❌ Not restoring conflicting lease for PID \(lease.pid, privacy: .public) \ + on \(lease.udid, privacy: .public) + """ + ) + dropped += 1 + continue + } + + leases[lease.pid] = .init( + udid: lease.udid, + config: lease.config, + exclusive: lease.exclusive, + slotIndex: lease.slotIndex, + leaserStartTime: lease.leaserStartTime + ) + + // Recreate the slot the device occupies, so it is neither handed to an + // incompatible lease nor deleted as idle while its owner is still running. + var slots = simulatorSlots[lease.config] ?? [] + while slots.count <= lease.slotIndex { + slots.append(.empty) + } + slots[lease.slotIndex] = .active(lease.udid, exclusive: lease.exclusive) + simulatorSlots[lease.config] = slots + + incrementReferenceCount(for: lease.udid) + + if deleteOnPIDExit { + registerReleaseOnExit(for: lease.pid) + } + + adopted += 1 + } + + Logger.simulatorManager.info( + """ + ♻️ Restored \(adopted, privacy: .public) lease(s) from a previous simulator \ + manager; dropped \(dropped, privacy: .public) whose process had exited + """ + ) + + // The dropped entries are gone for good, so rewrite the file rather than let a + // later crash re-adopt them. + persistLeases() + } + + /// Whether the process that took `lease` is the one still running under that + /// PID, rather than an unrelated process that reused the number. + private func leaserSurvived(_ lease: PersistedLease) -> Bool { + guard processIsRunning(lease.pid) else { return false } + + // A record without a start time predates that field. Fall back to liveness + // alone: adopting on a PID collision costs one device held until that process + // exits, which is better than dropping every lease on a mixed-version upgrade. + guard let persistedStart = lease.leaserStartTime else { return true } + + guard let currentStart = processStartTime(lease.pid) else { return false } + + guard currentStart == persistedStart else { + Logger.simulatorManager.info( + """ + 👻 PID \(lease.pid, privacy: .public) is running but started at a different \ + time than the lease recorded; treating the leaser as exited + """ + ) + return false + } + + return true + } + + /// Whether `lease` can be adopted without contradicting one already restored. + private func canAdopt(_ lease: PersistedLease) -> Bool { + for existing in leases.values { + if existing.udid == lease.udid, existing.exclusive || lease.exclusive { + return false + } + + if existing.config == lease.config, + existing.slotIndex == lease.slotIndex, + existing.udid != lease.udid { + return false + } + } + + return true + } + + /// Mirrors the current leases to disk. + /// + /// Called on every change rather than at shutdown, because the daemon is not + /// always shut down politely: `start.sh` escalates to `kill -9`, which runs no + /// cleanup. Each lease costs a small JSON write, against provisioning a + /// simulator that takes seconds. + private func persistLeases() { + guard let leaseStore else { return } + + leaseStore.save( + leases.map { pid, lease in + .init( + pid: pid, + leaserStartTime: lease.leaserStartTime, + udid: lease.udid, + config: lease.config, + exclusive: lease.exclusive, + slotIndex: lease.slotIndex + ) + } + ) + } + + func lease( + to leaser: PID, + exclusive: Bool, + config: SimulatorConfig + ) async throws -> SimulatorUDID { + // Each process can only lease one simulator at a time + if let existingLease = leases[leaser] { + throw SimulatorManagerError.alreadyLeased(udid: existingLease.udid) + } + + Logger.simulatorManager.info( + """ + 🔒 Leasing \(exclusive ? "exclusive" : "non-exclusive", privacy: .public) \ + \(config, privacy: .public) simulator for PID \(leaser, privacy: .public) + """ + ) + + // Check liveness before provisioning rather than after. `getSimulator()` can + // await for minutes while it clones, boots and runs the post-boot script, and + // a leaser that dies during that window (e.g. killed by its build tool's test + // timeout while queued for a simulator) used to be detected only afterwards -- + // releasing the lease a millisecond after granting it, and deleting the device + // out from under a test that had already been handed the UDID. + // + // Only meaningful when we track leaser exit at all; otherwise the caller owns + // the lease lifetime and the PID need not be a live process. + if deleteOnPIDExit, !processIsRunning(leaser) { + Logger.simulatorManager.info( + "👋 PID \(leaser, privacy: .public) exited before its lease could be provisioned" + ) + throw SimulatorManagerError.leaserExited + } + + // `getSimulator()` will increment the reference count for the simulator + let (simulator, slotIndex) = try await getSimulator(for: config, exclusive: exclusive) + + _ = recentlyLeased.insert(config) + + // Re-check now that provisioning is done: the leaser may have exited while we + // were cloning and booting. Record the lease first so `release` can unwind the + // reference count and put the device back in the idle pool, then hand it back + // rather than returning a UDID nobody will use. + leases[leaser] = .init( + udid: simulator, + config: config, + exclusive: exclusive, + slotIndex: slotIndex, + leaserStartTime: processStartTime(leaser) + ) + persistLeases() + + if deleteOnPIDExit, !processIsRunning(leaser) { + Logger.simulatorManager.info( + """ + 👋 PID \(leaser, privacy: .public) exited while its simulator was being \ + provisioned; returning \(simulator, privacy: .public) to the pool + """ + ) + try await release(for: leaser) + throw SimulatorManagerError.leaserExited + } + + Logger.simulatorManager.info( + "🔒 Leased simulator \(simulator, privacy: .public) to PID \(leaser, privacy: .public)" + ) + + if deleteOnPIDExit { + registerReleaseOnExit(for: leaser) + } + + return simulator + } + + /// The number of leases whose leasing process is still running. + /// + /// Exposed over HTTP for diagnostics -- "was anything actually leased when this + /// daemon was replaced?" is otherwise hard to answer after the fact. Leases held + /// by exited processes are excluded, since those are already reclaimed or about + /// to be. + func liveLeaseCount() -> Int { + return leases.keys.count(where: { processIsRunning($0) }) + } + + func release(for leaser: PID) async throws { + guard let lease = leases.removeValue(forKey: leaser) else { + // If the manager recently restarted, we might not have the state of all leases. Since + // `SimulatorControl` will return existing simulators matching a given name, the dangling + // simulator will eventually get picked back up again and properly reference counted. So we + // will return an error here, and ignore it in the test runner. + throw SimulatorManagerError.noLease + } + + Logger.simulatorManager.info( + "🔓 Releasing simulator \(lease.udid, privacy: .public) for PID \(leaser, privacy: .public)" + ) + + // Recorded before the device is torn down, so a daemon replaced mid-release + // does not adopt a lease whose simulator is already going away. + persistLeases() + + removeReleaseOnExit(for: leaser) + + await simulatorControl.cleanTempFiles(in: lease.udid) + + try await decrementReferenceCount( + for: lease.udid, + config: lease.config, + slotIndex: lease.slotIndex + ) + } + + private func getBase( + for config: SimulatorConfig + ) async throws -> SimulatorUDID { + if let existingTask = getBaseSimulatorTasks[config] { + return try await existingTask.value + } + + // We use a task to prevent data races that can occur when the `await` on `simulatorControl` + // blocks. This ensures that multiple callers trying to get a base simulator will all wait + // for the same simulator to be returned. + let task = Task { + defer { + getBaseSimulatorTasks.removeValue(forKey: config) + } + + Logger.simulatorManager.info("📱 Creating \(config, privacy: .public) base simulator") + + let baseSimulator = + try await simulatorControl + .createBase( + name: config.baseDeviceName(), + with: config, + runtimeIdentifier: config.runtimeIdentifier() + ) + + Logger.simulatorManager.info( + "📱 Created \(config, privacy: .public) base simulator: \(baseSimulator, privacy: .public)" + ) + + return baseSimulator + } + + getBaseSimulatorTasks[config] = task + + return try await task.value + } + + private func incrementReferenceCount(for simulator: SimulatorUDID) { + var count = referenceCount[simulator] ?? 0 + count += 1 + referenceCount[simulator] = count + + Logger.simulatorManager.debug( + """ + 🔼 Reference count for simulator \(simulator, privacy: .public) is now \ + \(count, privacy: .public) + """ + ) + } + + private func decrementReferenceCount( + for simulator: SimulatorUDID, + config: SimulatorConfig, + slotIndex: Int + ) async throws { + guard var count = referenceCount[simulator] else { + // Simulator was already deleted, nothing to do + return + } + + count -= 1 + referenceCount[simulator] = count + + Logger.simulatorManager.debug( + "🔽 Reference count for \(simulator, privacy: .public) is now \(count, privacy: .public)" + ) + + guard count == 0 else { + return + } + + // Wait a bit before deleting simulators, to allow them to be reused + await pendingDeletion(simulator, config: config, slotIndex: slotIndex) + } + + // Warning: We must update slots before we `await` on anything in this function (unless that + // method updates slots before `await`ing on anything). + private func getSimulator( + for config: SimulatorConfig, + exclusive: Bool + ) async throws -> (simulator: SimulatorUDID, slotIndex: Int) { + if simulatorSlots.keys.contains(config) == false { + simulatorSlots[config] = [] + } + + // Need to sort so we reuse the the correct slots + let sortedSlots = simulatorSlots[config]!.enumerated().sorted { lhs, rhs in + let lhsSortOrder = lhs.element.sortOrder + let rhsSortOrder = rhs.element.sortOrder + + guard lhsSortOrder == rhsSortOrder else { + // Sort by sort order first + return lhsSortOrder < rhsSortOrder + } + + // If the sort order is the same, sort by index + return lhs.offset < rhs.offset + } + + for (index, slot) in sortedSlots { + switch slot { + case .active(let simulator, false) where exclusive != true: + // We have an active non-exclusive simulator, so reuse it + return try await ( + reuseSimulator(simulator, config: config, exclusive: exclusive, slotIndex: index), + slotIndex: index + ) + + case .pendingDeletion(let simulator, let task): + // We have a pending deletion, so we can reuse it + Logger.simulatorManager.info( + """ + ♻️ Turning a pending deletion of simulator \(simulator, privacy: .public) into an \ + active \(exclusive ? "exclusive" : "non-exclusive", privacy: .public) simulator + """ + ) + + simulatorSlots[config]![index] = .active(simulator, exclusive: exclusive) + + task.cancel() + + return try await ( + reuseSimulator(simulator, config: config, exclusive: exclusive, slotIndex: index), + slotIndex: index + ) + + case .empty: + let task = createCloneTask(config: config, exclusive: exclusive, slotIndex: index) + simulatorSlots[config]![index] = .pendingCreation(task, exclusive: exclusive) + return try await (task.value, slotIndex: index) + + case .pendingCreation(let task, false) where exclusive == false: + // We have a non-exclusive simulator pending creation, so reuse it + let simulator = try await task.value + + // We call `incrementReferenceCount()` instead of `reuseSimulator()` here, because the + // simulator is freshly created, so we can (hopefully) assume it is in a good state + incrementReferenceCount(for: simulator) + + return (simulator, slotIndex: index) + + default: + // Ignore incompatible slots + break + } + } + + // If we got here, we need to add a new slot + let index = simulatorSlots[config]!.count + let task = createCloneTask(config: config, exclusive: exclusive, slotIndex: index) + simulatorSlots[config]!.append(.pendingCreation(task, exclusive: exclusive)) + return try await (task.value, slotIndex: index) + } + + private func reuseSimulator( + _ simulator: SimulatorUDID, + config: SimulatorConfig, + exclusive: Bool, + slotIndex: Int + ) async throws -> SimulatorUDID { + incrementReferenceCount(for: simulator) + + do { + // Wait for it to boot. This shouldn't be necessary, but sometimes the simulator will + // reboot because of a migration. This also guards against a simulator being deleted out + // from under us, as it will error, and we can then "delete" it and return a new one. + try await simulatorControl.ensureBooted( + simulator, + context: "getSimulator, reused: \(config.cloneDeviceName(index: slotIndex))" + ) + + return simulator + } catch let error as ProcessError { + // 148 happens for "Invalid device". So it either has already been deleted or it's corrupt + // in some way. Either way, we will "delete" it and return a new one. + guard error.exitCode == 148 else { + throw error + } + + Logger.simulatorManager.warning( + """ + ⚠️ Boot of existing simulator \(simulator, privacy: .public) failed; deleting and \ + returning a new simulator: \(error, privacy: .public) + """ + ) + + // If we fail to delete, don't throw an error + try? await delete( + simulator, + config: config, + slotIndex: slotIndex, + // We can't clean up slots, since we assign to it below + cleanUpSlots: false, + context: "getSimulator, reused: \(config.cloneDeviceName(index: slotIndex))" + ) + + let task = createCloneTask(config: config, exclusive: exclusive, slotIndex: slotIndex) + simulatorSlots[config]![slotIndex] = .pendingCreation(task, exclusive: exclusive) + return try await task.value + } + } + + private func createCloneTask( + config: SimulatorConfig, + exclusive: Bool, + slotIndex: Int + ) -> Task { + return Task { + do { + let simulator = try await simulatorControl.clone( + getBase(for: config), + name: config.cloneDeviceName(index: slotIndex), + deviceType: config.deviceType, + runtimeIdentifier: config.runtimeIdentifier(), + postBoot: postBoot + ) + + simulatorSlots[config]![slotIndex] = .active(simulator, exclusive: exclusive) + + // We want to increment the reference count as soon as we get back from `await`, to ensure + // that when we suspend and potentially decrement the reference count, we don't delete the + // simulator before we have a chance to use it. Also, since we created the simulator, we + // should be responsible for incrementing the reference count. Any functions that reuse + // this task need to increment the reference count as well. + incrementReferenceCount(for: simulator) + + return simulator + } catch { + // If we fail to create the clone, we need to empty the slot, instead + // of leaving it in a pending state + simulatorSlots[config]![slotIndex] = .empty + + throw error + } + } + } + + private func registerReleaseOnExit(for leaser: PID) { + let processSource = + DispatchSource.makeProcessSource(identifier: leaser, eventMask: .exit, queue: .main) + + // Avoid double handling of exit in case the process exits between + // `processSource.resume()` and the check with `kill` below. The event handler runs + // on `.main`, while the liveness check below runs on whatever thread calls this + // method, so the flag guarding against a double call must itself be synchronized. + let handledExitLock = NSLock() + var handledExit = false + let onExitHandler: () -> Void = { [weak self] in + handledExitLock.lock() + let alreadyHandled = handledExit + handledExit = true + handledExitLock.unlock() + guard !alreadyHandled else { return } + + Task { + guard let self else { return } + + Logger.simulatorManager.debug("👋 PID \(leaser, privacy: .public) exited") + + try await self.release(for: leaser) + } + } + + processSource.setEventHandler { onExitHandler() } + processSource.resume() + + // Check to see if the process is already dead. There is no `setCancelHandler`, so + // handle the exit directly here rather than relying on cancellation to do it. + guard processIsRunning(leaser) else { + processSource.cancel() + onExitHandler() + return + } + + leaserExitListeners[leaser] = processSource + } + + private func removeReleaseOnExit(for leaser: PID) { + guard let leaserExitListener = leaserExitListeners[leaser] else { return } + leaserExitListeners.removeValue(forKey: leaser) + leaserExitListener.cancel() + } + + private func pendingDeletion( + _ simulator: SimulatorUDID, + config: SimulatorConfig, + slotIndex: Int + ) async { + guard deleteIdleAfter > 0 || deleteRecentlyUsedIdleAfter > 0 else { + // If we fail to delete, don't throw an error + try? await delete( + simulator, + config: config, + slotIndex: slotIndex, + cleanUpSlots: true, + context: "pendingDeletion immediate" + ) + return + } + + let task = Task { + Logger.simulatorManager.info( + """ + 💤 Scheduling delete of simulator \(simulator, privacy: .public) in \ + \(self.deleteIdleAfter, privacy: .public) to \ + \(self.deleteRecentlyUsedIdleAfter, privacy: .public) seconds + """ + ) + + let now = Date() + let shortDeadline = now.addingTimeInterval(TimeInterval(deleteIdleAfter)) + let recentlyUsedDeadline = now.addingTimeInterval(TimeInterval(deleteRecentlyUsedIdleAfter)) + + while true { + let remainingTime: TimeInterval + if recentlyLeased.contains(config) { + remainingTime = recentlyUsedDeadline.timeIntervalSinceNow + } else { + remainingTime = shortDeadline.timeIntervalSinceNow + } + + if remainingTime <= 0 { + break + } + + // Sleep for up-to 1 second before next check + try await Task.sleep(for: .seconds(min(remainingTime, 1))) + } + + guard case .pendingDeletion(let slotSimulator, _) = simulatorSlots[config]![slotIndex], + simulator == slotSimulator else { + // Simulator was reused, no need to delete + return + } + + // If we fail to delete, don't throw an error + try? await delete( + simulator, + config: config, + slotIndex: slotIndex, + cleanUpSlots: true, + context: "pendingDeletion delayed" + ) + } + + simulatorSlots[config]![slotIndex] = .pendingDeletion(simulator, task) + } + + private func delete( + _ simulator: SimulatorUDID, + config: SimulatorConfig, + slotIndex: Int, + cleanUpSlots: Bool, + context: @escaping @autoclosure () -> String? + ) async throws { + let name = config.cloneDeviceName(index: slotIndex) + + Logger.simulatorManager.info( + "🗑️ Deleting simulator \(simulator, privacy: .public) (\(name, privacy: .public))" + ) + + simulatorSlots[config]![slotIndex] = .deleting(simulator) + + referenceCount.removeValue(forKey: simulator) + + defer { + // Even if we fail to delete, we need to set the slot to empty + simulatorSlots[config]![slotIndex] = .empty + + if cleanUpSlots { + // Shorten up the array by removing any empty slots at the end + while case .empty = simulatorSlots[config]!.last { + simulatorSlots[config]!.removeLast() + } + } + } + + try await simulatorControl.delete(simulator, name: name, context: context()) + + Logger.simulatorManager.info( + "🗑️ Deleted simulator \(simulator, privacy: .public) (\(name, privacy: .public)" + ) + } + + // MARK: Child Process Management + + private nonisolated func createStartChildProcessTask(path: String) -> Task { + return Task.detached { [weak self] in + let process: Process + do { + guard let self else { return } + process = try await self.createProcess(path: path) + // `self` drops out of scope here, so `SimulatorManager` can deinit + } catch { + Logger.simulatorManager.info( + """ + ❌ Failed to create child process at "\(path, privacy: .public)": \ + \(error, privacy: .public) + """ + ) + return + } + + await withCheckedContinuation { cont in + process.terminationHandler = { proc in + let exitCode = proc.terminationStatus + Logger.simulatorManager.warning( + """ + ⚠️ "\(path, privacy: .public)" exited with code: \(exitCode, privacy: .public) + """ + ) + cont.resume() + } + + do { + Logger.simulatorManager.info( + #"🧒 Starting "\#(path, privacy: .public)""# + ) + try process.run() + } catch { + Logger.simulatorManager.info( + """ + ❌ Failed to start "\(path, privacy: .public)": \ + \(error, privacy: .public) + """ + ) + cont.resume() + } + } + } + } + + private func createProcess(path: String) throws -> Process { + let process = Process() + + process.executableURL = URL(fileURLWithPath: path) + + let outPTY = try PTY() + process.standardOutput = FileHandle(fileDescriptor: outPTY.child, closeOnDealloc: true) + let outQueue = DispatchQueue(label: "com.example.simulator_manager.child_process.out") + let outWatcher = watch(fd: outPTY.parent, queue: outQueue) { line in + Logger.childProcess.info("[\(path, privacy: .public)] \(line, privacy: .public)") + } + + let errPTY = try PTY() + process.standardError = FileHandle(fileDescriptor: errPTY.child, closeOnDealloc: true) + let errQueue = DispatchQueue(label: "com.example.simulator_manager.child_process.err") + let errWatcher = watch(fd: errPTY.parent, queue: errQueue) { line in + Logger.childProcess.error("[\(path, privacy: .public)] \(line, privacy: .public)") + } + + childProcesses[path] = (process, outWatcher, errWatcher) + + return process + } +} + +/// Installs a DispatchSourceRead on `fd`. +private func watch( + fd: Int32, + queue: DispatchQueue, + onLine: @escaping (String) -> Void +) -> DispatchSourceRead { + let src = DispatchSource.makeReadSource(fileDescriptor: fd, queue: queue) + var buffer = Data() + + src.setEventHandler { + var tmp = [UInt8](repeating: 0, count: 4096) + let n = read(fd, &tmp, tmp.count) + guard n > 0 else { + src.cancel() + close(fd) + return + } + + buffer.append(contentsOf: tmp[0..?) { + let response = unwrapOutboundIn(data) + + write(context: context, response: response) + } + + private func write(context: ChannelHandlerContext, response: SimulatorManagerResponse) { + let message = response.message + "\n" + + context.write( + wrapOutboundOut( + .init( + head: .init( + version: .http1_1, + status: response.status, + headers: .defaultHeaders(for: message) + ), + body: context.channel.allocator.buffer(string: message) + ) + ), + promise: nil + ) + } +} + +extension HTTPHeaders { + static func defaultHeaders(for message: String) -> HTTPHeaders { + var headers = HTTPHeaders() + headers.add(name: "Content-Length", value: "\(message.utf8.count)") + headers.add(name: "Content-Type", value: "text/plain") + return headers + } +} diff --git a/#/SimulatorRequestHandler.swift b/#/SimulatorRequestHandler.swift new file mode 100644 index 0000000..82ada11 --- /dev/null +++ b/#/SimulatorRequestHandler.swift @@ -0,0 +1,104 @@ +import NIOHTTP1 + +final class SimulatorRequestHandler { + private let simulatorManager: SimulatorManager + + init(simulatorManager: SimulatorManager) { + self.simulatorManager = simulatorManager + } + + /// The number of leases still held by running processes, for diagnostics. + func liveLeaseCount() async -> Int { + return await simulatorManager.liveLeaseCount() + } + + func handleRequest( + method: HTTPMethod, + pathComponents: [String], + queryParameters: [String: String] + ) async throws -> SimulatorManagerResponse { + switch method { + case .POST: + guard pathComponents.count >= 1 else { + return .init( + status: .badRequest, + message: "Must specify " + ) + } + + guard let leaser = PID(pathComponents[0]) else { + return .init(status: .badRequest, message: "Leaser PID must be an integer") + } + guard let exclusiveString = queryParameters["exclusive"] else { + return .init(status: .badRequest, message: "Must specify 'exclusive' query parameter") + } + let exclusive = exclusiveString == "1" + guard let deviceType = queryParameters["deviceType"] else { + return .init(status: .badRequest, message: "Must specify 'deviceType' query parameter") + } + guard let os = queryParameters["os"] else { + return .init(status: .badRequest, message: "Must specify 'os' query parameter") + } + guard let version = queryParameters["version"] else { + return .init(status: .badRequest, message: "Must specify 'version' query parameter") + } + + let config = SimulatorConfig( + deviceType: deviceType, + os: os, + version: version + ) + + do { + return try await .init( + status: .created, + message: simulatorManager + .lease(to: leaser, exclusive: exclusive, config: config) + ) + } catch SimulatorManagerError.alreadyLeased(let udid) { + return .init( + status: .badRequest, + // FIXME: Get this from the error itself + message: "PID \(leaser) has already leased another simulator: \(udid)" + ) + } catch SimulatorManagerError.leaserExited { + // Nobody is left to read this response, but answering rather than throwing + // keeps it out of the server's error path. + return .init( + status: .gone, + message: "PID \(leaser) exited before its simulator was provisioned" + ) + } + + case .DELETE: + guard pathComponents.count >= 1 else { + return .init( + status: .badRequest, + message: "Must specify " + ) + } + + guard let leaser = PID(pathComponents[0]) else { + return .init(status: .badRequest, message: "Leaser PID must be an integer") + } + + do { + try await simulatorManager.release(for: leaser) + + return .init( + status: .ok, + message: "Success" + ) + } catch SimulatorManagerError.noLease { + return .init( + status: .notFound, + // FIXME: Get this message from the error itself + message: "PID \(leaser) doesn't have a simulator leased" + ) + } + + default: + return .init(status: .methodNotAllowed, message: "Unsupported HTTP method: \(method)") + } + } +} From 268c49b984c69066479665b859512d9bd8cb3173 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 21:25:03 +0200 Subject: [PATCH 07/11] more --- .../Sources/Package.resolved | 176 ++++++++++++++++++ tools/simulator_manager/Sources/Package.swift | 16 ++ 2 files changed, 192 insertions(+) create mode 100644 tools/simulator_manager/Sources/Package.resolved create mode 100644 tools/simulator_manager/Sources/Package.swift diff --git a/tools/simulator_manager/Sources/Package.resolved b/tools/simulator_manager/Sources/Package.resolved new file mode 100644 index 0000000..0691b04 --- /dev/null +++ b/tools/simulator_manager/Sources/Package.resolved @@ -0,0 +1,176 @@ +{ + "pins" : [ + { + "identity" : "shellout", + "kind" : "remoteSourceControl", + "location" : "https://github.com/JohnSundell/ShellOut", + "state" : { + "revision" : "e1577acf2b6e90086d01a6d5e2b8efdaae033568", + "version" : "2.3.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", + "version" : "1.8.2" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", + "version" : "1.19.4" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", + "version" : "4.5.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", + "version" : "1.7.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "db774a277f60063a32d854f2980299caf06da041", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", + "version" : "2.101.3" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras", + "state" : { + "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", + "version" : "1.34.3" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", + "version" : "1.45.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", + "version" : "2.37.2" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", + "version" : "2.12.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", + "version" : "1.8.1" + } + } + ], + "version" : 2 +} diff --git a/tools/simulator_manager/Sources/Package.swift b/tools/simulator_manager/Sources/Package.swift new file mode 100644 index 0000000..e2830e1 --- /dev/null +++ b/tools/simulator_manager/Sources/Package.swift @@ -0,0 +1,16 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +// This manifest exists only so `rules_swift_package_manager` can resolve and vendor +// this daemon's third-party dependencies for Bazel (see the BUILD file in this +// directory). It declares no targets/products of its own -- the daemon is built as +// a `swift_binary` directly from these sources, not as a Swift package. +let package = Package( + name: "macsimulatormanager", + dependencies: [ + .package(url: "https://github.com/JohnSundell/ShellOut", from: "2.3.0"), + .package(url: "https://github.com/apple/swift-argument-parser", from: "1.8.2"), + .package(url: "https://github.com/apple/swift-nio-extras", from: "1.34.3"), + ] +) From 947b5968ade7bbaac41c5a6c9b385777815d8a1d Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 20 Aug 2026 21:25:30 +0200 Subject: [PATCH 08/11] more --- #/AccumulatedHTTPHandler.swift | 80 -- #/HTTPServer.swift | 170 ----- #/LEASE_LIFECYCLE.md | 279 ------- #/LRUSet.swift | 49 -- #/LeaseStore.swift | 112 --- #/Logger.swift | 10 - #/Main.swift | 105 --- #/ORPHAN_SIMULATOR_FIX.md | 195 ----- #/PTY.swift | 38 - #/Package.resolved | 176 ----- #/Package.swift | 16 - #/README.md | 141 ---- #/SimulatorControl.swift | 746 ------------------- #/SimulatorManager.swift | 1058 --------------------------- #/SimulatorManagerHTTPHandler.swift | 92 --- #/SimulatorRequestHandler.swift | 104 --- 16 files changed, 3371 deletions(-) delete mode 100644 #/AccumulatedHTTPHandler.swift delete mode 100644 #/HTTPServer.swift delete mode 100644 #/LEASE_LIFECYCLE.md delete mode 100644 #/LRUSet.swift delete mode 100644 #/LeaseStore.swift delete mode 100644 #/Logger.swift delete mode 100644 #/Main.swift delete mode 100644 #/ORPHAN_SIMULATOR_FIX.md delete mode 100644 #/PTY.swift delete mode 100644 #/Package.resolved delete mode 100644 #/Package.swift delete mode 100644 #/README.md delete mode 100644 #/SimulatorControl.swift delete mode 100644 #/SimulatorManager.swift delete mode 100644 #/SimulatorManagerHTTPHandler.swift delete mode 100644 #/SimulatorRequestHandler.swift diff --git a/#/AccumulatedHTTPHandler.swift b/#/AccumulatedHTTPHandler.swift deleted file mode 100644 index 0595e09..0000000 --- a/#/AccumulatedHTTPHandler.swift +++ /dev/null @@ -1,80 +0,0 @@ -import NIO -import NIOHTTP1 -import os.log - -extension Logger { - static let accumulatedHTTP = simulatorManager(category: "accumulated_http") -} - -struct FullHTTPRequest { - let head: HTTPRequestHead - var body: ByteBuffer -} - -struct FullHTTPResponse { - let head: HTTPResponseHead - var body: ByteBuffer -} - -final class AccumulatedHTTPHandler: ChannelInboundHandler, ChannelOutboundHandler { - typealias InboundIn = HTTPServerRequestPart - typealias InboundOut = FullHTTPRequest - - typealias OutboundIn = FullHTTPResponse - typealias OutboundOut = HTTPServerResponsePart - - private var requestHead: HTTPRequestHead? - private var bodyBuffer: ByteBuffer? - - func channelRead(context: ChannelHandlerContext, data: NIOAny) { - let part = self.unwrapInboundIn(data) - - switch part { - case .head(let head): - self.requestHead = head - self.bodyBuffer = context.channel.allocator.buffer(capacity: 0) - - case .body(var chunk): - self.bodyBuffer?.writeBuffer(&chunk) - - case .end: - if let head = requestHead, let body = bodyBuffer { - Logger.accumulatedHTTP.info( - """ - ▶️ Received \(head.method.rawValue, privacy: .public) request for \ - \(head.uri, privacy: .public) - """ - ) - - let fullRequest = FullHTTPRequest(head: head, body: body) - context.fireChannelRead(self.wrapInboundOut(fullRequest)) - } - - self.requestHead = nil - self.bodyBuffer = nil - } - } - - func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise?) { - let fullResponse = unwrapOutboundIn(data) - - Logger.accumulatedHTTP.info( - "◀️ Sending \(fullResponse.head.status, privacy: .public) response" - ) - - context.write(wrapOutboundOut(.head(fullResponse.head)), promise: nil) - - if fullResponse.body.readableBytes > 0 { - context.write(wrapOutboundOut(.body(.byteBuffer(fullResponse.body))), promise: nil) - } - - context.write(wrapOutboundOut(.end(nil)), promise: promise) - } - - func errorCaught(context: ChannelHandlerContext, error: Error) { - Logger.accumulatedHTTP.error( - "❌ \(error.localizedDescription, privacy: .public)" - ) - context.close(promise: nil) - } -} diff --git a/#/HTTPServer.swift b/#/HTTPServer.swift deleted file mode 100644 index ad14e58..0000000 --- a/#/HTTPServer.swift +++ /dev/null @@ -1,170 +0,0 @@ -import Foundation -import NIO -import NIOExtras -import NIOHTTP1 -import NIOPosix -import os.log - -extension Logger { - static let httpServer = simulatorManager(category: "server") -} - -final class HTTPServer { - private let simulatorRequestHandler: SimulatorRequestHandler - - private let version: String - - private var serverShutdownHandler: (() -> Void)? - - init(simulatorRequestHandler: SimulatorRequestHandler, version: String) { - self.simulatorRequestHandler = simulatorRequestHandler - self.version = version - } - - func run(pidPath: String, unixSocketPath: String) async throws { - let socketURL = URL(fileURLWithPath: unixSocketPath) - let pidURL = URL(fileURLWithPath: pidPath) - - // Remove existing files if they exist - try? FileManager.default.removeItem(at: socketURL) - try? FileManager.default.removeItem(at: pidURL) - - try String(ProcessInfo.processInfo.processIdentifier).write( - to: pidURL, - atomically: true, - encoding: .utf8 - ) - - let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) - - do { - // This nested block is necessary to ensure that all the destructors for objects defined - // inside are called before the final call to `eventLoopGroup.syncShutdownGracefully()`. A - // possible side effect of not doing this is a run-time error "Cannot schedule tasks on an - // EventLoop that has already shut down". - let quiesce = ServerQuiescingHelper(group: eventLoopGroup) - let fullyShutdownPromise: EventLoopPromise = eventLoopGroup.next().makePromise() - serverShutdownHandler = { - Logger.httpServer.info("⚠️ Shutting down server") - quiesce.initiateShutdown(promise: fullyShutdownPromise) - } - - do { - let serverChannel = try await ServerBootstrap(group: eventLoopGroup) - .serverChannelOption(ChannelOptions.backlog, value: 256) - .serverChannelInitializer { channel in - return channel.eventLoop.makeCompletedFuture { - try channel.pipeline.syncOperations.addHandler( - quiesce.makeServerChannelHandler(channel: channel) - ) - } - } - .bind(unixDomainSocketPath: unixSocketPath, childChannelInitializer: { childChannel in - return childChannel.eventLoop.makeCompletedFuture { - try childChannel.pipeline.syncOperations.addHandlers([ - HTTPResponseEncoder(), - ByteToMessageHandler(HTTPRequestDecoder()), - AccumulatedHTTPHandler(), - SimulatorManagerHTTPHandler(), - ]) - - return try NIOAsyncChannel( - wrappingChannelSynchronously: childChannel, - configuration: .init() - ) - } - }) - - Logger.httpServer.info("🔌 Server running on UDS at \(unixSocketPath, privacy: .public)") - - try await withThrowingDiscardingTaskGroup { group in - try await serverChannel.executeThenClose { inbound in - for try await connectionChannel in inbound { - group.addTask { - do { - try await self.handleConnection( - channel: connectionChannel - ) - } catch { - // We don't throw here, as it locks up the whole server - Logger.httpServer.error( - """ - ❌ Caught connection error: \(error, privacy: .public) - """ - ) - } - } - } - } - } - } catch { - Logger.httpServer.error("❌ Caught top-level error: \(error, privacy: .public)") - try await eventLoopGroup.shutdownGracefully() - throw error - } - - try await fullyShutdownPromise.futureResult.get() - } - - try await eventLoopGroup.shutdownGracefully() - Logger.httpServer.info("✅ Server shut down") - - // Cleanup files - try? FileManager.default.removeItem(at: socketURL) - try? FileManager.default.removeItem(at: pidURL) - } - - private func handleConnection( - channel: NIOAsyncChannel - ) async throws { - try await channel.executeThenClose { inbound, outbound in - for try await request in inbound { - try await outbound.write(handleRequest(request)) - } - } - } - - private func handleRequest( - _ request: SimulatorManagerRequest - ) async -> SimulatorManagerResponse { - switch request.path { - case "simulator": - do { - return try await simulatorRequestHandler.handleRequest( - method: request.method, - pathComponents: request.pathComponents, - queryParameters: request.queryParameters - ) - } catch { - Logger.httpServer.error( - "❌ simulatorRequestHandler.handleRequest error: \(error, privacy: .public)" - ) - - return .init( - status: .internalServerError, - message: "Internal server error: \(error)" - ) - } - - case "version": - return .init( - status: .ok, - message: version - ) - - case "leases": - let count = await simulatorRequestHandler.liveLeaseCount() - return .init(status: .ok, message: String(count)) - - case "shutdown": - // Shutting down with leases outstanding is safe: they are mirrored to disk, - // and the replacement daemon adopts the ones whose process is still running. - Logger.httpServer.info("⚠️ Shutdown request received") - serverShutdownHandler?() - return .init(status: .ok, message: "Server shutting down") - - default: - return .init(status: .badRequest, message: "Unknown method: \(request.path)") - } - } -} diff --git a/#/LEASE_LIFECYCLE.md b/#/LEASE_LIFECYCLE.md deleted file mode 100644 index 4747b3a..0000000 --- a/#/LEASE_LIFECYCLE.md +++ /dev/null @@ -1,279 +0,0 @@ -# The lifecycle of a simulator lease - -This is a reasoning walkthrough of everything that happens to one lease, from -`POST /simulator/` to the device eventually being deleted (or handed to -someone else). It's meant to make the state machine in `SimulatorManager.swift` -easy to hold in your head, not to restate the architecture already covered in -`README.md`. - -## The actors - -- **A slot** (`SimulatorSlot`) is one device-shaped bucket for a `SimulatorConfig`. - It's `empty`, `pendingCreation`, `active`, `pendingDeletion`, or `deleting`. -- **A lease** (`SimulatorLease` in memory, `PersistedLease` on disk) is the - daemon's record that PID `p` is holding device `udid` in slot `i`. -- **A reference count** (`referenceCount[udid]`) is how many leases currently - point at that device. Non-exclusive leases for the same config can share one - device, so this can be > 1; exclusive leases never share, so it's always - exactly 1 while active. - -A lease and a slot are related but not the same thing: the slot is about the -*device*, the lease is about *who's holding it*. A non-exclusive device can -outlive any single lease on it (reused by the next leaser), and a lease always -maps to exactly one slot at a time. - -## Phase 1 — request arrives - -`SimulatorRequestHandler` parses the HTTP request and calls -`SimulatorManager.lease(to:exclusive:config:)`. First check, before anything -else: does this PID already have a lease? (`leases[leaser]` in -`SimulatorManager.swift:305`). One process, one simulator — this isn't a -counter, it's a hard rule enforced synchronously (no `await` before the -check), so two concurrent requests from the same PID can't both slip through. - -## Phase 2 — is the leaser even still there? - -Before doing anything expensive, the manager checks `processIsRunning(leaser)` -(only when `deleteOnPIDExit` is set — otherwise the caller owns the PID's -lifetime and it need not be a real process at all, e.g. in tests). This looks -redundant — surely the caller wouldn't ask for a lease if it's already dead — -but the point isn't "is it dead right now," it's "is it dead *before we commit -to a multi-minute operation*." Provisioning is the expensive part; checking -liveness is nearly free. This is the cheap half of a two-part guard; the -expensive half is Phase 4. - -## Phase 3 — finding a device: the slot state machine - -This is the core of `getSimulator()`. Slots for the requested config are -sorted by `SimulatorSlot.sortOrder` and walked in order, taking the first one -that matches. The order (after the fix earlier in this session) is: - -1. **`active`, non-exclusive, and the request is non-exclusive** — reuse it - directly. This is the fast path: no provisioning at all, just - `ensureBooted` as a sanity check. -2. **`pendingDeletion`** — a device that's fully created and booted, just - waiting out its idle timer. Reusing it is just as cheap as case 1; the - *only* reason it wasn't in case 1 is that its reference count already hit - zero once. Grabbing it cancels the scheduled deletion task and reassigns - its exclusivity to whatever the new lease wants — which is fine, since - "pending deletion" means nobody currently holds it, so there's no - conflicting owner to worry about. -3. **`empty`** — nobody's even started building a device for this slot index. - Start a fresh clone. -4. **`pendingCreation`, non-exclusive, and the request is non-exclusive** — - someone else is already cloning a device for this config; wait on their - task and count as a second (or third...) leaser of it once it's ready. - -The interesting design choice is putting `empty` *ahead of* `pendingCreation`. -It means a new non-exclusive request would rather kick off a second, -independent clone than wait behind someone else's in-flight one, even though -the in-flight one might finish sooner. My read: this trades a bit of -redundant provisioning work for not making a request's latency depend on -someone else's request. It also means the pool tends to grow to "however many -concurrent first-requests there were for a config" before it starts -consolidating onto reuse — which matches the doc's framing of `pendingCreation` -sharing as a bonus for latecomers, not the primary mechanism. - -Exclusive requests only ever match case 3 (fresh empty slot) or fall through -to appending a brand new slot — they never share `active`, `pendingCreation`, -or (implicitly, since case 2 always wins first if reached) another lease's -slot. `pendingDeletion` is the one case exclusive requests *do* match, and -that's safe precisely because "pending deletion" implies zero current owners. - -If nothing matches, a new slot is appended at the end of the array — the pool -for a config only grows on demand, never pre-allocated. - -## Phase 4 — provisioning, and when the reference count actually increments - -This part is subtle because of the actor-isolation rule spelled out in a -comment above `getSimulator`: slot state must be updated *before* the first -`await`, or a concurrent call could observe a stale slot. That rule shapes -where reference counting happens too — it can't just happen "whenever," it has -to happen at the exact point where the device becomes claimed, before any -suspension point could let the device get swept into deletion by someone else -finishing first. Three different code paths each own this responsibility: - -- **Reuse (`reuseSimulator`)**: increments immediately, *then* awaits - `ensureBooted`. If boot fails with "invalid device" (exit 148), it deletes - the corpse and clones a replacement — the increment from the original - attempt is moot because `delete()` wipes the reference count entry outright, - and the replacement gets its own fresh increment via `createCloneTask`. -- **Fresh clone (`createCloneTask`)**: increments right after the clone - finishes, before returning, on the theory (per the comment in the code) - that whoever created the device should be the one to count it, and anyone - who later shares that same task must increment separately. -- **Joining an in-flight `pendingCreation`**: increments after awaiting the - shared task's result — this is the "anyone who shares it counts separately" - half of the previous point. - -So for a config with 3 non-exclusive leases sharing one device, there are 3 -separate `incrementReferenceCount` calls across possibly all three code paths, -never a batch increment. That's what makes "decrement to zero → pendingDeletion" -in Phase 6 a safe signal: it really does mean the last leaser let go. - -Once a device is in hand, the lease is recorded in `leases[leaser]` and -persisted to disk (`persistLeases()`) *before* the second liveness check. Then -liveness is checked again — the expensive half of the Phase 2 guard. Cloning -and booting can take a real amount of wall-clock time (comment says -"minutes"), long enough for a test harness's timeout to have killed the leaser -already. If that happened, the manager doesn't hand back a UDID nobody will -ever release — it calls `release(for: leaser)` on itself immediately and -reports `leaserExited`. Recording the lease first (even though it's about to -be released) is what lets `release()` work at all; it operates purely on -`leases[leaser]`, it has no other way to find the device. - -Only after surviving both liveness checks does the manager register a process- -exit watcher (`registerReleaseOnExit`) — no point watching a PID you're not -going to hand anything to. - -## Phase 5 — active - -The device sits in `leases[leaser]` and as `.active(udid, exclusive)` in its -slot, reference-counted. Nothing happens to it on its own; it just waits for -one of the release triggers below. Multiple non-exclusive leases can be -pointing at the same UDID simultaneously, each with their own entry in -`leases`, all decrementing the same shared `referenceCount[udid]` independently -whenever *they* release. - -## Phase 6 — release: three ways in - -1. **Explicit `DELETE /simulator/`** → `SimulatorRequestHandler` → - `SimulatorManager.release(for:)`. -2. **The leasing process exits** without ever calling DELETE — caught by the - `DispatchSourceProcess` registered in Phase 4, which calls `release(for:)` - itself. This is what makes leases safe against a build tool that - `kill -9`s a test that forgot to clean up. -3. **The daemon is replaced and the leaser didn't survive the handover** — - handled entirely in `restoreLeases()` at startup, not in `release()` at all; - see Phase 8. The lease is just never adopted, so no explicit release call - ever happens for it, and the device is later picked up implicitly by name. - -`release()` itself is small and ordered deliberately: - -1. Pop the lease out of `leases` (also the point where a nonexistent lease - surfaces as `SimulatorManagerError.noLease` — expected after a daemon - restart dropped a lease the caller doesn't know is gone). -2. Persist the *now-shorter* lease list — before touching the device, so a - daemon replaced mid-release doesn't try to re-adopt a lease whose device is - about to be torn down. -3. Cancel the exit watcher (no longer relevant). -4. Clean temp files on the device. -5. Decrement the reference count, which is where the real branch happens. - -## Phase 7 — idle, and the reference count reaching zero - -`decrementReferenceCount` only does something interesting when the count hits -zero: it hands off to `pendingDeletion()`. If both idle timers are configured -as zero, deletion is immediate — no grace period at all. Otherwise a task is -scheduled that polls once a second, and *at every tick* re-decides which -deadline applies by checking `recentlyLeased.contains(config)` fresh. That's a -live check, not a snapshot taken when the task started — a config that gets -leased again elsewhere while this device is winding down flips it from the -short deadline to the long one (or the reverse, if it falls out of the -LRU set's capacity) mid-wait. It's a genuinely dynamic decision, not "pick a -deadline once." - -Two ways out of `pendingDeletion`: - -- **Resurrection**: a new lease for the same config finds this slot before the - timer expires (Phase 3, case 2) and turns it back into `active`. The - scheduled task gets cancelled, and because `Task.sleep` throws on - cancellation, the task body simply unwinds without ever calling `delete()`. -- **Timeout**: the deadline is reached, the task double-checks the slot still - holds *this exact device* (guarding against a resurrection racing the - cancellation), and calls `delete()` for real — `simctl delete`, slot to - `.deleting` then `.empty`, and (if `cleanUpSlots: true`) trims trailing empty - slots off the end of the array so the pool doesn't grow unboundedly with - dead slot indices. - -## Phase 8 — surviving a daemon restart - -This is the one part of the lifecycle that doesn't originate from an HTTP -request at all — it happens once, at startup, before the server even binds. -`start.sh` replaces the daemon on every new version, which would otherwise -silently orphan every in-flight lease. The fix is that every lease mutation in -Phases 4, 6, and 7 already persists the *entire* current lease set to disk -(atomically), so `restoreLeases()` just has to read that file back and decide, -per lease, whether to trust it: - -1. **Did the leasing process survive?** Not just "is this PID alive" — PIDs - recycle, so it also compares the process's actual start time - (`kinfo_proc`) against what was recorded at lease time. A live PID with a - *different* start time means a different process entirely; treated as - exited. -2. **Does adopting it contradict something already restored?** Two leases - can't share one exclusive device, and one slot can't hold two devices. A - file that claims otherwise is treated as corrupt rather than trusted. - -A lease that passes both checks gets its slot rebuilt as `.active`, its -reference count bumped, and (if configured) a fresh exit watcher — from that -point on it's indistinguishable from a lease that was granted moments ago by -this same daemon. A lease that fails either check is simply dropped — not -released, not cleaned up, just forgotten. Its device isn't touched at all. -`createBase`/`clone` look up devices by name before creating new ones, so an -orphaned device gets rediscovered and correctly reference-counted the next -time something happens to lease that same config — but if nothing ever leases -that exact config again, this lazy reconciliation never fires. See Phase 9 -for the backstop that catches that case. - -## Phase 9 — the orphan reaper - -Every cleanup path above is triggered by an event: a release call, a PID -exit, an idle timer, or "the same config got leased again." Each of those can -miss a device — Phase 8 above is one way (a lease that never gets re-leased), -and a `simctl delete` call inside `delete()` failing silently (it's called as -`try? await delete(...)`, and the slot is reset to `.empty` in a `defer` -regardless of whether the delete actually succeeded) is another. Either way, -nothing else ever looks for that device again. - -`delete()` failing isn't necessarily transient, either: `simctl delete` isn't -reliable against a still-booted device, and nothing in the original lifecycle -ever shuts a clone down before deleting it (a clone goes straight from -"active" to "delete this" — there's no shutdown step in between). A device -that's genuinely wedged (unresponsive, the same profile as a `launchd_sim` -that's been running for days) will fail both `shutdown` and `delete` for the -same reason every time, not just once. - -`startReaper(interval:)` runs a sweep on a timer, independent of any lease -event, that reconciles against reality instead of the event stream: it lists -every simulator named with the manager's clone prefix -(`SimulatorConfig.cloneDeviceName`, so base simulators and anything not -created by this manager are never touched) and deletes any whose UDID has no -entry in `referenceCount` — the same dictionary key presence check works here -as everywhere else, since an entry exists from the moment a device is claimed -until `delete()` removes it, including through the idle-timer grace period. - -To avoid deleting a clone that's mid-creation (it exists on disk for a moment -before `createCloneTask` resumes and records it in `referenceCount`), a -device must show up as unknown on two consecutive sweeps before it's reaped. -Deletion goes straight through `simulatorControl`, not through `delete()`: -an orphan has no slot pointing at it, so there's nothing in `simulatorSlots` -for `delete()`'s bookkeeping to update. - -`SimulatorControl.delete()` itself now shuts the device down before each of -its own delete attempts (see `shutdownSimulator` in `SimulatorControl.swift`), -so most orphans clean up on the reaper's first attempt. For the wedged case -where that still isn't enough, the reaper tracks how many sweeps in a row a -given UDID has failed to delete (`orphanDeleteFailureCounts`). After three -consecutive failures, `reapOrphan` treats the device as stuck rather than -unlucky and calls `SimulatorControl.forceKillLaunchdSim(for:)`, which finds -the device's `launchd_sim` process by `pgrep -f ` (the process's own -command line embeds its data path, so the UDID is a safe, specific pattern), -double-checks the process name before signaling it, and sends it `SIGKILL` -directly — bypassing `simctl` entirely. One more delete attempt follows; if -that still fails, the device is logged loudly as needing manual cleanup -rather than retried silently forever. - -## Where this got subtle - -- The "mutate slot before the first `await`" rule is easy to state and easy to - violate by accident; it's the reason reference-count bookkeeping is smeared - across three different functions instead of living in one place. -- `pendingDeletion` slots deliberately don't remember their old exclusivity - flag — and don't need to, because reaching that state already implies zero - current owners. -- The dynamic (not snapshotted) recheck of `recentlyLeased` inside the idle- - deletion poll loop means the *duration* of a device's grace period can - change while it's already ticking. -- Persistence is a side effect of every state change, not a periodic snapshot - or a shutdown hook — which is what makes it safe against `kill -9`. diff --git a/#/LRUSet.swift b/#/LRUSet.swift deleted file mode 100644 index 1f903fd..0000000 --- a/#/LRUSet.swift +++ /dev/null @@ -1,49 +0,0 @@ -/// A `Set` that has a maximum capacity and evicts the least recently used item -// when full. -struct LRUSet { - private let capacity: Int - // An array to keep track of the order in which elements were inserted. - // The first element in the array is the least recently used. - private var order: [Element] = [] - - // A Set to enable fast O(1) membership tests. - private var storage: Set = [] - - init(capacity: Int) { - precondition(capacity > 0, "Capacity must be greater than zero.") - self.capacity = capacity - } - - // Returns an element that was evicted from the set, or nil if nothing was evicted - // (including when `element` was already present and just moved to most-recently-used). - mutating func insert(_ element: Element) -> Element? { - if storage.contains(element) { - if let index = order.firstIndex(of: element) { - order.remove(at: index) - } - - order.append(element) - return nil - } - - let evicted: Element? - if order.count >= capacity, let oldest = order.first { - order.removeFirst() - evicted = storage.remove(oldest) - } else { - evicted = nil - } - - order.append(element) - storage.insert(element) - return evicted - } - - func contains(_ element: Element) -> Bool { - return storage.contains(element) - } - - var elements: [Element] { - return order - } -} diff --git a/#/LeaseStore.swift b/#/LeaseStore.swift deleted file mode 100644 index 4c4cebd..0000000 --- a/#/LeaseStore.swift +++ /dev/null @@ -1,112 +0,0 @@ -import Foundation -import os - -extension Logger { - static let leaseStore = simulatorManager(category: "manager.lease-store") -} - -/// A lease, in the form it takes on disk. -/// -/// Holds everything needed to rebuild the daemon's in-memory bookkeeping for one -/// lease: which device, under which configuration, in which slot, and whether it -/// was owned exclusively. Nothing here has to be re-derived by asking `simctl`. -struct PersistedLease: Codable, Equatable { - let pid: PID - /// The leaser's start time, used to tell "PID 500 is still running" apart from - /// "PID 500 exited and something unrelated is now PID 500". - /// - /// Optional so a record written by a build that could not read the start time - /// still loads; such a record is restored on liveness alone. - let leaserStartTime: UInt64? - let udid: SimulatorUDID - let config: SimulatorConfig - let exclusive: Bool - let slotIndex: Int -} - -/// Where the daemon keeps its leases so a successor can pick them up. -protocol LeaseStore: Sendable { - /// Replaces the stored set with `leases`. Failures are logged, not thrown: - /// losing persistence degrades a restart, but failing the lease that triggered - /// the write would break a test that is otherwise fine. - func save(_ leases: [PersistedLease]) - - /// The stored set, or empty if there is nothing readable to restore. - func load() -> [PersistedLease] -} - -/// A `LeaseStore` backed by a JSON file. -/// -/// Written atomically, so a reader never sees a half-written set and a crash -/// mid-write leaves the previous contents intact. Combined with writing on every -/// change, this means the file is always current -- there is no flush-on-shutdown -/// step for a `kill -9` to skip. -struct FileLeaseStore: LeaseStore { - let path: String - - func save(_ leases: [PersistedLease]) { - let url = URL(fileURLWithPath: path) - do { - let encoder = JSONEncoder() - encoder.outputFormatting = [.prettyPrinted, .sortedKeys] - let data = try encoder.encode(leases) - - // `.atomic` writes an auxiliary file alongside the destination and renames - // it over the top, which is the whole reason this is durable. Doing that by - // hand would only duplicate it, and would leave a stray file of our own - // naming behind if we crashed between the write and the rename. - try data.write(to: url, options: .atomic) - } catch { - Logger.leaseStore.error( - """ - ❌ Failed to persist \(leases.count, privacy: .public) lease(s) to \ - \(path, privacy: .public): \(error, privacy: .public) - """ - ) - } - } - - func load() -> [PersistedLease] { - let url = URL(fileURLWithPath: path) - guard FileManager.default.fileExists(atPath: path) else { - return [] - } - - do { - let data = try Data(contentsOf: url) - return try JSONDecoder().decode([PersistedLease].self, from: data) - } catch { - // A corrupt or stale-format file must not stop the daemon from starting. - // The cost of ignoring it is the old behavior: leases predating the restart - // are unknown, and their releases report no lease. - Logger.leaseStore.error( - """ - ❌ Failed to read leases from \(path, privacy: .public); continuing with \ - none: \(error, privacy: .public) - """ - ) - return [] - } - } -} - -/// When `pid` started, in microseconds since the epoch, or nil if it cannot be -/// determined (typically because the process is gone). -/// -/// A PID alone does not identify a process across a daemon restart: PIDs are -/// recycled, so a lease for a dead PID could otherwise be restored onto whatever -/// unrelated process now holds that number, tying up a device until it exits. -/// Start time distinguishes them -- the kernel assigns it at fork, so a recycled -/// PID has a different one. -func processStartTime(_ pid: PID) -> UInt64? { - var info = kinfo_proc() - var size = MemoryLayout.stride - var mib: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, pid] - - guard sysctl(&mib, UInt32(mib.count), &info, &size, nil, 0) == 0, size > 0 else { - return nil - } - - let startTime = info.kp_proc.p_starttime - return UInt64(startTime.tv_sec) * 1_000_000 + UInt64(startTime.tv_usec) -} diff --git a/#/Logger.swift b/#/Logger.swift deleted file mode 100644 index 39507ba..0000000 --- a/#/Logger.swift +++ /dev/null @@ -1,10 +0,0 @@ -import os.log - -extension Logger { - static func simulatorManager(category: String) -> Logger { - Logger( - subsystem: "com.example.tools.simulator_manager", - category: category - ) - } -} diff --git a/#/Main.swift b/#/Main.swift deleted file mode 100644 index add0094..0000000 --- a/#/Main.swift +++ /dev/null @@ -1,105 +0,0 @@ -import ArgumentParser - -@main -struct Main: AsyncParsableCommand { - // This is set externally to prevent having to recompile the manager just for `start.sh` changes - @Option(help: "Version of the simulator manager") - var version: String - - @Option(help: "Path to where the pid should be written") - var pidPath: String - - @Option(help: "Path to where the unix domain socket should be created") - var unixSocketPath: String - - @Option(help: "Number of seconds to wait before deleting a recently used idle simulator") - var deleteRecentlyUsedIdleAfter: UInt16 - - @Option(help: "Number of seconds to wait before deleting a non-recently used idle simulator") - var deleteIdleAfter: UInt16 - - @Option( - help: """ - The number of simulators to keep in the recently used list; affects wether \ - 'delete-recently-used-idle-after' or 'delete-idle-after' is used when determining when to \ - delete an unused simulator - """ - ) - var recentlyUsedCapacity = 1 - - @Option( - name: .customLong("startup-process"), - help: """ - The path to a startup process that will be run when the simulator manager is started. This \ - process will not be relaunched if it exits. - - To pass custom arguments to the process you should wrap it in a script. - - Setting this flag multiple times will result in multiple startup process being launched. - """ - ) - var startupProcesses: [String] = [] - - @Option(help: "Path to an executable that will run after a simulator clone is booted") - var postBoot: String - - @Option( - help: """ - Path to a file where leases are mirrored, so that a simulator manager started \ - to replace this one adopts the leases of tests that are still running. Omit to \ - keep leases only in memory, in which case a restart loses them. - """ - ) - var leasePath: String? - - @Option( - help: """ - Seconds between sweeps of the orphan reaper, which deletes clone simulators the \ - manager has lost track of (e.g. left behind by a prior daemon's restart, or a \ - failed deletion). 0 disables the reaper. - """ - ) - var reapIntervalSeconds: UInt16 = 300 - - func validate() throws { - guard recentlyUsedCapacity > 0 else { - throw ValidationError( - """ - 'recently-used-capacity' must be greater than 0. - """ - ) - } - - guard Set(startupProcesses).count == startupProcesses.count else { - throw ValidationError("'startup-process' must be unique.") - } - } - - func run() async throws { - let simulatorManager = SimulatorManager( - simulatorControl: RealSimulatorControl(), - deleteRecentlyUsedIdleAfter: deleteRecentlyUsedIdleAfter, - deleteIdleAfter: deleteIdleAfter, - recentlyUsedCapacity: recentlyUsedCapacity, - deleteOnPIDExit: true, - startupProcesses: startupProcesses, - postBoot: postBoot, - leaseStore: leasePath.map { FileLeaseStore(path: $0) } - ) - - // Before serving, so the first release to arrive already sees the leases this - // daemon inherited. - await simulatorManager.restoreLeases() - - await simulatorManager.startReaper(interval: .seconds(Int(reapIntervalSeconds))) - - try await simulatorManager.startChildProcesses() - - try await HTTPServer( - simulatorRequestHandler: SimulatorRequestHandler( - simulatorManager: simulatorManager - ), - version: version - ).run(pidPath: pidPath, unixSocketPath: unixSocketPath) - } -} diff --git a/#/ORPHAN_SIMULATOR_FIX.md b/#/ORPHAN_SIMULATOR_FIX.md deleted file mode 100644 index 45d9398..0000000 --- a/#/ORPHAN_SIMULATOR_FIX.md +++ /dev/null @@ -1,195 +0,0 @@ -# Orphaned simulator fix - -Customers reported resource contention caused by `launchd_sim` processes that -had been running for days, tied to CoreSimulator devices the daemon no longer -knew about: - -```text -PID PPID RSS ELAPSED COMMAND -78247 1 9728 05-23:05:41 launchd_sim .../Devices/B9A13CD5-.../data/var/run/launchd_bootstrap.plist -78293 1 11600 05-23:05:40 launchd_sim .../Devices/88362C6D-.../... -78337 1 9968 05-23:05:40 launchd_sim .../Devices/B8CD72C0-.../... -``` - -This document explains the two root causes and the fix. See -[`LEASE_LIFECYCLE.md`](LEASE_LIFECYCLE.md) for the full lease state machine -this fix adds to (Phase 9). - -## Root causes - -Every existing cleanup path in `SimulatorManager.swift` is event-driven: an -explicit `DELETE /simulator/`, a PID-exit watcher, an idle timer, or a -device getting rediscovered by name the next time its config is leased -again. There was no path that checked the manager's bookkeeping against what -CoreSimulator actually had running. Two specific gaps let devices escape all -of those events at once. - -### 1. Daemon restarts drop leases without touching their devices - -`start.sh` replaces the daemon on every version change. On startup, -`restoreLeases()` (`SimulatorManager.swift:166`) re-adopts a persisted lease -only if its PID is still alive *and* its recorded process start time matches -(to rule out PID reuse). A lease that fails either check is dropped — -intentionally, per the existing code comment: "not released, not cleaned up, -just forgotten." The device itself is never shut down or deleted. - -The intended recovery path was that `createBase`/`clone` look up devices by -name, so an orphaned device would get picked up and reference-counted again -the next time something leased that same `SimulatorConfig`. If that exact -config was never requested again, nothing ever looked for the device again — -it, and its `launchd_sim`, ran forever. - -### 2. Failed `simctl delete` calls were swallowed silently - -In `delete()` (`SimulatorManager.swift:758-792`), the slot was reset to -`.empty` and the reference-count entry removed in a `defer` — regardless of -whether the `simctl delete` call inside it actually succeeded. Both call -sites invoked this as `try? await delete(...)`, discarding any error with no -retry. If `simctl delete` failed for any reason (CoreSimulator daemon busy, a -lingering child process, disk I/O), the manager's bookkeeping said "deleted" -while the real device kept running — and since its slot was now empty, the -manager would never look at that UDID again. - -Both gaps produce the same outcome: a real, running simulator with zero -corresponding state in the manager. - -## The fix - -### Retry `simctl delete` before giving up - -`SimulatorControl.swift`, `SimulatorDeleteOrExistenceMutex.unlockedDelete` -now retries the underlying `simctl delete` up to 3 times, 2 seconds apart, -before rethrowing. This doesn't touch `SimulatorManager`'s actor-isolated -slot state machine, so it carries no risk to the "mutate slot before the -first `await`" invariant documented above `getSimulator()`. It simply makes -the existing call sites succeed more often instead of falling straight -through to `try?`. - -### Periodic orphan reaper - -The real fix: a sweep that reconciles the manager's state against reality, -independent of any lease event, so it catches a device regardless of *how* -it became untracked. - -- **`SimulatorControl.listManagedClones()`** (new protocol method) lists - every simulator whose name starts with `EXAMPLE_BAZEL_CLONE_` — the prefix - this manager already uses for every clone it creates - (`SimulatorConfig.cloneDeviceName`), now hoisted into the shared constant - `managedCloneNamePrefix`. Base simulators (`EXAMPLE_BAZEL_BASE_...`) are - excluded — they're intentionally long-lived templates and are never - reference-counted, so they must never be touched by the reaper. Anything - without this prefix (a developer's own simulator, Xcode's own devices) is - untouched. -- **`SimulatorManager.reapOrphanedSimulators()`** (new, private) computes - `known = Set(referenceCount.keys)` and treats any listed clone UDID not in - that set as an orphan candidate. `referenceCount` is the right thing to - check: a device gets an entry the instant it's claimed, before any - `await`, and the entry is removed only in `delete()` — including while the - device sits in its idle-timer grace period, where the count is `0` but the - key stays. So "no entry" reliably means "the manager has no idea this - exists." -- **Two-sweep confirmation.** A clone that's mid-creation exists on disk (via - `simctl clone`) for a moment before `createCloneTask` resumes and records - it in `referenceCount`. To avoid mistaking that window for an orphan, a - device must show up as unknown on two consecutive sweeps - (`previousOrphanCandidates` intersected with the current sweep's - candidates) before it's deleted. -- **Deletes bypass `delete()`.** An orphan has no slot pointing at it in - `simulatorSlots`, so there's nothing for `delete()`'s slot bookkeeping to - update. The reaper calls `simulatorControl.delete()` directly instead — the - same lower-level operation `delete()` itself wraps. -- **`SimulatorManager.startReaper(interval:)`** (new, public) runs the sweep - on a loop and is started from `Main.swift` right after `restoreLeases()`. - The interval is configurable via the new `--reap-interval-seconds` flag - (default 300; `0` disables the reaper), and the task is cancelled in - `deinit` alongside the other background tasks. - -## Follow-up: the report didn't go away - -After the fix above, the same customer report came back. Investigating that -turned up two separate things: - -**The fix likely never shipped.** The commit was still local to this branch -(`git status` showed it 1 commit ahead of `origin/yannic-simulator-manager`, -never pushed), this package has no `BUILD` file or `Package.swift` — nothing -wires it into a build or deploy pipeline — and this branch's history includes -an earlier commit titled `DO NOT MERGE: Simulator manager`. Nothing here -indicates a build customers actually run has changed at all. - -**Independent of that, a real gap in the reaper itself.** Both the original -`delete()` and the new reaper's delete path called `simctl delete` without -ever calling `shutdown` first, and both discarded failures with `try?` and no -escalation. That's survivable for a healthy device, but a device that's -genuinely wedged — exactly the profile of a `launchd_sim` that's been running -for days — will fail `shutdown` and `delete` for the same underlying reason -every time. The reaper would correctly identify such a device as orphaned -after two sweeps, then retry a delete that fails identically every 5 minutes, -forever, silently. Retrying a deterministically broken operation isn't a fix. -See the next section for what addresses this. - -## Additional fix: shut down before delete, escalate when delete keeps failing - -- **`shutdownSimulator`** (new private free function in `SimulatorControl.swift`) - factors out the shutdown-with-"already shut down"-handling logic that - previously only backed the public `shutdown()` method. `unlockedDelete` now - calls it before each of its (still up to 3) delete attempts, best-effort - (`try?` — a shutdown failure for any other reason shouldn't block trying - delete anyway, since delete is what actually matters). -- **`SimulatorControl.forceKillLaunchdSim(for:)`** (new protocol method) is the - last resort for a device that keeps failing to delete. It finds the - device's `launchd_sim` process with `pgrep -f ` (the process's own - command line embeds its data path, so the UDID is a safe, specific - pattern), double-checks the process name before signaling it (`ps -p - -o comm=`, must end in `launchd_sim`), and sends `SIGKILL` directly, - bypassing `simctl` entirely. -- **`SimulatorManager` tracks consecutive reap failures per UDID** - (`orphanDeleteFailureCounts`). `reapOrphan` (new, factored out of - `reapOrphanedSimulators`) deletes normally on each sweep; after 3 - consecutive failures for the same UDID, it calls `forceKillLaunchdSim` and - makes one more delete attempt. If that still fails, it logs loudly ("needs - manual cleanup") instead of retrying the same failing call forever. The - failure count resets whenever a delete succeeds or the UDID stops being an - orphan at all. - -## Files changed - -| File | Change | -|:-----|:-------| -| `SimulatorControl.swift` | Retry + shutdown-first logic in `unlockedDelete`; new `listManagedClones()` and `forceKillLaunchdSim(for:)`; hoisted `managedCloneNamePrefix` constant; factored-out `shutdownSimulator` | -| `SimulatorManager.swift` | New `startReaper(interval:)`, `reapOrphanedSimulators()`, and `reapOrphan(_:)`; new `reaperTask`, `previousOrphanCandidates`, and `orphanDeleteFailureCounts` state | -| `Main.swift` | New `--reap-interval-seconds` flag; starts the reaper after `restoreLeases()` | -| `LEASE_LIFECYCLE.md` | Corrected Phase 8's claim that nothing hunts down orphans; added Phase 9 describing the reaper and its escalation path | - -## What this does not fix - -- The slot state machine itself (`SimulatorSlot` cases, `getSimulator`, the - "mutate before first `await`" rule) is unchanged. The reaper works - alongside it by going straight to `simulatorControl`, so the delicate - actor-isolation invariant didn't need to be touched. -- The Go port at `experiments/yannic/macsimulatormanager/go` was out of - scope for this fix — the bug report and investigation were both scoped to - the Swift implementation. - -## Verification status - -There is no `BUILD` file or `Package.swift` for this target yet, so it isn't -wired into Bazel and can't be built with `swift build` either. This change -has been verified by code review against the existing invariants (see -`LEASE_LIFECYCLE.md`), not by compiling or running it. Before relying on -this in production: - -1. Confirm this branch is actually the source for whatever customers run — - given the missing build target and the `DO NOT MERGE` history noted - above, that's not yet established. If it is, push this branch and get it - merged; if there's a separate deploy path, find it. -2. Set up a build target (Bazel or an ad hoc `Package.swift`) so this code - compiles and can be typechecked — it currently depends on `ShellOut`, - `ArgumentParser`, and SwiftNIO, none of which are vendored here. -3. Manually exercise the restart-orphan path: lease a config, `kill -9` the - daemon, start a new instance with a short `--reap-interval-seconds`, and - confirm the orphaned clone disappears from `xcrun simctl list devices` - within two sweep intervals without touching unrelated simulators. -4. Manually exercise the wedged-device escalation path: lease a config, then - independently hang or otherwise make its clone's `simctl shutdown`/`delete` - fail out-of-band, and confirm the reaper force-kills its `launchd_sim` - after 3 failed sweeps rather than retrying forever. diff --git a/#/PTY.swift b/#/PTY.swift deleted file mode 100644 index 6f11ed4..0000000 --- a/#/PTY.swift +++ /dev/null @@ -1,38 +0,0 @@ -import Darwin - -struct PTY { - let parent: Int32 - let child: Int32 - - init() throws { - var parentFd: Int32 = 0 - var childFd: Int32 = 0 - - // NULL for name/pw/termios/winsize = defaults - let result = openpty(&parentFd, &childFd, nil, nil, nil) - guard result == 0 else { - throw Errno(rawValue: errno) - } - - self.parent = parentFd - self.child = childFd - } -} - -/// Simple POSIX errno wrapper. -struct Errno: Error, RawRepresentable { - /// The raw POSIX error number. - let rawValue: Int32 - - init(rawValue: Int32) { - self.rawValue = rawValue - } -} - -extension Errno: CustomStringConvertible { - var description: String { - var buf = [CChar](repeating: 0, count: 256) - strerror_r(rawValue, &buf, buf.count) - return String(cString: buf) - } -} diff --git a/#/Package.resolved b/#/Package.resolved deleted file mode 100644 index 0691b04..0000000 --- a/#/Package.resolved +++ /dev/null @@ -1,176 +0,0 @@ -{ - "pins" : [ - { - "identity" : "shellout", - "kind" : "remoteSourceControl", - "location" : "https://github.com/JohnSundell/ShellOut", - "state" : { - "revision" : "e1577acf2b6e90086d01a6d5e2b8efdaae033568", - "version" : "2.3.0" - } - }, - { - "identity" : "swift-algorithms", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-algorithms.git", - "state" : { - "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", - "version" : "1.2.1" - } - }, - { - "identity" : "swift-argument-parser", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-argument-parser", - "state" : { - "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", - "version" : "1.8.2" - } - }, - { - "identity" : "swift-asn1", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-asn1.git", - "state" : { - "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", - "version" : "1.7.1" - } - }, - { - "identity" : "swift-async-algorithms", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-async-algorithms.git", - "state" : { - "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", - "version" : "1.1.5" - } - }, - { - "identity" : "swift-atomics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-atomics.git", - "state" : { - "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", - "version" : "1.3.1" - } - }, - { - "identity" : "swift-certificates", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-certificates.git", - "state" : { - "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", - "version" : "1.19.4" - } - }, - { - "identity" : "swift-collections", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-collections.git", - "state" : { - "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", - "version" : "1.6.0" - } - }, - { - "identity" : "swift-crypto", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-crypto.git", - "state" : { - "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", - "version" : "4.5.1" - } - }, - { - "identity" : "swift-http-structured-headers", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-http-structured-headers.git", - "state" : { - "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", - "version" : "1.7.0" - } - }, - { - "identity" : "swift-http-types", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-http-types.git", - "state" : { - "revision" : "db774a277f60063a32d854f2980299caf06da041", - "version" : "1.6.0" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", - "version" : "1.15.0" - } - }, - { - "identity" : "swift-nio", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio.git", - "state" : { - "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", - "version" : "2.101.3" - } - }, - { - "identity" : "swift-nio-extras", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-extras", - "state" : { - "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", - "version" : "1.34.3" - } - }, - { - "identity" : "swift-nio-http2", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-http2.git", - "state" : { - "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", - "version" : "1.45.0" - } - }, - { - "identity" : "swift-nio-ssl", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-ssl.git", - "state" : { - "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", - "version" : "2.37.2" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics.git", - "state" : { - "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", - "version" : "1.1.1" - } - }, - { - "identity" : "swift-service-lifecycle", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swift-server/swift-service-lifecycle.git", - "state" : { - "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", - "version" : "2.12.0" - } - }, - { - "identity" : "swift-system", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-system.git", - "state" : { - "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", - "version" : "1.8.1" - } - } - ], - "version" : 2 -} diff --git a/#/Package.swift b/#/Package.swift deleted file mode 100644 index e2830e1..0000000 --- a/#/Package.swift +++ /dev/null @@ -1,16 +0,0 @@ -// swift-tools-version: 5.9 - -import PackageDescription - -// This manifest exists only so `rules_swift_package_manager` can resolve and vendor -// this daemon's third-party dependencies for Bazel (see the BUILD file in this -// directory). It declares no targets/products of its own -- the daemon is built as -// a `swift_binary` directly from these sources, not as a Swift package. -let package = Package( - name: "macsimulatormanager", - dependencies: [ - .package(url: "https://github.com/JohnSundell/ShellOut", from: "2.3.0"), - .package(url: "https://github.com/apple/swift-argument-parser", from: "1.8.2"), - .package(url: "https://github.com/apple/swift-nio-extras", from: "1.34.3"), - ] -) diff --git a/#/README.md b/#/README.md deleted file mode 100644 index 3d9eda7..0000000 --- a/#/README.md +++ /dev/null @@ -1,141 +0,0 @@ -# Mac simulator manager - -A long-running daemon that manages the lifecycle of iOS Simulator devices and -hands them out, over HTTP, as leased resources for tests running on a Mac -worker. It exists so that concurrent test runs share a small pool of -simulators instead of each provisioning and tearing down its own, which is -slow and resource-hungry. - -The daemon speaks HTTP over a Unix domain socket rather than a TCP port, -since it is only ever talked to by processes on the same machine. - -## Running it - -`Main.swift` is a `swift-argument-parser` command. Notable options: - -- `--pid-path` / `--unix-socket-path` — where the daemon writes its PID file - and creates the listening socket. -- `--delete-idle-after` / `--delete-recently-used-idle-after` — how long an - unleased simulator is kept around before deletion, depending on whether its - config was recently leased (see [Idle deletion](#idle-deletion)). -- `--recently-used-capacity` — how many distinct simulator configs count as - "recently used" at once. -- `--startup-process` (repeatable) — extra processes to launch alongside the - daemon; see [Child processes](#child-processes). -- `--post-boot` — a script run against every freshly booted clone (see - [`SimulatorControl.swift`](SimulatorControl.swift)). -- `--lease-path` — where leases are mirrored to disk so a replacement daemon - can adopt them; see [Restarts and lease persistence](#restarts-and-lease-persistence). - -The `--version` flag is not read from source; it is passed in externally -(e.g. by a wrapper script) so the manager doesn't need to be recompiled just -to change how it reports its own version. - -## HTTP API - -Requests are parsed by `SimulatorManagerHTTPHandler.swift` and routed by -`HTTPServer.swift` on the first path component: - -| Method | Path | Query params | Description | -|--------|---------------------|--------------------------------------------|--------------| -| `POST` | `/simulator/` | `exclusive`, `deviceType`, `os`, `version` | Lease a simulator matching the given config to the process ``. Returns the simulator's UDID. | -| `DELETE` | `/simulator/` | | Release the simulator leased to ``. | -| `GET` | `/version` | | The daemon's version string. | -| `GET` | `/leases` | | The number of leases whose leasing process is still alive; for diagnostics. | -| `POST` | `/shutdown` | | Begin a graceful shutdown. | - -`exclusive=1` requests a simulator that no other lease may share; otherwise -the manager may hand out a simulator that is already leased non-exclusively -for the same config. Request parsing and response mapping live in -[`SimulatorRequestHandler.swift`](SimulatorRequestHandler.swift). - -## Architecture - -``` -HTTPServer (SwiftNIO, Unix domain socket) - └─ AccumulatedHTTPHandler buffers HTTP head/body/end into one FullHTTPRequest - └─ SimulatorManagerHTTPHandler parses method/path/query into a SimulatorManagerRequest - └─ SimulatorRequestHandler maps HTTP requests to SimulatorManager calls - └─ SimulatorManager (actor) lease/slot/reference-count bookkeeping - └─ SimulatorControl wraps `xcrun simctl` (create/clone/boot/delete) -``` - -### `SimulatorManager` - -[`SimulatorManager.swift`](SimulatorManager.swift) is the core state machine, -implemented as an actor so its bookkeeping is safe under concurrent leases. -For each `SimulatorConfig` (device type, OS, version) it keeps an array of -slots, each of which is `empty`, `pendingCreation`, `active`, `pendingDeletion`, -or `deleting`. Leasing a config walks the slots in a fixed preference order -(reuse an active non-exclusive simulator, then a pending deletion, then an -empty slot, then a pending creation) and either reuses what's there or clones -a fresh device from a per-config base simulator (created lazily and cached in -`getBaseSimulatorTasks`). - -Every active simulator has a reference count. Leasing increments it; releasing -decrements it, and it is only queued for deletion once it drops to zero, so a -non-exclusive simulator with multiple leasers survives until all of them -release it. - -### Idle deletion - -Once a simulator's reference count hits zero it isn't deleted immediately — -`pendingDeletion` schedules a delayed delete so a device can be reused by the -next lease for the same config. The wait is either `deleteIdleAfter` or the -longer `deleteRecentlyUsedIdleAfter`, chosen by whether that config appears in -`recentlyLeased`, an `LRUSet` (see [`LRUSet.swift`](LRUSet.swift)) capped at -`recentlyUsedCapacity` distinct configs. This keeps simulators for -in-demand configs around longer while letting one-off configs get cleaned up -quickly. - -### Restarts and lease persistence - -A new daemon version replaces the running one (the caller kills the old -process and starts the new one), which would otherwise lose track of leases -held by tests that are still running. `LeaseStore.swift` mirrors every lease -change to a JSON file; on startup, `restoreLeases()` reads it back and rebuilds -enough state (the lease, the slot holding the device, the reference count, and -an exit listener) for `release` to work normally. A lease is only adopted if -its leasing process is still running under the *same* start time (`kinfo_proc` -via `processStartTime`, not just the same PID — PIDs get recycled) and doesn't -conflict with another lease already restored; otherwise it's dropped, and its -device is picked up again automatically the next time something leases that -config. - -A leaser's exit is also watched for directly, via -`DispatchSource.makeProcessSource`, so a lease is released automatically if -its process dies without calling `DELETE /simulator/`. - -### `SimulatorControl` - -[`SimulatorControl.swift`](SimulatorControl.swift) wraps `xcrun simctl` calls -(`create`, `clone`, `bootstatus`, `shutdown`, `delete`, `list devices`) behind -a protocol, so `SimulatorManager` can be tested against a fake. Concurrent -calls for the same base/clone name are coalesced onto a single in-flight -`Task`, and deletion/existence checks for a given name are serialized through -`SimulatorDeleteOrExistenceMutex` so a `clone` can't observe a device that's -mid-deletion. - -### Child processes - -The daemon can launch extra long-lived processes alongside itself -(`--startup-process`). Each one's stdout/stderr is captured through a -`PTY` (see [`PTY.swift`](PTY.swift), needed because plain pipes make some -tools line-buffer differently) and logged line-by-line via `os.Logger`. -These processes are not restarted if they exit. - -## Files - -| File | Purpose | -|------|---------| -| `Main.swift` | CLI entry point; wires flags into a `SimulatorManager` and `HTTPServer`. | -| `HTTPServer.swift` | SwiftNIO server bound to a Unix domain socket; top-level request routing. | -| `AccumulatedHTTPHandler.swift` | Buffers streamed HTTP request parts into one in-memory request/response. | -| `SimulatorManagerHTTPHandler.swift` | Parses the HTTP request into method/path/query; serializes responses. | -| `SimulatorRequestHandler.swift` | Translates `/simulator` requests into `SimulatorManager` lease/release calls. | -| `SimulatorManager.swift` | Core actor: slots, reference counts, leases, idle deletion, child processes. | -| `SimulatorControl.swift` | `simctl`-backed implementation of creating, cloning, booting, and deleting simulators. | -| `LeaseStore.swift` | Persists leases to disk so a replacement daemon can adopt them. | -| `LRUSet.swift` | Fixed-capacity, least-recently-used set used to track recently leased configs. | -| `PTY.swift` | Minimal pseudo-terminal wrapper used to capture child process output. | -| `Logger.swift` | Shared `os.Logger` subsystem/category helper. | diff --git a/#/SimulatorControl.swift b/#/SimulatorControl.swift deleted file mode 100644 index b4b46c1..0000000 --- a/#/SimulatorControl.swift +++ /dev/null @@ -1,746 +0,0 @@ -import Foundation -import os.log -import ShellOut - -typealias SimulatorUDID = String - -extension Logger { - static let simulatorControl = simulatorManager(category: "control") -} - -struct SimulatorConfig: Hashable, Equatable, Codable { - let deviceType: String - let os: String - let version: String -} - -extension SimulatorConfig: CustomStringConvertible { - var description: String { - return "\(deviceType) (\(os) \(version))" - } -} - -struct SimCtlDevices: Decodable { - let devices: [String: [SimCtlDevice]] -} - -struct SimCtlDevice: Decodable { - let name: String - let udid: String -} - -struct ProcessError: Error { - let command: String - let context: String? - let exitCode: Int32 - let stdOut: String - let stdErr: String -} - -extension ProcessError: CustomStringConvertible { - var description: String { - let contextStr: String - if let context { - contextStr = " (\(context))" - } else { - contextStr = "" - } - - return """ - "\(command)"\(contextStr) failed with exit code \(exitCode): - \(stdOut)\(stdErr) - """ - } -} - -extension ProcessError: LocalizedError { - var errorDescription: String? { - return description - } -} - -protocol SimulatorControl: Actor { - // Creates a base simulator with the given config. - // - // It also boots and shuts down the simulator, making it ready for cloning. - // - // If an existing simulator with the same name already exists, that is returned instead of - // creating a new one. This is to support the simulator manager being restarted and losing state. - func createBase( - name: String, - with config: SimulatorConfig, - runtimeIdentifier: String - ) async throws -> SimulatorUDID - - // Clones a base simulator. - // - // It also boots the cloned simulator, making it ready for use. - // - // If an existing simulator with the same name already exists, that is returned instead of - // creating a new one. This is to support the simulator manager being restarted and losing state. - func clone( - _ baseSimulator: SimulatorUDID, - name: String, - deviceType: String, - runtimeIdentifier: String, - postBoot: String? - ) async throws -> SimulatorUDID - - func ensureBooted( - _ simulator: SimulatorUDID, - context: @escaping @autoclosure () -> String? - ) async throws - - func cleanTempFiles(in simulator: SimulatorUDID) - - func delete( - _ simulator: SimulatorUDID, - name: String, - context: @escaping @autoclosure () -> String? - ) async throws - - func getExisting( - name: String, - deviceType: String, - runtimeIdentifier: String, - context: @escaping @autoclosure () -> String? - ) async throws -> String? - - // Every clone-named simulator that currently exists, across all runtimes. - // - // Used by the orphan reaper to reconcile the manager's bookkeeping against reality. - // Deliberately excludes base simulators, which are meant to be long-lived and are - // never reference-counted. - func listManagedClones() async throws -> [SimCtlDevice] - - // Kills the `launchd_sim` process for `simulator` directly, bypassing `simctl` - // entirely. Best-effort and never throws: this is the last resort for a device - // that keeps failing `delete()` through normal means (a wedged simulator can fail - // both `shutdown` and `delete` indefinitely), used by the orphan reaper's - // escalation path. - func forceKillLaunchdSim(for simulator: SimulatorUDID) async -} - -actor RealSimulatorControl: SimulatorControl { - private var createBaseTasks: [String: Task] = [:] - private var cloneTasks: [String: Task] = [:] - - private var deleteAndExistenceMutexes: [String: (SimulatorDeleteOrExistenceMutex, Int)] = [:] - - func createBase( - name: String, - with config: SimulatorConfig, - runtimeIdentifier: String - ) async throws -> SimulatorUDID { - if let existingTask = createBaseTasks[name] { - return try await existingTask.value - } - - // We use a task to prevent data races that can occur when the `await` on `simctl` blocks. This - // ensures that multiple callers trying create a base simulator will all wait for the same - // simulator to be returned. - let task = Task { - defer { - createBaseTasks.removeValue(forKey: name) - } - - if let existingUDID = try await getExisting( - name: name, - deviceType: config.deviceType, - runtimeIdentifier: runtimeIdentifier, - context: "createBase" - ) { - Logger.simulatorControl.info( - """ - 📱 Base simulator "\(name, privacy: .public)" already exists, skipping creation: \ - \(existingUDID, privacy: .public) - """ - ) - - do { - // Under weird circumstances, the base simulator might be booted. This could happen if - // the simulator manager is killed in the process of creating a new base. Always call - // shutdown just in case. - try await shutdown(existingUDID, context: "createBase existing: \(name)") - } catch { - // If we fail to do what we need to, then we need to delete the faulty base simulator - Logger.simulatorControl.error( - """ - 📱 Failed to set up base simulator "\(name)" \(existingUDID, privacy: .public); deleting - """ - ) - - // If we fail to delete, don't throw _that_ error, throw the original error - try? await delete(existingUDID, name: name, context: "createBase existing: \(name)") - - throw error - } - - return existingUDID - } - - Logger.simulatorControl.info( - #"📱 Creating \#(config, privacy: .public) base simulator "\#(name, privacy: .public)""# - ) - - let udid = try await simctl( - ["create", name, config.deviceType, runtimeIdentifier] - ).trimmingCharacters(in: .whitespacesAndNewlines) - - do { - try await ensureBooted(udid, context: "createBase new: \(name)") - - // FIXME: Find a better way to know the simulator is ready - // Give the simulator some time to do some post-boot processing - try await Task.sleep(for: .seconds(5)) - - try await shutdown(udid, context: "createBase new: \(name)") - } catch { - // If we fail to do what we need to, then we need to delete the faulty base simulator - Logger.simulatorControl.error( - #""📱 Failed to set up base simulator "\#(name)" \#(udid, privacy: .public); deleting"# - ) - - // If we fail to delete, don't throw _that_ error, throw the original error - try? await delete(udid, name: name, context: "createBase new: \(name)") - - throw error - } - - Logger.simulatorControl.info( - """ - 📱 Created \(config, privacy: .public) base simulator \ - "\(name, privacy: .public)": \(udid, privacy: .public) - """ - ) - - return udid - } - - createBaseTasks[name] = task - - return try await task.value - } - - func clone( - _ baseSimulator: SimulatorUDID, - name: String, - deviceType: String, - runtimeIdentifier: String, - postBoot: String? = nil - ) async throws -> SimulatorUDID { - if let existingTask = cloneTasks[name] { - return try await existingTask.value - } - - // We use a task to prevent data races that can occur when the `await` on `simctl` blocks. This - // ensures that multiple callers trying create a base simulator will all wait for the same - // simulator to be returned. - let task = Task { - defer { - cloneTasks.removeValue(forKey: name) - } - - let udid: String - let isExisting: Bool - if let existingUDID = try await getExisting( - name: name, - deviceType: deviceType, - runtimeIdentifier: runtimeIdentifier, - context: "clone" - ) { - udid = existingUDID - isExisting = true - - // An existing simulator can be found if a previous simulator manager was killed before the - // clone was deleted. No tests _should_ be actively leasing the simulator. - Logger.simulatorControl.info( - """ - 📱 Cloned simulator "\(name, privacy: .public)" already exists, skipping creation: \ - \(udid, privacy: .public) - """ - ) - - // Wait for it to boot. This shouldn't be necessary, but sometimes the simulator will - // reboot because of a migration. - try await ensureBooted(udid, context: "clone, existing: \(name)") - } else { - isExisting = false - - Logger.simulatorControl.info( - """ - 📱 Cloning base simulator \(baseSimulator, privacy: .public) as \ - "\(name, privacy: .public)" - """ - ) - - udid = try await simctl( - ["clone", baseSimulator, name] - ).trimmingCharacters(in: .whitespacesAndNewlines) - - Logger.simulatorControl.info( - """ - 📱 Cloned base simulator \(baseSimulator, privacy: .public) as \ - "\(name, privacy: .public)": \(udid, privacy: .public) - """ - ) - - try await ensureBooted(udid, context: "clone, new: \(name)") - } - - if let postBoot { - Logger.simulatorControl.info( - """ - 📱 Running post-boot script "\(postBoot, privacy: .public)" on \ - \(udid, privacy: .public) - """ - ) - - do { - _ = try await subprocess(postBoot, env: ["SIMULATOR_UDID": udid]) - } catch { - throw NSError( - domain: "SimulatorControl", - code: 1, - userInfo: - [NSLocalizedDescriptionKey: "postBoot failed (isExisting: \(isExisting)): \(error)"] - ) - } - } - - return udid - } - - cloneTasks[name] = task - - return try await task.value - } - - func shutdown(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { - try await shutdownSimulator(simulator, context: context()) - } - - func cleanTempFiles(in simulator: SimulatorUDID) { - let fileManager = FileManager.default - - // Remove all files and directories under - // `data/Library/Caches/com.apple.containermanagerd/Dead/`, ignoring errors. There seems to be - // a bug where the simulator moves files here but never cleans them up. Maybe it's waiting for - // a reboot or something, which we never do. - let deadCachesPath = - "\(NSHomeDirectory())/Library/Developer/CoreSimulator/Devices/\(simulator)/data/Library/Caches/com.apple.containermanagerd/Dead" - guard let contents = try? fileManager.contentsOfDirectory(atPath: deadCachesPath) else { - return - } - for item in contents { - let itemPath = "\(deadCachesPath)/\(item)" - try? fileManager.removeItem(atPath: itemPath) - } - } - - func delete( - _ simulator: SimulatorUDID, - name: String, - context: @escaping @autoclosure () -> String? - ) async throws { - try await deleteAndExistenceMutex(name: name) { mutex in - try await mutex.unlockedDelete(simulator, context: context()) - } - } - - func getExisting( - name: String, - deviceType: String, - runtimeIdentifier: String, - context: @escaping @autoclosure () -> String? - ) async throws -> String? { - return try await deleteAndExistenceMutex(name: name) { mutex in - return try await mutex.unlockedGetExisting( - name: name, - deviceType: deviceType, - runtimeIdentifier: runtimeIdentifier, - context: context() - ) - } - } - - func listManagedClones() async throws -> [SimCtlDevice] { - let output = try await simctl(["list", "devices", "-j"], context: "listManagedClones") - - guard let jsonData = output.data(using: .utf8) else { - throw NSError( - domain: "SimulatorControl", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "Failed to convert output to data"] - ) - } - - let devicesByRuntime: [String: [SimCtlDevice]] - do { - devicesByRuntime = try JSONDecoder().decode(SimCtlDevices.self, from: jsonData).devices - } catch { - let json = String(data: jsonData, encoding: .utf8) ?? "" - Logger.simulatorControl.error( - """ - ❌ Failed to decode 'simctl list devices -j': \(error, privacy: .public). - Output: \(json, privacy: .public) - """ - ) - throw NSError( - domain: "SimulatorControl", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "Failed to decode output: \(error) - \(json)"] - ) - } - - return devicesByRuntime.values.flatMap { $0 }.filter { $0.name.hasPrefix(managedCloneNamePrefix) } - } - - func forceKillLaunchdSim(for simulator: SimulatorUDID) async { - // `launchd_sim`'s command line embeds the device's own data path (see the ps - // output that motivated this: ".../Devices//data/var/run/..."), so the - // UDID is a safe, specific `pgrep -f` pattern -- it can only match that one - // device's process tree. - let pids: [Int32] - do { - let output = try await subprocess("/usr/bin/pgrep", ["-f", simulator]) - pids = output.split(separator: "\n").compactMap { Int32($0.trimmingCharacters(in: .whitespaces)) } - } catch { - // `pgrep` exits non-zero (surfaced here as a thrown `ProcessError`) when nothing - // matches -- there's nothing left to kill. - return - } - - for pid in pids { - // Double check this is actually `launchd_sim` before signaling it. `pgrep -f` - // matches the UDID anywhere in the command line; being wrong here would kill an - // unrelated process that merely mentioned this device (e.g. in a log path). - guard let comm = try? await subprocess("/bin/ps", ["-p", "\(pid)", "-o", "comm="]), - comm.trimmingCharacters(in: .whitespacesAndNewlines).hasSuffix("launchd_sim") else { - continue - } - - Logger.simulatorControl.warning( - """ - 🔨 Force-killing launchd_sim (pid \(pid, privacy: .public)) for stuck simulator \ - \(simulator, privacy: .public) - """ - ) - kill(pid, SIGKILL) - } - } - - func ensureBooted(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { - for retriesLeft in (0...1).reversed() { - do { - // This private command boots the simulator if it isn't already, and waits for the - // appropriate amount of time until we can actually run tests - _ = try await simctl(["bootstatus", simulator, "-b"], context: context()) - break - } catch let error as ProcessError { - // Exit code 149 is related to the simulator already being booted - guard error.exitCode == 149 && retriesLeft > 0 else { - throw error - } - - // This is a known error that happens when the simulator is already booted. A retry - // should succeed. - Logger.simulatorControl.warning( - """ - ⚠️ Boot of simulator \(simulator, privacy: .public) failed, but probably \"already \ - booted\": \(error, privacy: .public) - """ - ) - } - } - } - - private func deleteAndExistenceMutex( - name: String, - _ call: (_ mutex: SimulatorDeleteOrExistenceMutex) async throws -> T - ) async throws -> T { - var (mutex, referenceCount) = - deleteAndExistenceMutexes[name] ?? (SimulatorDeleteOrExistenceMutex(), 0) - referenceCount += 1 - deleteAndExistenceMutexes[name] = (mutex, referenceCount) - - defer { - guard let mutexAndRef = deleteAndExistenceMutexes[name] else { - preconditionFailure( - """ - State of `deleteAndExistenceMutexes` changed unexpectedly. Expected value for "\(name)". - """ - ) - } - let mutex = mutexAndRef.0 - var referenceCount = mutexAndRef.1 - referenceCount -= 1 - - if referenceCount == 0 { - deleteAndExistenceMutexes.removeValue(forKey: name) - } else { - deleteAndExistenceMutexes[name] = (mutex, referenceCount) - } - } - - return try await mutex.withLock { - try await call(mutex) - } - } -} - -// An instance of this actor is created for each simulator name that is being checked for existence -// or being deleted. The actor is only called through `withLock()`, which will suspend on multiple -// calls to ensure that these operations are serialized. Without this, someone could try to call -// `clone()` while a deletion is pending, which will call `getExisting()`, and it can return the -// simulator that is in the process of being deleted. -actor SimulatorDeleteOrExistenceMutex { - private var isLocked = false - private var waiters: [CheckedContinuation] = [] - - /// Acquires, runs the work, and then releases the lock. - func withLock(_ work: () async throws -> T) async throws -> T { - await lock() - defer { unlock() } - return try await work() - } - - /// Acquires the lock. If already locked, will suspend until unlocked. - private func lock() async { - if !isLocked { - isLocked = true - } else { - await withCheckedContinuation { cont in - waiters.append(cont) - } - } - } - - /// Releases the lock and wakes one waiter (if any). - private func unlock() { - if !waiters.isEmpty { - let cont = waiters.removeFirst() - cont.resume() - } else { - isLocked = false - } - } - - func unlockedGetExisting( - name: String, - deviceType: String, - runtimeIdentifier: String, - context: @escaping @autoclosure () -> String? - ) async throws -> String? { - Logger.simulatorControl.debug( - #"🔍 Trying to find existing simulator "\#(name, privacy: .public)""# - ) - - let output = try await simctl(["list", "devices", "-j", deviceType], context: context()) - - guard let jsonData = output.data(using: .utf8) else { - throw NSError( - domain: "SimulatorControl", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "Failed to convert output to data"] - ) - } - - let jsonDecoder = JSONDecoder() - - let devicesByRuntime: [String: [SimCtlDevice]] - do { - devicesByRuntime = try jsonDecoder.decode(SimCtlDevices.self, from: jsonData).devices - } catch { - let json = String(data: jsonData, encoding: .utf8) ?? "" - Logger.simulatorControl.error( - """ - ❌ Failed to decode 'simctl list devices -j': \(error, privacy: .public). - Output: \(json, privacy: .public) - """ - ) - throw NSError( - domain: "SimulatorControl", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "Failed to decode output: \(error) - \(json)"] - ) - } - - if let devices = devicesByRuntime[runtimeIdentifier] { - for device in devices { - if device.name == name { - let udid = device.udid - - Logger.simulatorControl.debug( - #"🔍 Found existing simulator "\#(name, privacy: .public)": \#(udid, privacy: .public)"# - ) - - // Sometimes the simulator is not actually on disk, but it is in the list. If this - // happens, "delete" it so simctl stops reporting it as existing. - if !FileManager.default.fileExists( - atPath: - "\(NSHomeDirectory())/Library/Developer/CoreSimulator/Devices/\(udid)" - ) { - Logger.simulatorControl.debug( - """ - ⚠️ Simulator \(udid, privacy: .public) doesn't actually exist on disk; "deleting" - """ - ) - - // If we fail to delete, don't throw an error - try? await unlockedDelete(udid, context: context()) - - return nil - } - - return udid - } - } - } - - Logger.simulatorControl.debug( - #"🔍 No existing simulator "\#(name, privacy: .public)" found"# - ) - - return nil - } - - func unlockedDelete( - _ simulator: SimulatorUDID, - context: @escaping @autoclosure () -> String? - ) async throws { - Logger.simulatorControl.info("🗑️ Deleting simulator \(simulator, privacy: .public)") - - // `simctl delete` can fail transiently (CoreSimulator daemon busy, a lingering - // child process, disk I/O) or because the device is still booted -- `delete` - // does not reliably shut a booted device down on its own. Callers treat a thrown - // error here as "give up until the next event," so shut down and retry a few - // times before surfacing failure -- it's much cheaper than leaving the simulator - // running until the orphan reaper's next sweep catches it. - let maxAttempts = 3 - for attempt in 1...maxAttempts { - // Best-effort: proceed to the delete attempt regardless of whether this - // succeeds. A shutdown failure for a reason other than "already shut down" - // (already handled inside `shutdownSimulator`) shouldn't block trying delete - // anyway, since delete is the operation that actually matters here. - try? await shutdownSimulator(simulator, context: context()) - - do { - _ = try await simctl(["delete", simulator], context: context()) - Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") - return - } catch { - guard attempt < maxAttempts else { - Logger.simulatorControl.error( - """ - ❌ Failed to delete simulator \(simulator, privacy: .public) after \ - \(maxAttempts, privacy: .public) attempts: \(error, privacy: .public) - """ - ) - throw error - } - - Logger.simulatorControl.warning( - """ - ⚠️ Delete attempt \(attempt, privacy: .public)/\(maxAttempts, privacy: .public) for \ - simulator \(simulator, privacy: .public) failed, retrying: \(error, privacy: .public) - """ - ) - try? await Task.sleep(for: .seconds(2)) - } - } - } -} - -/// Shared by `RealSimulatorControl.shutdown` and `unlockedDelete`, since delete needs -/// to shut the device down first and shouldn't reimplement this. -private func shutdownSimulator( - _ simulator: SimulatorUDID, - context: @escaping @autoclosure () -> String? -) async throws { - do { - _ = try await simctl(["shutdown", simulator], context: context()) - } catch let error as ProcessError { - // Exit code 149 is related to the simulator already being shut down - guard error.exitCode == 149 else { - throw error - } - - Logger.simulatorControl.warning( - """ - ⚠️ Shutdown failed, but probably \"already shut down\": \(error, privacy: .public) - """ - ) - } -} - -private func simctl( - _ args: [String], - context: @escaping @autoclosure () -> String? = nil -) async throws -> String { - return try await subprocess("/usr/bin/xcrun", ["simctl"] + args, context: context()) -} - -private func subprocess( - _ executable: String, - _ args: [String] = [], - env: [String: String] = [:], - context: @escaping @autoclosure () -> String? = nil -) async throws -> String { - return try await Task { try syncSubprocess(executable, args, env: env, context: context()) }.value -} - -private func syncSubprocess( - _ executable: String, - _ args: [String] = [], - env: [String: String] = [:], - context: @escaping @autoclosure () -> String? = nil -) throws -> String { - let quotedArgs = args.map { "'\($0)'" } - - var newEnv = ProcessInfo.processInfo.environment.merging(env) { _, new in new } - newEnv["PWD"] = FileManager.default.currentDirectoryPath - - let process = Process() - process.environment = ProcessInfo.processInfo.environment.merging(env) { _, new in new } - - let command = "\(executable) \(quotedArgs.joined(separator: " "))" - - Logger.simulatorControl.debug(#"🛠️ Running "\#(command, privacy: .public)""#) - - do { - return try shellOut( - to: executable, - arguments: quotedArgs, - process: process - ) - } catch let error as ShellOutError { - throw ProcessError( - command: command, - context: context(), - exitCode: error.terminationStatus, - stdOut: error.output, - stdErr: error.message - ) - } -} - -/// Prefix shared by every simulator this manager creates for cloning (see -/// `SimulatorConfig.cloneDeviceName`). Used to scope the orphan reaper to devices -/// it manages, so it never touches a developer's own simulators or base templates. -let managedCloneNamePrefix = "EXAMPLE_BAZEL_CLONE_" - -extension SimulatorConfig { - func baseDeviceName() -> String { - return "EXAMPLE_BAZEL_BASE_\(deviceType)_\(version)" - } - - func cloneDeviceName(index: Int) -> String { - return "\(managedCloneNamePrefix)\(deviceType)_\(version)_\(index)" - } - - func runtimeIdentifier() -> String { - let runtimeVersion = version.replacingOccurrences(of: ".", with: "-") - return "com.apple.CoreSimulator.SimRuntime.\(os)-\(runtimeVersion)" - } -} diff --git a/#/SimulatorManager.swift b/#/SimulatorManager.swift deleted file mode 100644 index a488b5e..0000000 --- a/#/SimulatorManager.swift +++ /dev/null @@ -1,1058 +0,0 @@ -import Foundation -import os -import ShellOut - -typealias PID = pid_t - -extension Logger { - static let simulatorManager = simulatorManager(category: "manager") - static let childProcess = simulatorManager(category: "manager.child-process") -} - -enum SimulatorManagerError: Error { - case alreadyLeased(udid: SimulatorUDID) - case noLease - case leaserExited -} - -private struct SimulatorLease { - let udid: SimulatorUDID - let config: SimulatorConfig - let exclusive: Bool - let slotIndex: Int - /// When the leasing process started, captured at lease time. - /// - /// Persisted so a successor daemon can tell a still-running leaser from a - /// recycled PID. Nil when it could not be read. - let leaserStartTime: UInt64? -} - -/// Whether `pid` is still running. -/// -/// `kill(pid, 0)` reports failure for two unrelated reasons, and only one of them -/// means the process is gone: `ESRCH` (no such process) versus `EPERM` (it exists -/// but we may not signal it). Treating `EPERM` as death would release a live -/// leaser's device out from under it, so only `ESRCH` counts. -func processIsRunning(_ pid: PID) -> Bool { - if kill(pid, 0) == 0 { - return true - } - return errno != ESRCH -} - -private enum SimulatorSlot { - case empty - case pendingCreation(Task, exclusive: Bool) - case active(SimulatorUDID, exclusive: Bool) - case pendingDeletion(SimulatorUDID, Task) - case deleting(SimulatorUDID) -} - -extension SimulatorSlot { - var sortOrder: Int { - switch self { - // Try to use an active simulator first (should only be one for non-exclusive) - case .active: - return 0 - - // A pending deletion is already created and booted, so reuse it before waiting - // on a pending creation or starting a fresh one. - case .pendingDeletion: - return 1 - - // Use an empty slot before waiting on a pending creation, so a lease doesn't - // block on someone else's in-flight clone when a fresh slot is free to start. - case .empty: - return 2 - - case .pendingCreation: - return 3 - - // Deleting simulator can't be used, so put it at the end - case .deleting: - return 4 - } - } -} - -private enum SimulatorSlotResult { - case active(SimulatorUDID, slotIndex: Int) - case pending(Task, slotIndex: Int) -} - -actor SimulatorManager { - private let simulatorControl: SimulatorControl - - private var simulatorSlots: [SimulatorConfig: [SimulatorSlot]] = [:] - private var referenceCount: [SimulatorUDID: Int] = [:] - private var leases: [PID: SimulatorLease] = [:] - - private var leaserExitListeners: [PID: DispatchSourceProcess] = [:] - - private var getBaseSimulatorTasks: [SimulatorConfig: Task] = [:] - - private let deleteIdleAfter: UInt16 - private let deleteRecentlyUsedIdleAfter: UInt16 - private let deleteOnPIDExit: Bool - - /// Where leases are mirrored so a successor daemon can adopt them. Nil disables - /// persistence, which is the default in tests that do not care about it. - private let leaseStore: LeaseStore? - - private var recentlyLeased: LRUSet - - private var startupProcessPaths: [String] - private var postBoot: String? - private var childProcessTasks: [Task] = [] - private var childProcesses: [String: (Process, DispatchSourceRead, DispatchSourceRead)] = [:] - - private var reaperTask: Task? - /// Clone UDIDs that looked orphaned on the *previous* sweep. A device must appear - /// unknown on two consecutive sweeps before the reaper deletes it, so a clone that's - /// mid-creation (it exists on disk before `createCloneTask` resumes and records it in - /// `referenceCount`) never gets caught by a single unlucky sweep. - private var previousOrphanCandidates: Set = [] - /// How many sweeps in a row the reaper has failed to delete a confirmed orphan. - /// Reset once a delete succeeds, or once the UDID stops being an orphan at all - /// (claimed by a new lease, or already cleaned up some other way). - private var orphanDeleteFailureCounts: [SimulatorUDID: Int] = [:] - /// After this many consecutive failed deletes, a device is treated as wedged - /// rather than merely unlucky: normal `simctl shutdown`/`delete` retries have - /// already been exhausted inside `SimulatorControl.delete()` every sweep, so more - /// of the same is unlikely to help. Escalate to killing its `launchd_sim` directly. - private let forceKillAfterFailedReapAttempts = 3 - - init( - simulatorControl: SimulatorControl, - deleteRecentlyUsedIdleAfter: UInt16, - deleteIdleAfter: UInt16, - recentlyUsedCapacity: Int, - deleteOnPIDExit: Bool, - startupProcesses: [String] = [], - postBoot: String? = nil, - leaseStore: LeaseStore? = nil - ) { - self.simulatorControl = simulatorControl - self.deleteIdleAfter = deleteIdleAfter - self.deleteRecentlyUsedIdleAfter = deleteRecentlyUsedIdleAfter - self.deleteOnPIDExit = deleteOnPIDExit - self.recentlyLeased = LRUSet(capacity: recentlyUsedCapacity) - self.startupProcessPaths = startupProcesses - self.postBoot = postBoot - self.leaseStore = leaseStore - - // Change the working directory to some place stable, since on RBE the runfiles directory can - // get cleaned up - FileManager.default.changeCurrentDirectoryPath("/tmp") - } - - deinit { - reaperTask?.cancel() - - for task in childProcessTasks { - task.cancel() - } - - for (process, outWatcher, errWatcher) in childProcesses.values { - process.terminate() - outWatcher.cancel() - errWatcher.cancel() - } - } - - func startChildProcesses() throws { - for path in startupProcessPaths { - childProcessTasks.append(createStartChildProcessTask(path: path)) - } - } - - /// Starts the periodic sweep that deletes clone simulators the manager has lost - /// track of. - /// - /// Every other cleanup path is event-driven: an explicit release, a PID-exit - /// watcher, an idle timer, or "rediscovered by name on the next lease of the same - /// config." Each of those can miss a device -- a daemon restart drops a lease - /// whose device is never leased again (see `restoreLeases`), or a `simctl delete` - /// call fails and is swallowed (see `delete`) -- and nothing else ever looks for - /// it again. This sweep is the backstop: it reconciles against what CoreSimulator - /// actually has running, independent of how a device became untracked. - /// - /// `interval <= .zero` disables it, matching how `deleteIdleAfter` of 0 means - /// "immediately" elsewhere in this file rather than "never." - func startReaper(interval: Duration) { - guard interval > .zero else { return } - - reaperTask = Task { - while !Task.isCancelled { - try? await Task.sleep(for: interval) - guard !Task.isCancelled else { break } - await reapOrphanedSimulators() - } - } - } - - /// One sweep of the orphan reaper: list every clone simulator that exists, and - /// delete the ones with no entry in `referenceCount`. - /// - /// `referenceCount` is the right ground truth to check against, not `leases` or - /// `simulatorSlots`: a device gets an entry in it the instant it's claimed (before - /// any `await`, per the invariant on `getSimulator`), and the entry is removed - /// only in `delete()` -- including while the device sits in the idle-timer grace - /// period, where the count is `0` but the key stays. So "no entry" reliably means - /// "the manager has no idea this exists," not merely "nothing is leasing it right - /// now." - /// - /// Bypasses `delete()` deliberately: that function updates a slot in - /// `simulatorSlots`, but an orphan by definition has no slot pointing at it, so - /// there is nothing there to update. This calls `simulatorControl` directly, the - /// same lower-level operation `delete()` itself wraps. - private func reapOrphanedSimulators() async { - let known = Set(referenceCount.keys) - - let managedClones: [SimCtlDevice] - do { - managedClones = try await simulatorControl.listManagedClones() - } catch { - Logger.simulatorManager.error( - "❌ Orphan reaper failed to list simulators, skipping this sweep: \(error, privacy: .public)" - ) - return - } - - let currentOrphanCandidates = Set(managedClones.map(\.udid)).subtracting(known) - let confirmedOrphans = currentOrphanCandidates.intersection(previousOrphanCandidates) - previousOrphanCandidates = currentOrphanCandidates - - // Forget the failure count for anything that isn't a confirmed orphan any more - // (claimed by a new lease, or already cleaned up), so a UDID that's reused later - // starts with a clean slate rather than inheriting an old device's history. - orphanDeleteFailureCounts = orphanDeleteFailureCounts.filter { confirmedOrphans.contains($0.key) } - - guard !confirmedOrphans.isEmpty else { return } - - for device in managedClones where confirmedOrphans.contains(device.udid) { - await reapOrphan(device) - } - } - - /// Deletes one confirmed-orphaned device, escalating to a direct kill of its - /// `launchd_sim` if it has already failed to delete - /// `forceKillAfterFailedReapAttempts` sweeps in a row. - /// - /// `SimulatorControl.delete()` already shuts the device down and retries a few - /// times internally before throwing, so a failure reaching here means those - /// retries were exhausted -- consistent with a genuinely wedged device (the same - /// profile as the multi-day-old `launchd_sim` processes that motivated this), not - /// a one-off transient error. Retrying the same call every 5-minute sweep forever - /// would just fail the same way forever, silently; escalating is what makes this a - /// backstop instead of another silent no-op. - private func reapOrphan(_ device: SimCtlDevice) async { - Logger.simulatorManager.warning( - """ - 🧹 Reaping orphaned simulator \(device.udid, privacy: .public) \ - (\(device.name, privacy: .public)); the manager has no lease or reference to it - """ - ) - - do { - try await simulatorControl.delete(device.udid, name: device.name, context: "orphan reaper") - orphanDeleteFailureCounts.removeValue(forKey: device.udid) - return - } catch { - let failures = (orphanDeleteFailureCounts[device.udid] ?? 0) + 1 - orphanDeleteFailureCounts[device.udid] = failures - - Logger.simulatorManager.error( - """ - ❌ Orphan reaper failed to delete \(device.udid, privacy: .public) \ - (\(device.name, privacy: .public)), attempt \(failures, privacy: .public): \ - \(error, privacy: .public) - """ - ) - - guard failures >= forceKillAfterFailedReapAttempts else { return } - - Logger.simulatorManager.error( - """ - 🔨 \(device.udid, privacy: .public) (\(device.name, privacy: .public)) has failed to \ - delete \(failures, privacy: .public) sweeps in a row; force-killing its launchd_sim - """ - ) - - await simulatorControl.forceKillLaunchdSim(for: device.udid) - - do { - try await simulatorControl.delete( - device.udid, - name: device.name, - context: "orphan reaper, post force-kill" - ) - orphanDeleteFailureCounts.removeValue(forKey: device.udid) - } catch { - Logger.simulatorManager.error( - """ - ❌ \(device.udid, privacy: .public) (\(device.name, privacy: .public)) still failed to \ - delete after force-killing launchd_sim: \(error, privacy: .public). Needs manual cleanup. - """ - ) - } - } - } - - /// Adopts the leases a previous daemon left behind. - /// - /// Leases used to live only in this actor's memory, so replacing the daemon -- - /// which `start.sh` does on any version change, on every lease -- lost the - /// bookkeeping for tests that were still running. Their release calls then - /// reached a daemon that had never heard of them and reported "doesn't have a - /// simulator leased". - /// - /// Restoring rebuilds enough state for `release` to work: the lease itself, the - /// slot holding the device, a reference count, and an exit listener so a leaser - /// that dies during the handover is still cleaned up. - /// - /// Leases whose process is gone are dropped rather than adopted. The device they - /// held is left alone: `createBase`/`clone` find existing devices by name, so it - /// gets picked up and reference counted by the next lease of that config. - func restoreLeases() { - guard let leaseStore else { return } - - let persisted = leaseStore.load() - guard !persisted.isEmpty else { return } - - var adopted = 0 - var dropped = 0 - - for lease in persisted { - guard leaserSurvived(lease) else { - dropped += 1 - continue - } - - // Two leases cannot share an exclusive device, and a slot cannot hold two - // different devices. A file that says otherwise is inconsistent -- possibly - // hand-edited, or written by a version with different slot semantics -- so - // prefer dropping the lease over corrupting the slot bookkeeping. - guard canAdopt(lease) else { - Logger.simulatorManager.error( - """ - ❌ Not restoring conflicting lease for PID \(lease.pid, privacy: .public) \ - on \(lease.udid, privacy: .public) - """ - ) - dropped += 1 - continue - } - - leases[lease.pid] = .init( - udid: lease.udid, - config: lease.config, - exclusive: lease.exclusive, - slotIndex: lease.slotIndex, - leaserStartTime: lease.leaserStartTime - ) - - // Recreate the slot the device occupies, so it is neither handed to an - // incompatible lease nor deleted as idle while its owner is still running. - var slots = simulatorSlots[lease.config] ?? [] - while slots.count <= lease.slotIndex { - slots.append(.empty) - } - slots[lease.slotIndex] = .active(lease.udid, exclusive: lease.exclusive) - simulatorSlots[lease.config] = slots - - incrementReferenceCount(for: lease.udid) - - if deleteOnPIDExit { - registerReleaseOnExit(for: lease.pid) - } - - adopted += 1 - } - - Logger.simulatorManager.info( - """ - ♻️ Restored \(adopted, privacy: .public) lease(s) from a previous simulator \ - manager; dropped \(dropped, privacy: .public) whose process had exited - """ - ) - - // The dropped entries are gone for good, so rewrite the file rather than let a - // later crash re-adopt them. - persistLeases() - } - - /// Whether the process that took `lease` is the one still running under that - /// PID, rather than an unrelated process that reused the number. - private func leaserSurvived(_ lease: PersistedLease) -> Bool { - guard processIsRunning(lease.pid) else { return false } - - // A record without a start time predates that field. Fall back to liveness - // alone: adopting on a PID collision costs one device held until that process - // exits, which is better than dropping every lease on a mixed-version upgrade. - guard let persistedStart = lease.leaserStartTime else { return true } - - guard let currentStart = processStartTime(lease.pid) else { return false } - - guard currentStart == persistedStart else { - Logger.simulatorManager.info( - """ - 👻 PID \(lease.pid, privacy: .public) is running but started at a different \ - time than the lease recorded; treating the leaser as exited - """ - ) - return false - } - - return true - } - - /// Whether `lease` can be adopted without contradicting one already restored. - private func canAdopt(_ lease: PersistedLease) -> Bool { - for existing in leases.values { - if existing.udid == lease.udid, existing.exclusive || lease.exclusive { - return false - } - - if existing.config == lease.config, - existing.slotIndex == lease.slotIndex, - existing.udid != lease.udid { - return false - } - } - - return true - } - - /// Mirrors the current leases to disk. - /// - /// Called on every change rather than at shutdown, because the daemon is not - /// always shut down politely: `start.sh` escalates to `kill -9`, which runs no - /// cleanup. Each lease costs a small JSON write, against provisioning a - /// simulator that takes seconds. - private func persistLeases() { - guard let leaseStore else { return } - - leaseStore.save( - leases.map { pid, lease in - .init( - pid: pid, - leaserStartTime: lease.leaserStartTime, - udid: lease.udid, - config: lease.config, - exclusive: lease.exclusive, - slotIndex: lease.slotIndex - ) - } - ) - } - - func lease( - to leaser: PID, - exclusive: Bool, - config: SimulatorConfig - ) async throws -> SimulatorUDID { - // Each process can only lease one simulator at a time - if let existingLease = leases[leaser] { - throw SimulatorManagerError.alreadyLeased(udid: existingLease.udid) - } - - Logger.simulatorManager.info( - """ - 🔒 Leasing \(exclusive ? "exclusive" : "non-exclusive", privacy: .public) \ - \(config, privacy: .public) simulator for PID \(leaser, privacy: .public) - """ - ) - - // Check liveness before provisioning rather than after. `getSimulator()` can - // await for minutes while it clones, boots and runs the post-boot script, and - // a leaser that dies during that window (e.g. killed by its build tool's test - // timeout while queued for a simulator) used to be detected only afterwards -- - // releasing the lease a millisecond after granting it, and deleting the device - // out from under a test that had already been handed the UDID. - // - // Only meaningful when we track leaser exit at all; otherwise the caller owns - // the lease lifetime and the PID need not be a live process. - if deleteOnPIDExit, !processIsRunning(leaser) { - Logger.simulatorManager.info( - "👋 PID \(leaser, privacy: .public) exited before its lease could be provisioned" - ) - throw SimulatorManagerError.leaserExited - } - - // `getSimulator()` will increment the reference count for the simulator - let (simulator, slotIndex) = try await getSimulator(for: config, exclusive: exclusive) - - _ = recentlyLeased.insert(config) - - // Re-check now that provisioning is done: the leaser may have exited while we - // were cloning and booting. Record the lease first so `release` can unwind the - // reference count and put the device back in the idle pool, then hand it back - // rather than returning a UDID nobody will use. - leases[leaser] = .init( - udid: simulator, - config: config, - exclusive: exclusive, - slotIndex: slotIndex, - leaserStartTime: processStartTime(leaser) - ) - persistLeases() - - if deleteOnPIDExit, !processIsRunning(leaser) { - Logger.simulatorManager.info( - """ - 👋 PID \(leaser, privacy: .public) exited while its simulator was being \ - provisioned; returning \(simulator, privacy: .public) to the pool - """ - ) - try await release(for: leaser) - throw SimulatorManagerError.leaserExited - } - - Logger.simulatorManager.info( - "🔒 Leased simulator \(simulator, privacy: .public) to PID \(leaser, privacy: .public)" - ) - - if deleteOnPIDExit { - registerReleaseOnExit(for: leaser) - } - - return simulator - } - - /// The number of leases whose leasing process is still running. - /// - /// Exposed over HTTP for diagnostics -- "was anything actually leased when this - /// daemon was replaced?" is otherwise hard to answer after the fact. Leases held - /// by exited processes are excluded, since those are already reclaimed or about - /// to be. - func liveLeaseCount() -> Int { - return leases.keys.count(where: { processIsRunning($0) }) - } - - func release(for leaser: PID) async throws { - guard let lease = leases.removeValue(forKey: leaser) else { - // If the manager recently restarted, we might not have the state of all leases. Since - // `SimulatorControl` will return existing simulators matching a given name, the dangling - // simulator will eventually get picked back up again and properly reference counted. So we - // will return an error here, and ignore it in the test runner. - throw SimulatorManagerError.noLease - } - - Logger.simulatorManager.info( - "🔓 Releasing simulator \(lease.udid, privacy: .public) for PID \(leaser, privacy: .public)" - ) - - // Recorded before the device is torn down, so a daemon replaced mid-release - // does not adopt a lease whose simulator is already going away. - persistLeases() - - removeReleaseOnExit(for: leaser) - - await simulatorControl.cleanTempFiles(in: lease.udid) - - try await decrementReferenceCount( - for: lease.udid, - config: lease.config, - slotIndex: lease.slotIndex - ) - } - - private func getBase( - for config: SimulatorConfig - ) async throws -> SimulatorUDID { - if let existingTask = getBaseSimulatorTasks[config] { - return try await existingTask.value - } - - // We use a task to prevent data races that can occur when the `await` on `simulatorControl` - // blocks. This ensures that multiple callers trying to get a base simulator will all wait - // for the same simulator to be returned. - let task = Task { - defer { - getBaseSimulatorTasks.removeValue(forKey: config) - } - - Logger.simulatorManager.info("📱 Creating \(config, privacy: .public) base simulator") - - let baseSimulator = - try await simulatorControl - .createBase( - name: config.baseDeviceName(), - with: config, - runtimeIdentifier: config.runtimeIdentifier() - ) - - Logger.simulatorManager.info( - "📱 Created \(config, privacy: .public) base simulator: \(baseSimulator, privacy: .public)" - ) - - return baseSimulator - } - - getBaseSimulatorTasks[config] = task - - return try await task.value - } - - private func incrementReferenceCount(for simulator: SimulatorUDID) { - var count = referenceCount[simulator] ?? 0 - count += 1 - referenceCount[simulator] = count - - Logger.simulatorManager.debug( - """ - 🔼 Reference count for simulator \(simulator, privacy: .public) is now \ - \(count, privacy: .public) - """ - ) - } - - private func decrementReferenceCount( - for simulator: SimulatorUDID, - config: SimulatorConfig, - slotIndex: Int - ) async throws { - guard var count = referenceCount[simulator] else { - // Simulator was already deleted, nothing to do - return - } - - count -= 1 - referenceCount[simulator] = count - - Logger.simulatorManager.debug( - "🔽 Reference count for \(simulator, privacy: .public) is now \(count, privacy: .public)" - ) - - guard count == 0 else { - return - } - - // Wait a bit before deleting simulators, to allow them to be reused - await pendingDeletion(simulator, config: config, slotIndex: slotIndex) - } - - // Warning: We must update slots before we `await` on anything in this function (unless that - // method updates slots before `await`ing on anything). - private func getSimulator( - for config: SimulatorConfig, - exclusive: Bool - ) async throws -> (simulator: SimulatorUDID, slotIndex: Int) { - if simulatorSlots.keys.contains(config) == false { - simulatorSlots[config] = [] - } - - // Need to sort so we reuse the the correct slots - let sortedSlots = simulatorSlots[config]!.enumerated().sorted { lhs, rhs in - let lhsSortOrder = lhs.element.sortOrder - let rhsSortOrder = rhs.element.sortOrder - - guard lhsSortOrder == rhsSortOrder else { - // Sort by sort order first - return lhsSortOrder < rhsSortOrder - } - - // If the sort order is the same, sort by index - return lhs.offset < rhs.offset - } - - for (index, slot) in sortedSlots { - switch slot { - case .active(let simulator, false) where exclusive != true: - // We have an active non-exclusive simulator, so reuse it - return try await ( - reuseSimulator(simulator, config: config, exclusive: exclusive, slotIndex: index), - slotIndex: index - ) - - case .pendingDeletion(let simulator, let task): - // We have a pending deletion, so we can reuse it - Logger.simulatorManager.info( - """ - ♻️ Turning a pending deletion of simulator \(simulator, privacy: .public) into an \ - active \(exclusive ? "exclusive" : "non-exclusive", privacy: .public) simulator - """ - ) - - simulatorSlots[config]![index] = .active(simulator, exclusive: exclusive) - - task.cancel() - - return try await ( - reuseSimulator(simulator, config: config, exclusive: exclusive, slotIndex: index), - slotIndex: index - ) - - case .empty: - let task = createCloneTask(config: config, exclusive: exclusive, slotIndex: index) - simulatorSlots[config]![index] = .pendingCreation(task, exclusive: exclusive) - return try await (task.value, slotIndex: index) - - case .pendingCreation(let task, false) where exclusive == false: - // We have a non-exclusive simulator pending creation, so reuse it - let simulator = try await task.value - - // We call `incrementReferenceCount()` instead of `reuseSimulator()` here, because the - // simulator is freshly created, so we can (hopefully) assume it is in a good state - incrementReferenceCount(for: simulator) - - return (simulator, slotIndex: index) - - default: - // Ignore incompatible slots - break - } - } - - // If we got here, we need to add a new slot - let index = simulatorSlots[config]!.count - let task = createCloneTask(config: config, exclusive: exclusive, slotIndex: index) - simulatorSlots[config]!.append(.pendingCreation(task, exclusive: exclusive)) - return try await (task.value, slotIndex: index) - } - - private func reuseSimulator( - _ simulator: SimulatorUDID, - config: SimulatorConfig, - exclusive: Bool, - slotIndex: Int - ) async throws -> SimulatorUDID { - incrementReferenceCount(for: simulator) - - do { - // Wait for it to boot. This shouldn't be necessary, but sometimes the simulator will - // reboot because of a migration. This also guards against a simulator being deleted out - // from under us, as it will error, and we can then "delete" it and return a new one. - try await simulatorControl.ensureBooted( - simulator, - context: "getSimulator, reused: \(config.cloneDeviceName(index: slotIndex))" - ) - - return simulator - } catch let error as ProcessError { - // 148 happens for "Invalid device". So it either has already been deleted or it's corrupt - // in some way. Either way, we will "delete" it and return a new one. - guard error.exitCode == 148 else { - throw error - } - - Logger.simulatorManager.warning( - """ - ⚠️ Boot of existing simulator \(simulator, privacy: .public) failed; deleting and \ - returning a new simulator: \(error, privacy: .public) - """ - ) - - // If we fail to delete, don't throw an error - try? await delete( - simulator, - config: config, - slotIndex: slotIndex, - // We can't clean up slots, since we assign to it below - cleanUpSlots: false, - context: "getSimulator, reused: \(config.cloneDeviceName(index: slotIndex))" - ) - - let task = createCloneTask(config: config, exclusive: exclusive, slotIndex: slotIndex) - simulatorSlots[config]![slotIndex] = .pendingCreation(task, exclusive: exclusive) - return try await task.value - } - } - - private func createCloneTask( - config: SimulatorConfig, - exclusive: Bool, - slotIndex: Int - ) -> Task { - return Task { - do { - let simulator = try await simulatorControl.clone( - getBase(for: config), - name: config.cloneDeviceName(index: slotIndex), - deviceType: config.deviceType, - runtimeIdentifier: config.runtimeIdentifier(), - postBoot: postBoot - ) - - simulatorSlots[config]![slotIndex] = .active(simulator, exclusive: exclusive) - - // We want to increment the reference count as soon as we get back from `await`, to ensure - // that when we suspend and potentially decrement the reference count, we don't delete the - // simulator before we have a chance to use it. Also, since we created the simulator, we - // should be responsible for incrementing the reference count. Any functions that reuse - // this task need to increment the reference count as well. - incrementReferenceCount(for: simulator) - - return simulator - } catch { - // If we fail to create the clone, we need to empty the slot, instead - // of leaving it in a pending state - simulatorSlots[config]![slotIndex] = .empty - - throw error - } - } - } - - private func registerReleaseOnExit(for leaser: PID) { - let processSource = - DispatchSource.makeProcessSource(identifier: leaser, eventMask: .exit, queue: .main) - - // Avoid double handling of exit in case the process exits between - // `processSource.resume()` and the check with `kill` below. The event handler runs - // on `.main`, while the liveness check below runs on whatever thread calls this - // method, so the flag guarding against a double call must itself be synchronized. - let handledExitLock = NSLock() - var handledExit = false - let onExitHandler: () -> Void = { [weak self] in - handledExitLock.lock() - let alreadyHandled = handledExit - handledExit = true - handledExitLock.unlock() - guard !alreadyHandled else { return } - - Task { - guard let self else { return } - - Logger.simulatorManager.debug("👋 PID \(leaser, privacy: .public) exited") - - try await self.release(for: leaser) - } - } - - processSource.setEventHandler { onExitHandler() } - processSource.resume() - - // Check to see if the process is already dead. There is no `setCancelHandler`, so - // handle the exit directly here rather than relying on cancellation to do it. - guard processIsRunning(leaser) else { - processSource.cancel() - onExitHandler() - return - } - - leaserExitListeners[leaser] = processSource - } - - private func removeReleaseOnExit(for leaser: PID) { - guard let leaserExitListener = leaserExitListeners[leaser] else { return } - leaserExitListeners.removeValue(forKey: leaser) - leaserExitListener.cancel() - } - - private func pendingDeletion( - _ simulator: SimulatorUDID, - config: SimulatorConfig, - slotIndex: Int - ) async { - guard deleteIdleAfter > 0 || deleteRecentlyUsedIdleAfter > 0 else { - // If we fail to delete, don't throw an error - try? await delete( - simulator, - config: config, - slotIndex: slotIndex, - cleanUpSlots: true, - context: "pendingDeletion immediate" - ) - return - } - - let task = Task { - Logger.simulatorManager.info( - """ - 💤 Scheduling delete of simulator \(simulator, privacy: .public) in \ - \(self.deleteIdleAfter, privacy: .public) to \ - \(self.deleteRecentlyUsedIdleAfter, privacy: .public) seconds - """ - ) - - let now = Date() - let shortDeadline = now.addingTimeInterval(TimeInterval(deleteIdleAfter)) - let recentlyUsedDeadline = now.addingTimeInterval(TimeInterval(deleteRecentlyUsedIdleAfter)) - - while true { - let remainingTime: TimeInterval - if recentlyLeased.contains(config) { - remainingTime = recentlyUsedDeadline.timeIntervalSinceNow - } else { - remainingTime = shortDeadline.timeIntervalSinceNow - } - - if remainingTime <= 0 { - break - } - - // Sleep for up-to 1 second before next check - try await Task.sleep(for: .seconds(min(remainingTime, 1))) - } - - guard case .pendingDeletion(let slotSimulator, _) = simulatorSlots[config]![slotIndex], - simulator == slotSimulator else { - // Simulator was reused, no need to delete - return - } - - // If we fail to delete, don't throw an error - try? await delete( - simulator, - config: config, - slotIndex: slotIndex, - cleanUpSlots: true, - context: "pendingDeletion delayed" - ) - } - - simulatorSlots[config]![slotIndex] = .pendingDeletion(simulator, task) - } - - private func delete( - _ simulator: SimulatorUDID, - config: SimulatorConfig, - slotIndex: Int, - cleanUpSlots: Bool, - context: @escaping @autoclosure () -> String? - ) async throws { - let name = config.cloneDeviceName(index: slotIndex) - - Logger.simulatorManager.info( - "🗑️ Deleting simulator \(simulator, privacy: .public) (\(name, privacy: .public))" - ) - - simulatorSlots[config]![slotIndex] = .deleting(simulator) - - referenceCount.removeValue(forKey: simulator) - - defer { - // Even if we fail to delete, we need to set the slot to empty - simulatorSlots[config]![slotIndex] = .empty - - if cleanUpSlots { - // Shorten up the array by removing any empty slots at the end - while case .empty = simulatorSlots[config]!.last { - simulatorSlots[config]!.removeLast() - } - } - } - - try await simulatorControl.delete(simulator, name: name, context: context()) - - Logger.simulatorManager.info( - "🗑️ Deleted simulator \(simulator, privacy: .public) (\(name, privacy: .public)" - ) - } - - // MARK: Child Process Management - - private nonisolated func createStartChildProcessTask(path: String) -> Task { - return Task.detached { [weak self] in - let process: Process - do { - guard let self else { return } - process = try await self.createProcess(path: path) - // `self` drops out of scope here, so `SimulatorManager` can deinit - } catch { - Logger.simulatorManager.info( - """ - ❌ Failed to create child process at "\(path, privacy: .public)": \ - \(error, privacy: .public) - """ - ) - return - } - - await withCheckedContinuation { cont in - process.terminationHandler = { proc in - let exitCode = proc.terminationStatus - Logger.simulatorManager.warning( - """ - ⚠️ "\(path, privacy: .public)" exited with code: \(exitCode, privacy: .public) - """ - ) - cont.resume() - } - - do { - Logger.simulatorManager.info( - #"🧒 Starting "\#(path, privacy: .public)""# - ) - try process.run() - } catch { - Logger.simulatorManager.info( - """ - ❌ Failed to start "\(path, privacy: .public)": \ - \(error, privacy: .public) - """ - ) - cont.resume() - } - } - } - } - - private func createProcess(path: String) throws -> Process { - let process = Process() - - process.executableURL = URL(fileURLWithPath: path) - - let outPTY = try PTY() - process.standardOutput = FileHandle(fileDescriptor: outPTY.child, closeOnDealloc: true) - let outQueue = DispatchQueue(label: "com.example.simulator_manager.child_process.out") - let outWatcher = watch(fd: outPTY.parent, queue: outQueue) { line in - Logger.childProcess.info("[\(path, privacy: .public)] \(line, privacy: .public)") - } - - let errPTY = try PTY() - process.standardError = FileHandle(fileDescriptor: errPTY.child, closeOnDealloc: true) - let errQueue = DispatchQueue(label: "com.example.simulator_manager.child_process.err") - let errWatcher = watch(fd: errPTY.parent, queue: errQueue) { line in - Logger.childProcess.error("[\(path, privacy: .public)] \(line, privacy: .public)") - } - - childProcesses[path] = (process, outWatcher, errWatcher) - - return process - } -} - -/// Installs a DispatchSourceRead on `fd`. -private func watch( - fd: Int32, - queue: DispatchQueue, - onLine: @escaping (String) -> Void -) -> DispatchSourceRead { - let src = DispatchSource.makeReadSource(fileDescriptor: fd, queue: queue) - var buffer = Data() - - src.setEventHandler { - var tmp = [UInt8](repeating: 0, count: 4096) - let n = read(fd, &tmp, tmp.count) - guard n > 0 else { - src.cancel() - close(fd) - return - } - - buffer.append(contentsOf: tmp[0..?) { - let response = unwrapOutboundIn(data) - - write(context: context, response: response) - } - - private func write(context: ChannelHandlerContext, response: SimulatorManagerResponse) { - let message = response.message + "\n" - - context.write( - wrapOutboundOut( - .init( - head: .init( - version: .http1_1, - status: response.status, - headers: .defaultHeaders(for: message) - ), - body: context.channel.allocator.buffer(string: message) - ) - ), - promise: nil - ) - } -} - -extension HTTPHeaders { - static func defaultHeaders(for message: String) -> HTTPHeaders { - var headers = HTTPHeaders() - headers.add(name: "Content-Length", value: "\(message.utf8.count)") - headers.add(name: "Content-Type", value: "text/plain") - return headers - } -} diff --git a/#/SimulatorRequestHandler.swift b/#/SimulatorRequestHandler.swift deleted file mode 100644 index 82ada11..0000000 --- a/#/SimulatorRequestHandler.swift +++ /dev/null @@ -1,104 +0,0 @@ -import NIOHTTP1 - -final class SimulatorRequestHandler { - private let simulatorManager: SimulatorManager - - init(simulatorManager: SimulatorManager) { - self.simulatorManager = simulatorManager - } - - /// The number of leases still held by running processes, for diagnostics. - func liveLeaseCount() async -> Int { - return await simulatorManager.liveLeaseCount() - } - - func handleRequest( - method: HTTPMethod, - pathComponents: [String], - queryParameters: [String: String] - ) async throws -> SimulatorManagerResponse { - switch method { - case .POST: - guard pathComponents.count >= 1 else { - return .init( - status: .badRequest, - message: "Must specify " - ) - } - - guard let leaser = PID(pathComponents[0]) else { - return .init(status: .badRequest, message: "Leaser PID must be an integer") - } - guard let exclusiveString = queryParameters["exclusive"] else { - return .init(status: .badRequest, message: "Must specify 'exclusive' query parameter") - } - let exclusive = exclusiveString == "1" - guard let deviceType = queryParameters["deviceType"] else { - return .init(status: .badRequest, message: "Must specify 'deviceType' query parameter") - } - guard let os = queryParameters["os"] else { - return .init(status: .badRequest, message: "Must specify 'os' query parameter") - } - guard let version = queryParameters["version"] else { - return .init(status: .badRequest, message: "Must specify 'version' query parameter") - } - - let config = SimulatorConfig( - deviceType: deviceType, - os: os, - version: version - ) - - do { - return try await .init( - status: .created, - message: simulatorManager - .lease(to: leaser, exclusive: exclusive, config: config) - ) - } catch SimulatorManagerError.alreadyLeased(let udid) { - return .init( - status: .badRequest, - // FIXME: Get this from the error itself - message: "PID \(leaser) has already leased another simulator: \(udid)" - ) - } catch SimulatorManagerError.leaserExited { - // Nobody is left to read this response, but answering rather than throwing - // keeps it out of the server's error path. - return .init( - status: .gone, - message: "PID \(leaser) exited before its simulator was provisioned" - ) - } - - case .DELETE: - guard pathComponents.count >= 1 else { - return .init( - status: .badRequest, - message: "Must specify " - ) - } - - guard let leaser = PID(pathComponents[0]) else { - return .init(status: .badRequest, message: "Leaser PID must be an integer") - } - - do { - try await simulatorManager.release(for: leaser) - - return .init( - status: .ok, - message: "Success" - ) - } catch SimulatorManagerError.noLease { - return .init( - status: .notFound, - // FIXME: Get this message from the error itself - message: "PID \(leaser) doesn't have a simulator leased" - ) - } - - default: - return .init(status: .methodNotAllowed, message: "Unsupported HTTP method: \(method)") - } - } -} From 8ce6bb51db2c062b9d00b3beff439aef0b7e9b18 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Fri, 21 Aug 2026 09:50:38 +0200 Subject: [PATCH 09/11] more --- MODULE.bazel | 2 +- go.mod | 4 +- go.sum | 2 + tools/simulator_manager/BUILD | 2 +- .../Sources/ORPHAN_SIMULATOR_FIX.md | 36 +- tools/simulator_manager/Sources/README.md | 16 +- tools/simulator_manager/go/BUILD.bazel | 25 + tools/simulator_manager/go/README.md | 162 ++++ tools/simulator_manager/go/http_server.go | 119 +++ tools/simulator_manager/go/lease_store.go | 107 +++ tools/simulator_manager/go/logger.go | 21 + tools/simulator_manager/go/lru_set.go | 58 ++ tools/simulator_manager/go/main.go | 113 +++ tools/simulator_manager/go/pty.go | 24 + .../simulator_manager/go/simulator_control.go | 453 ++++++++++ .../simulator_manager/go/simulator_manager.go | 823 ++++++++++++++++++ .../go/simulator_request_handler.go | 166 ++++ tools/simulator_manager/start.sh | 2 +- 18 files changed, 2117 insertions(+), 18 deletions(-) create mode 100644 tools/simulator_manager/go/BUILD.bazel create mode 100644 tools/simulator_manager/go/README.md create mode 100644 tools/simulator_manager/go/http_server.go create mode 100644 tools/simulator_manager/go/lease_store.go create mode 100644 tools/simulator_manager/go/logger.go create mode 100644 tools/simulator_manager/go/lru_set.go create mode 100644 tools/simulator_manager/go/main.go create mode 100644 tools/simulator_manager/go/pty.go create mode 100644 tools/simulator_manager/go/simulator_control.go create mode 100644 tools/simulator_manager/go/simulator_manager.go create mode 100644 tools/simulator_manager/go/simulator_request_handler.go diff --git a/MODULE.bazel b/MODULE.bazel index a3ee6a5..3b82a80 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -99,7 +99,7 @@ go_sdk.download(version = GO_VERSION) go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") -use_repo(go_deps, "com_github_google_go_cmp") +use_repo(go_deps, "com_github_google_go_cmp", "org_golang_x_sys") bazel_dep(name = "rules_java", version = "9.6.1") diff --git a/go.mod b/go.mod index 49d14c0..202b1bd 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/EngFlow/example -go 1.21.2 +go 1.25.0 require github.com/google/go-cmp v0.6.0 + +require golang.org/x/sys v0.47.0 diff --git a/go.sum b/go.sum index 5a8d551..e028c78 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/tools/simulator_manager/BUILD b/tools/simulator_manager/BUILD index 54fda20..115cc17 100644 --- a/tools/simulator_manager/BUILD +++ b/tools/simulator_manager/BUILD @@ -49,7 +49,7 @@ sh_binary( data = [ ":install_post_boot_script", ":prepare_simulator", - ":simulator_manager", + "//tools/simulator_manager/go", ], visibility = ["//visibility:public"], deps = ["@bazel_tools//tools/bash/runfiles"], diff --git a/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md index 45d9398..46bdcb3 100644 --- a/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md +++ b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md @@ -172,24 +172,34 @@ See the next section for what addresses this. ## Verification status -There is no `BUILD` file or `Package.swift` for this target yet, so it isn't -wired into Bazel and can't be built with `swift build` either. This change -has been verified by code review against the existing invariants (see -`LEASE_LIFECYCLE.md`), not by compiling or running it. Before relying on -this in production: +`bazel build //experiments/yannic/macsimulatormanager/swift:macsimulatormanager` +now succeeds for real (see [`README.md`](README.md#building-it)), and +`bazel-bin/experiments/yannic/macsimulatormanager/swift/macsimulatormanager +--help` runs and prints the expected flags, including +`--reap-interval-seconds`. That confirms the code compiles and links against +`ShellOut`/`ArgumentParser`/SwiftNIO correctly, but not the runtime behavior +described above — there's no CoreSimulator runtime installed on the machine +this was built on (`xcrun simctl list runtimes` is empty there), so the +lease/reaper/escalation logic itself is still verified only by code review +against the invariants in `LEASE_LIFECYCLE.md`, not by exercising it. +Getting the toolchain wired up also required one unplanned fix: this repo's +`common:clang` config enforces `-Werror=sign-compare` globally (including for +host tools), which `rules_swift`'s own bundled `tools/common/process.cc` +doesn't build clean under — exempted via a `--per_file_copt` in `.bazelrc` +rather than patching upstream, mirroring the existing `.pb.cc` exemption +right above it. + +Before relying on this in production: 1. Confirm this branch is actually the source for whatever customers run — - given the missing build target and the `DO NOT MERGE` history noted - above, that's not yet established. If it is, push this branch and get it - merged; if there's a separate deploy path, find it. -2. Set up a build target (Bazel or an ad hoc `Package.swift`) so this code - compiles and can be typechecked — it currently depends on `ShellOut`, - `ArgumentParser`, and SwiftNIO, none of which are vendored here. -3. Manually exercise the restart-orphan path: lease a config, `kill -9` the + given the `DO NOT MERGE` history noted above, that's not yet established. + If it is, push this branch and get it merged; if there's a separate + deploy path, find it. +2. Manually exercise the restart-orphan path: lease a config, `kill -9` the daemon, start a new instance with a short `--reap-interval-seconds`, and confirm the orphaned clone disappears from `xcrun simctl list devices` within two sweep intervals without touching unrelated simulators. -4. Manually exercise the wedged-device escalation path: lease a config, then +3. Manually exercise the wedged-device escalation path: lease a config, then independently hang or otherwise make its clone's `simctl shutdown`/`delete` fail out-of-band, and confirm the reaper force-kills its `launchd_sim` after 3 failed sweeps rather than retrying forever. diff --git a/tools/simulator_manager/Sources/README.md b/tools/simulator_manager/Sources/README.md index 3d9eda7..1a334af 100644 --- a/tools/simulator_manager/Sources/README.md +++ b/tools/simulator_manager/Sources/README.md @@ -9,9 +9,23 @@ slow and resource-hungry. The daemon speaks HTTP over a Unix domain socket rather than a TCP port, since it is only ever talked to by processes on the same machine. +## Building it + +```bash +bazel build //experiments/yannic/macsimulatormanager/swift:macsimulatormanager +``` + +Third-party dependencies (`ShellOut`, `swift-argument-parser`, SwiftNIO) are +resolved via [`rules_swift_package_manager`](https://github.com/cgrindel/rules_swift_package_manager) +from the `Package.swift` / `Package.resolved` pair in this directory, not +hand-vendored. If you change `Package.swift`, regenerate `Package.resolved` +with `swift package resolve` and run `bazel mod tidy` at the repo root to +pick up any new or changed external repos in `MODULE.bazel`. + ## Running it -`Main.swift` is a `swift-argument-parser` command. Notable options: +`Main.swift` is a `swift-argument-parser` command; `bazel run` works the same +way, with arguments after `--`. Notable options: - `--pid-path` / `--unix-socket-path` — where the daemon writes its PID file and creates the listening socket. diff --git a/tools/simulator_manager/go/BUILD.bazel b/tools/simulator_manager/go/BUILD.bazel new file mode 100644 index 0000000..af7e2be --- /dev/null +++ b/tools/simulator_manager/go/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_lib", + srcs = [ + "http_server.go", + "lease_store.go", + "logger.go", + "lru_set.go", + "main.go", + "pty.go", + "simulator_control.go", + "simulator_manager.go", + "simulator_request_handler.go", + ], + importpath = "github.com/EngFlow/example/tools/simulator_manager/go", + visibility = ["//visibility:private"], + deps = ["@org_golang_x_sys//unix"], +) + +go_binary( + name = "go", + embed = [":go_lib"], + visibility = ["//visibility:public"], +) diff --git a/tools/simulator_manager/go/README.md b/tools/simulator_manager/go/README.md new file mode 100644 index 0000000..e8d926b --- /dev/null +++ b/tools/simulator_manager/go/README.md @@ -0,0 +1,162 @@ +# Mac Simulator Manager (Go) + +This is a Go translation of the Swift Mac Simulator Manager located in `../swift/`. + +## Architecture + +The simulator manager is an HTTP server that manages iOS/macOS simulator instances for testing. It provides lease-based access to simulators with the following features: + +- **Lease Management**: PIDs can lease simulators (exclusively or shared) +- **Automatic Cleanup**: Simulators are automatically deleted when idle +- **Persistence**: Leases are persisted to disk so daemon restarts don't lose state +- **Base + Clone Pattern**: Creates base simulators and clones them for efficiency +- **Child Process Management**: Can launch and monitor startup processes + +## Key Components + +### Main Application (`main.go`) +Entry point that parses command-line arguments and starts the server. + +### HTTP Server (`http_server.go`) +Serves requests on a Unix domain socket with endpoints: +- `POST /simulator/` - Lease a simulator +- `DELETE /simulator/` - Release a simulator +- `GET /version` - Get manager version +- `GET /leases` - Get count of live leases +- `POST /shutdown` - Shutdown the server + +### Simulator Manager (`simulator_manager.go`) +Core orchestration logic: +- Manages simulator slots per configuration +- Handles lease/release operations +- Tracks reference counts +- Implements automatic deletion with LRU eviction + +### Simulator Control (`simulator_control.go`) +Low-level interface to `simctl`: +- Creates base simulators +- Clones simulators +- Boots and shuts down simulators +- Deletes simulators + +### Lease Store (`lease_store.go`) +Persists leases to disk in JSON format with atomic writes. Includes process start time to detect PID reuse. + +### Supporting Components +- `lru_set.go` - LRU cache for tracking recently used configurations +- `pty.go` - PTY creation for child process I/O +- `logger.go` - Structured logging setup + +## Translation Notes + +### Differences from Swift + +These are concrete, verified behavioral or structural differences between the +two implementations -- not just "different language, same thing" restatements. + +1. **Concurrency model**: + - Swift's `SimulatorManager` and `SimulatorControl` are actors, so the + compiler serializes access and only allows suspension at explicit + `await` points. Go has no equivalent, so the same invariants are + enforced by hand with `sync.Mutex`, manually unlocking around any + blocking call (channel receive, `simctl` invocation) and re-locking + afterward -- easy to get subtly wrong in a future change, unlike the + actor version. + - Coalescing concurrent callers onto one in-flight operation (`getBase`, + `createBase`, `clone`) uses Swift's `Task`, which is directly + awaitable and memoizes its result. Go instead uses a buffered channel + (`chan taskResult`) that every caller receives from. + - `SimulatorDeleteOrExistenceMutex`, which serializes delete/existence + checks for a given device name, is a hand-rolled actor with an explicit + waiter queue in Swift; Go uses a plain `sync.Mutex`. + +2. **Leaser-exit detection is slower in Go**: + - Swift watches for a leaser's exit with `DispatchSource.makeProcessSource` + (kqueue `EVFILT_PROC`), so `release` runs as soon as the kernel reports + the exit. + - Go's `registerReleaseOnExit` instead polls once a second with + `time.Ticker` + `kill(pid, 0)`, so reclaiming a dead leaser's simulator + can lag up to ~1s behind Swift. (The separate idle-deletion timer in + `pendingDeletion` already polls once a second in *both* implementations, + so that part carried over unchanged.) + +3. **`POST /shutdown` is abrupt in Go, graceful in Swift**: + - Swift's only shutdown path is `POST /shutdown`, which triggers + SwiftNIO's `ServerQuiescingHelper`: stop accepting connections, let + in-flight ones finish, then fall through to the cleanup code that + removes the socket and PID files. + - Go additionally handles SIGINT/SIGTERM by calling `http.Server.Shutdown` + (a comparable graceful drain), but its `POST /shutdown` handler just + calls `os.Exit(0)` from a goroutine after writing the response. That + skips draining any other in-flight requests and skips the deferred + socket/PID-file cleanup in `HTTPServer.Run`, which only runs after + `server.Serve()` returns -- which `os.Exit` prevents. An HTTP-triggered + shutdown in the Go version can leave stale socket/PID files behind. + +4. **Post-boot script failures lose detail in Go**: + - Swift runs the post-boot script through the same `subprocess`/`shellOut` + helper used for every `simctl` call, so a failure comes back as a + `ProcessError` with captured stdout/stderr. + - Go's `Clone` invokes the post-boot script directly with `exec.Command` + instead of the shared `subprocess`/`simctl` helper: stdout/stderr are + discarded rather than captured, and a failure surfaces as a wrapped + `*exec.ExitError` rather than a `*ProcessError`. + +5. **`--post-boot` required vs. optional**: + - Swift declares `postBoot` as a non-optional `String` with no default, so + `swift-argument-parser` requires the flag and fails startup without it. + - Go's flag defaults to `""` and is treated as "skip the post-boot step", + so it's optional. + +6. **Debug logging is unreachable in Go**: + - Go's `slog` handler is hardcoded to `slog.LevelInfo`, so every + `Debug`-level call (reference-count changes, existing-simulator lookups) + is compiled but never emitted, and there's no flag to raise the level. + - Swift's `os.Logger` calls at `.debug` are still captured live by the + unified logging system (visible via `log stream`/Console), just not + persisted long-term. + +7. **HTTP framework**: + - Swift hand-rolls the server on SwiftNIO: a channel pipeline of + `AccumulatedHTTPHandler` (buffers head/body/end into one in-memory + request) followed by `SimulatorManagerHTTPHandler` (parses + method/path/query), with one task per connection. + - Go uses the standard library's `net/http` + `http.ServeMux`, which + already delivers a fully parsed, buffered `*http.Request` per call, so + there's no need for an equivalent two-stage accumulate/parse pipeline. + +8. **Child-process output capture**: + - Swift watches each PTY with `DispatchSource.makeReadSource` + (event-driven, woken by the runloop on readability). + - Go uses pipes instead of PTYs (to avoid CGO) and parks a dedicated + goroutine in a blocking `Read` loop per pipe. This works identically + for line-buffered output capture but wouldn't preserve interactive + terminal behavior if child processes expected a real TTY. + +9. **JSON serialization**: both use built-in JSON support with struct tags; + no behavioral difference here. + +### Build + +```bash +# With Bazel +bazel build //experiments/yannic/macsimulatormanager/go:macsimulatormanager + +# With Go +cd experiments/yannic/macsimulatormanager/go +go build -o macsimulatormanager +``` + +### Usage + +```bash +./macsimulatormanager \ + --version="1.0.0" \ + --pid-path="/tmp/simulator-manager.pid" \ + --unix-socket-path="/tmp/simulator-manager.sock" \ + --delete-recently-used-idle-after=300 \ + --delete-idle-after=60 \ + --recently-used-capacity=1 \ + --post-boot="/path/to/post-boot-script.sh" \ + --lease-path="/tmp/leases.json" +``` diff --git a/tools/simulator_manager/go/http_server.go b/tools/simulator_manager/go/http_server.go new file mode 100644 index 0000000..a641935 --- /dev/null +++ b/tools/simulator_manager/go/http_server.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" +) + +type HTTPServer struct { + simulatorRequestHandler *SimulatorRequestHandler + version string +} + +func NewHTTPServer(simulatorRequestHandler *SimulatorRequestHandler, version string) *HTTPServer { + return &HTTPServer{ + simulatorRequestHandler: simulatorRequestHandler, + version: version, + } +} + +func (s *HTTPServer) Run(pidPath string, unixSocketPath string) error { + if err := os.Remove(unixSocketPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove existing socket: %w", err) + } + + if err := os.Remove(pidPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove existing PID file: %w", err) + } + + pid := os.Getpid() + if err := os.WriteFile(pidPath, []byte(strconv.Itoa(pid)), 0644); err != nil { + return fmt.Errorf("failed to write PID file: %w", err) + } + + listener, err := net.Listen("unix", unixSocketPath) + if err != nil { + return fmt.Errorf("failed to listen on unix socket: %w", err) + } + + httpServerLogger.Info("Server running on UDS", "path", unixSocketPath) + + mux := http.NewServeMux() + mux.HandleFunc("/simulator", s.handleSimulatorRequest) + mux.HandleFunc("/simulator/", s.handleSimulatorRequest) + mux.HandleFunc("/version", s.handleVersionRequest) + mux.HandleFunc("/leases", s.handleLeasesRequest) + mux.HandleFunc("/shutdown", s.handleShutdownRequest) + + server := &http.Server{Handler: mux} + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + go func() { + <-sigChan + httpServerLogger.Info("Shutting down server") + _ = server.Shutdown(context.Background()) + }() + + err = server.Serve(listener) + if err != nil && err != http.ErrServerClosed { + return err + } + + httpServerLogger.Info("Server shut down") + + _ = os.Remove(unixSocketPath) + _ = os.Remove(pidPath) + + return nil +} + +func (s *HTTPServer) handleSimulatorRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + + response := s.simulatorRequestHandler.HandleRequest(r.Method, r.URL.Path, r.URL.Query()) + + accumulatedHTTPLogger.Info("Sending response", "status", response.Status) + + w.WriteHeader(response.Status) + fmt.Fprintf(w, "%s\n", response.Message) +} + +func (s *HTTPServer) handleVersionRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + accumulatedHTTPLogger.Info("Sending response", "status", http.StatusOK) + + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "%s\n", s.version) +} + +func (s *HTTPServer) handleLeasesRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + + count := s.simulatorRequestHandler.LiveLeaseCount() + + accumulatedHTTPLogger.Info("Sending response", "status", http.StatusOK) + + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "%d\n", count) +} + +func (s *HTTPServer) handleShutdownRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + + httpServerLogger.Info("Shutdown request received") + + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "Server shutting down\n") + + go func() { + os.Exit(0) + }() +} diff --git a/tools/simulator_manager/go/lease_store.go b/tools/simulator_manager/go/lease_store.go new file mode 100644 index 0000000..e0a3ef1 --- /dev/null +++ b/tools/simulator_manager/go/lease_store.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "os" + + "golang.org/x/sys/unix" +) + +type SimulatorUDID = string + +// PersistedLease is a lease, in the form it takes on disk. +// +// Holds everything needed to rebuild the daemon's in-memory bookkeeping for one +// lease: which device, under which configuration, in which slot, and whether it +// was owned exclusively. Nothing here has to be re-derived by asking simctl. +type PersistedLease struct { + PID int32 `json:"pid"` + LeaserStartTime *uint64 `json:"leaserStartTime,omitempty"` + UDID SimulatorUDID `json:"udid"` + Config SimulatorConfig `json:"config"` + Exclusive bool `json:"exclusive"` + SlotIndex int `json:"slotIndex"` +} + +// LeaseStore is where the daemon keeps its leases so a successor can pick them up. +type LeaseStore interface { + // Save replaces the stored set with leases. Failures are logged, not thrown: + // losing persistence degrades a restart, but failing the lease that triggered + // the write would break a test that is otherwise fine. + Save(leases []PersistedLease) + + // Load returns the stored set, or empty if there is nothing readable to restore. + Load() []PersistedLease +} + +// FileLeaseStore is a LeaseStore backed by a JSON file. +// +// Written atomically, so a reader never sees a half-written set and a crash +// mid-write leaves the previous contents intact. Combined with writing on every +// change, this means the file is always current -- there is no flush-on-shutdown +// step for a kill -9 to skip. +type FileLeaseStore struct { + path string +} + +func NewFileLeaseStore(path string) *FileLeaseStore { + return &FileLeaseStore{path: path} +} + +func (f *FileLeaseStore) Save(leases []PersistedLease) { + data, err := json.MarshalIndent(leases, "", " ") + if err != nil { + leaseStoreLogger.Error("Failed to marshal leases", "error", err, "count", len(leases), "path", f.path) + return + } + + // Write atomically using a temp file and rename + tmpPath := f.path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0644); err != nil { + leaseStoreLogger.Error("Failed to write leases", "error", err, "count", len(leases), "path", f.path) + return + } + + if err := os.Rename(tmpPath, f.path); err != nil { + leaseStoreLogger.Error("Failed to rename leases file", "error", err, "count", len(leases), "path", f.path) + return + } +} + +func (f *FileLeaseStore) Load() []PersistedLease { + if _, err := os.Stat(f.path); os.IsNotExist(err) { + return nil + } + + data, err := os.ReadFile(f.path) + if err != nil { + leaseStoreLogger.Error("Failed to read leases; continuing with none", "error", err, "path", f.path) + return nil + } + + var leases []PersistedLease + if err := json.Unmarshal(data, &leases); err != nil { + leaseStoreLogger.Error("Failed to decode leases; continuing with none", "error", err, "path", f.path) + return nil + } + + return leases +} + +// processStartTime returns when pid started, in microseconds since the epoch, or nil if it cannot be +// determined (typically because the process is gone). +// +// A PID alone does not identify a process across a daemon restart: PIDs are +// recycled, so a lease for a dead PID could otherwise be restored onto whatever +// unrelated process now holds that number, tying up a device until it exits. +// Start time distinguishes them -- the kernel assigns it at fork, so a recycled +// PID has a different one. +func processStartTime(pid int32) *uint64 { + info, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid)) + if err != nil { + return nil + } + + startTime := uint64(info.Proc.P_starttime.Sec)*1_000_000 + uint64(info.Proc.P_starttime.Usec) + return &startTime +} diff --git a/tools/simulator_manager/go/logger.go b/tools/simulator_manager/go/logger.go new file mode 100644 index 0000000..e788273 --- /dev/null +++ b/tools/simulator_manager/go/logger.go @@ -0,0 +1,21 @@ +package main + +import ( + "log/slog" + "os" +) + +var ( + logger = newLogger("manager") + childProcessLogger = newLogger("manager.child-process") + httpServerLogger = newLogger("server") + accumulatedHTTPLogger = newLogger("accumulated_http") + simulatorControlLogger = newLogger("control") + leaseStoreLogger = newLogger("manager.lease-store") +) + +func newLogger(category string) *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })).With("subsystem", "com.example.tools.simulator_manager", "category", category) +} diff --git a/tools/simulator_manager/go/lru_set.go b/tools/simulator_manager/go/lru_set.go new file mode 100644 index 0000000..45ba636 --- /dev/null +++ b/tools/simulator_manager/go/lru_set.go @@ -0,0 +1,58 @@ +package main + +// LRUSet is a Set that has a maximum capacity and evicts the least recently used item when full. +type LRUSet[T comparable] struct { + capacity int + // An array to keep track of the order in which elements were inserted. + // The first element in the array is the least recently used. + order []T + // A map to enable fast O(1) membership tests. + storage map[T]struct{} +} + +func NewLRUSet[T comparable](capacity int) *LRUSet[T] { + if capacity <= 0 { + panic("Capacity must be greater than zero.") + } + return &LRUSet[T]{ + capacity: capacity, + order: make([]T, 0, capacity), + storage: make(map[T]struct{}), + } +} + +// Insert returns an element that was evicted from the set. +func (s *LRUSet[T]) Insert(element T) *T { + var evicted *T + if _, exists := s.storage[element]; exists { + // Remove from current position in order + for i, e := range s.order { + if e == element { + old := s.order[i] + evicted = &old + s.order = append(s.order[:i], s.order[i+1:]...) + break + } + } + s.order = append(s.order, element) + } else { + if len(s.order) >= s.capacity && len(s.order) > 0 { + oldest := s.order[0] + s.order = s.order[1:] + delete(s.storage, oldest) + evicted = &oldest + } + s.order = append(s.order, element) + s.storage[element] = struct{}{} + } + return evicted +} + +func (s *LRUSet[T]) Contains(element T) bool { + _, exists := s.storage[element] + return exists +} + +func (s *LRUSet[T]) Elements() []T { + return s.order +} diff --git a/tools/simulator_manager/go/main.go b/tools/simulator_manager/go/main.go new file mode 100644 index 0000000..11199ef --- /dev/null +++ b/tools/simulator_manager/go/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" +) + +type arrayFlags []string + +func (a *arrayFlags) String() string { + return strings.Join(*a, ", ") +} + +func (a *arrayFlags) Set(value string) error { + *a = append(*a, value) + return nil +} + +func main() { + var version string + var pidPath string + var unixSocketPath string + var deleteRecentlyUsedIdleAfter uint + var deleteIdleAfter uint + var recentlyUsedCapacity int + var startupProcesses arrayFlags + var postBoot string + var leasePath string + + flag.StringVar(&version, "version", "", "Version of the simulator manager") + flag.StringVar(&pidPath, "pid-path", "", "Path to where the pid should be written") + flag.StringVar(&unixSocketPath, "unix-socket-path", "", "Path to where the unix domain socket should be created") + flag.UintVar(&deleteRecentlyUsedIdleAfter, "delete-recently-used-idle-after", 0, "Number of seconds to wait before deleting a recently used idle simulator") + flag.UintVar(&deleteIdleAfter, "delete-idle-after", 0, "Number of seconds to wait before deleting a non-recently used idle simulator") + flag.IntVar(&recentlyUsedCapacity, "recently-used-capacity", 1, "The number of simulators to keep in the recently used list") + flag.Var(&startupProcesses, "startup-process", "The path to a startup process that will be run when the simulator manager is started") + flag.StringVar(&postBoot, "post-boot", "", "Path to an executable that will run after a simulator clone is booted") + flag.StringVar(&leasePath, "lease-path", "", "Path to a file where leases are mirrored") + + flag.Parse() + + if version == "" { + fmt.Fprintln(os.Stderr, "Error: --version is required") + flag.Usage() + os.Exit(1) + } + + if pidPath == "" { + fmt.Fprintln(os.Stderr, "Error: --pid-path is required") + flag.Usage() + os.Exit(1) + } + + if unixSocketPath == "" { + fmt.Fprintln(os.Stderr, "Error: --unix-socket-path is required") + flag.Usage() + os.Exit(1) + } + + if recentlyUsedCapacity <= 0 { + fmt.Fprintln(os.Stderr, "Error: --recently-used-capacity must be greater than 0") + os.Exit(1) + } + + seen := make(map[string]bool) + for _, proc := range startupProcesses { + if seen[proc] { + fmt.Fprintf(os.Stderr, "Error: --startup-process must be unique, found duplicate: %s\n", proc) + os.Exit(1) + } + seen[proc] = true + } + + var leaseStore LeaseStore + if leasePath != "" { + leaseStore = NewFileLeaseStore(leasePath) + } + + var postBootPtr *string + if postBoot != "" { + postBootPtr = &postBoot + } + + simulatorManager := NewSimulatorManager( + NewRealSimulatorControl(), + uint16(deleteRecentlyUsedIdleAfter), + uint16(deleteIdleAfter), + recentlyUsedCapacity, + true, // deleteOnPIDExit + startupProcesses, + postBootPtr, + leaseStore, + ) + + simulatorManager.RestoreLeases() + + if err := simulatorManager.StartChildProcesses(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to start child processes: %v\n", err) + os.Exit(1) + } + + httpServer := NewHTTPServer( + NewSimulatorRequestHandler(simulatorManager), + version, + ) + + if err := httpServer.Run(pidPath, unixSocketPath); err != nil { + fmt.Fprintf(os.Stderr, "Server error: %v\n", err) + os.Exit(1) + } +} diff --git a/tools/simulator_manager/go/pty.go b/tools/simulator_manager/go/pty.go new file mode 100644 index 0000000..ac0bb00 --- /dev/null +++ b/tools/simulator_manager/go/pty.go @@ -0,0 +1,24 @@ +package main + +import ( + "os" +) + +// PTY represents a pseudo-terminal pair (replaced with pipe for simplicity). +// For child process stdout/stderr capture, pipes work just as well as PTYs +// and don't require CGO. +type PTY struct { + Parent int + Child int +} + +func NewPTY() (*PTY, error) { + r, w, err := os.Pipe() + if err != nil { + return nil, err + } + return &PTY{ + Parent: int(r.Fd()), + Child: int(w.Fd()), + }, nil +} diff --git a/tools/simulator_manager/go/simulator_control.go b/tools/simulator_manager/go/simulator_control.go new file mode 100644 index 0000000..7c15165 --- /dev/null +++ b/tools/simulator_manager/go/simulator_control.go @@ -0,0 +1,453 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type SimulatorConfig struct { + DeviceType string `json:"deviceType"` + OS string `json:"os"` + Version string `json:"version"` +} + +func (c SimulatorConfig) String() string { + return fmt.Sprintf("%s (%s %s)", c.DeviceType, c.OS, c.Version) +} + +func (c SimulatorConfig) BaseDeviceName() string { + return fmt.Sprintf("EXAMPLE_BAZEL_BASE_%s_%s", c.DeviceType, c.Version) +} + +func (c SimulatorConfig) CloneDeviceName(index int) string { + return fmt.Sprintf("EXAMPLE_BAZEL_CLONE_%s_%s_%d", c.DeviceType, c.Version, index) +} + +func (c SimulatorConfig) RuntimeIdentifier() string { + runtimeVersion := strings.ReplaceAll(c.Version, ".", "-") + return fmt.Sprintf("com.apple.CoreSimulator.SimRuntime.%s-%s", c.OS, runtimeVersion) +} + +type SimCtlDevices struct { + Devices map[string][]SimCtlDevice `json:"devices"` +} + +type SimCtlDevice struct { + Name string `json:"name"` + UDID string `json:"udid"` +} + +type ProcessError struct { + Command string + Context string + ExitCode int + StdOut string + StdErr string +} + +func (e *ProcessError) Error() string { + contextStr := "" + if e.Context != "" { + contextStr = fmt.Sprintf(" (%s)", e.Context) + } + return fmt.Sprintf("\"%s\"%s failed with exit code %d:\n%s%s", + e.Command, contextStr, e.ExitCode, e.StdOut, e.StdErr) +} + +type SimulatorControl interface { + // CreateBase creates a base simulator with the given config. + // + // It also boots and shuts down the simulator, making it ready for cloning. + // + // If an existing simulator with the same name already exists, that is returned instead of + // creating a new one. This is to support the simulator manager being restarted and losing state. + CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) + + // Clone clones a base simulator. + // + // It also boots the cloned simulator, making it ready for use. + // + // If an existing simulator with the same name already exists, that is returned instead of + // creating a new one. This is to support the simulator manager being restarted and losing state. + Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) + + EnsureBooted(simulator SimulatorUDID, context string) error + + CleanTempFiles(simulator SimulatorUDID) + + Delete(simulator SimulatorUDID, name string, context string) error + + GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) +} + +type RealSimulatorControl struct { + createBaseTasks map[string]chan taskResult + createBaseTasksLock sync.Mutex + + cloneTasks map[string]chan taskResult + cloneTasksLock sync.Mutex + + deleteAndExistenceMutexes map[string]*deleteOrExistenceMutexEntry + deleteAndExistenceMutexesLock sync.Mutex +} + +type taskResult struct { + udid SimulatorUDID + err error +} + +type deleteOrExistenceMutexEntry struct { + mutex *SimulatorDeleteOrExistenceMutex + count int +} + +func NewRealSimulatorControl() *RealSimulatorControl { + return &RealSimulatorControl{ + createBaseTasks: make(map[string]chan taskResult), + cloneTasks: make(map[string]chan taskResult), + deleteAndExistenceMutexes: make(map[string]*deleteOrExistenceMutexEntry), + } +} + +func (r *RealSimulatorControl) CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { + r.createBaseTasksLock.Lock() + if existingTask, ok := r.createBaseTasks[name]; ok { + r.createBaseTasksLock.Unlock() + result := <-existingTask + return result.udid, result.err + } + + resultChan := make(chan taskResult, 1) + r.createBaseTasks[name] = resultChan + r.createBaseTasksLock.Unlock() + + go func() { + defer func() { + r.createBaseTasksLock.Lock() + delete(r.createBaseTasks, name) + r.createBaseTasksLock.Unlock() + }() + + udid, err := r.createBaseImpl(name, config, runtimeIdentifier) + resultChan <- taskResult{udid: udid, err: err} + }() + + result := <-resultChan + return result.udid, result.err +} + +func (r *RealSimulatorControl) createBaseImpl(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { + if existingUDID, err := r.GetExisting(name, config.DeviceType, runtimeIdentifier, "createBase"); err == nil && existingUDID != "" { + simulatorControlLogger.Info("Base simulator already exists, skipping creation", "name", name, "udid", existingUDID) + + if err := r.shutdown(existingUDID, "createBase existing: "+name); err != nil { + simulatorControlLogger.Error("Failed to set up base simulator; deleting", "name", name, "udid", existingUDID) + _ = r.Delete(existingUDID, name, "createBase existing: "+name) + return "", err + } + + return existingUDID, nil + } + + simulatorControlLogger.Info("Creating base simulator", "config", config, "name", name) + + udid, err := simctl([]string{"create", name, config.DeviceType, runtimeIdentifier}, "") + if err != nil { + return "", err + } + udid = strings.TrimSpace(udid) + + if err := r.EnsureBooted(udid, "createBase new: "+name); err != nil { + simulatorControlLogger.Error("Failed to set up base simulator; deleting", "name", name, "udid", udid) + _ = r.Delete(udid, name, "createBase new: "+name) + return "", err + } + + // Give the simulator some time to do some post-boot processing + time.Sleep(5 * time.Second) + + if err := r.shutdown(udid, "createBase new: "+name); err != nil { + simulatorControlLogger.Error("Failed to set up base simulator; deleting", "name", name, "udid", udid) + _ = r.Delete(udid, name, "createBase new: "+name) + return "", err + } + + simulatorControlLogger.Info("Created base simulator", "config", config, "name", name, "udid", udid) + + return udid, nil +} + +func (r *RealSimulatorControl) Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { + r.cloneTasksLock.Lock() + if existingTask, ok := r.cloneTasks[name]; ok { + r.cloneTasksLock.Unlock() + result := <-existingTask + return result.udid, result.err + } + + resultChan := make(chan taskResult, 1) + r.cloneTasks[name] = resultChan + r.cloneTasksLock.Unlock() + + go func() { + defer func() { + r.cloneTasksLock.Lock() + delete(r.cloneTasks, name) + r.cloneTasksLock.Unlock() + }() + + udid, err := r.cloneImpl(baseSimulator, name, deviceType, runtimeIdentifier, postBoot) + resultChan <- taskResult{udid: udid, err: err} + }() + + result := <-resultChan + return result.udid, result.err +} + +func (r *RealSimulatorControl) cloneImpl(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { + var udid string + var isExisting bool + + if existingUDID, err := r.GetExisting(name, deviceType, runtimeIdentifier, "clone"); err == nil && existingUDID != "" { + udid = existingUDID + isExisting = true + + simulatorControlLogger.Info("Cloned simulator already exists, skipping creation", "name", name, "udid", udid) + + if err := r.EnsureBooted(udid, "clone, existing: "+name); err != nil { + return "", err + } + } else { + isExisting = false + + simulatorControlLogger.Info("Cloning base simulator", "base", baseSimulator, "name", name) + + var err error + udid, err = simctl([]string{"clone", baseSimulator, name}, "") + if err != nil { + return "", err + } + udid = strings.TrimSpace(udid) + + simulatorControlLogger.Info("Cloned base simulator", "base", baseSimulator, "name", name, "udid", udid) + + if err := r.EnsureBooted(udid, "clone, new: "+name); err != nil { + return "", err + } + } + + if postBoot != nil && *postBoot != "" { + simulatorControlLogger.Info("Running post-boot script", "script", *postBoot, "udid", udid) + + cmd := exec.Command(*postBoot) + cmd.Env = append(os.Environ(), "SIMULATOR_UDID="+udid) + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("postBoot failed (isExisting: %v): %w", isExisting, err) + } + } + + return udid, nil +} + +func (r *RealSimulatorControl) shutdown(simulator SimulatorUDID, context string) error { + _, err := simctl([]string{"shutdown", simulator}, context) + if err != nil { + if pe, ok := err.(*ProcessError); ok && pe.ExitCode == 149 { + simulatorControlLogger.Warn("Shutdown failed, but probably already shut down", "error", err) + return nil + } + return err + } + return nil +} + +func (r *RealSimulatorControl) CleanTempFiles(simulator SimulatorUDID) { + homeDir, err := os.UserHomeDir() + if err != nil { + return + } + + deadCachesPath := filepath.Join(homeDir, "Library/Developer/CoreSimulator/Devices", simulator, + "data/Library/Caches/com.apple.containermanagerd/Dead") + + contents, err := os.ReadDir(deadCachesPath) + if err != nil { + return + } + + for _, item := range contents { + itemPath := filepath.Join(deadCachesPath, item.Name()) + _ = os.RemoveAll(itemPath) + } +} + +func (r *RealSimulatorControl) Delete(simulator SimulatorUDID, name string, context string) error { + return r.deleteAndExistenceMutex(name, func(mutex *SimulatorDeleteOrExistenceMutex) error { + return mutex.unlockedDelete(simulator, context) + }) +} + +func (r *RealSimulatorControl) GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { + var result string + err := r.deleteAndExistenceMutex(name, func(mutex *SimulatorDeleteOrExistenceMutex) error { + var err error + result, err = mutex.unlockedGetExisting(name, deviceType, runtimeIdentifier, context) + return err + }) + return result, err +} + +func (r *RealSimulatorControl) EnsureBooted(simulator SimulatorUDID, context string) error { + for retriesLeft := 1; retriesLeft >= 0; retriesLeft-- { + _, err := simctl([]string{"bootstatus", simulator, "-b"}, context) + if err == nil { + break + } + + if pe, ok := err.(*ProcessError); ok && pe.ExitCode == 149 && retriesLeft > 0 { + simulatorControlLogger.Warn("Boot failed, but probably already booted", "simulator", simulator, "error", err) + continue + } + + return err + } + return nil +} + +func (r *RealSimulatorControl) deleteAndExistenceMutex(name string, fn func(*SimulatorDeleteOrExistenceMutex) error) error { + r.deleteAndExistenceMutexesLock.Lock() + entry, ok := r.deleteAndExistenceMutexes[name] + if !ok { + entry = &deleteOrExistenceMutexEntry{ + mutex: NewSimulatorDeleteOrExistenceMutex(), + count: 0, + } + r.deleteAndExistenceMutexes[name] = entry + } + entry.count++ + r.deleteAndExistenceMutexesLock.Unlock() + + defer func() { + r.deleteAndExistenceMutexesLock.Lock() + entry.count-- + if entry.count == 0 { + delete(r.deleteAndExistenceMutexes, name) + } + r.deleteAndExistenceMutexesLock.Unlock() + }() + + return entry.mutex.WithLock(fn) +} + +type SimulatorDeleteOrExistenceMutex struct { + mu sync.Mutex +} + +func NewSimulatorDeleteOrExistenceMutex() *SimulatorDeleteOrExistenceMutex { + return &SimulatorDeleteOrExistenceMutex{} +} + +func (m *SimulatorDeleteOrExistenceMutex) WithLock(fn func(*SimulatorDeleteOrExistenceMutex) error) error { + m.mu.Lock() + defer m.mu.Unlock() + return fn(m) +} + +func (m *SimulatorDeleteOrExistenceMutex) unlockedGetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { + simulatorControlLogger.Debug("Trying to find existing simulator", "name", name) + + output, err := simctl([]string{"list", "devices", "-j", deviceType}, context) + if err != nil { + return "", err + } + + var devices SimCtlDevices + if err := json.Unmarshal([]byte(output), &devices); err != nil { + simulatorControlLogger.Error("Failed to decode 'simctl list devices -j'", "error", err, "output", output) + return "", fmt.Errorf("failed to decode output: %w - %s", err, output) + } + + if deviceList, ok := devices.Devices[runtimeIdentifier]; ok { + for _, device := range deviceList { + if device.Name == name { + udid := device.UDID + simulatorControlLogger.Debug("Found existing simulator", "name", name, "udid", udid) + + homeDir, _ := os.UserHomeDir() + devicePath := filepath.Join(homeDir, "Library/Developer/CoreSimulator/Devices", udid) + if _, err := os.Stat(devicePath); os.IsNotExist(err) { + simulatorControlLogger.Debug("Simulator doesn't actually exist on disk; deleting", "udid", udid) + _ = m.unlockedDelete(udid, context) + return "", nil + } + + return udid, nil + } + } + } + + simulatorControlLogger.Debug("No existing simulator found", "name", name) + return "", nil +} + +func (m *SimulatorDeleteOrExistenceMutex) unlockedDelete(simulator SimulatorUDID, context string) error { + simulatorControlLogger.Info("Deleting simulator", "udid", simulator) + + if _, err := simctl([]string{"delete", simulator}, context); err != nil { + simulatorControlLogger.Error("Failed to delete simulator", "udid", simulator, "error", err) + return err + } + + simulatorControlLogger.Info("Deleted simulator", "udid", simulator) + return nil +} + +func simctl(args []string, context string) (string, error) { + return subprocess("/usr/bin/xcrun", append([]string{"simctl"}, args...), nil, context) +} + +func subprocess(executable string, args []string, env map[string]string, context string) (string, error) { + cmd := exec.Command(executable, args...) + if env != nil { + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + } + + quotedArgs := make([]string, len(args)) + for i, arg := range args { + quotedArgs[i] = fmt.Sprintf("'%s'", arg) + } + command := fmt.Sprintf("%s %s", executable, strings.Join(quotedArgs, " ")) + + simulatorControlLogger.Debug("Running command", "command", command) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + exitCode := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } + return "", &ProcessError{ + Command: command, + Context: context, + ExitCode: exitCode, + StdOut: stdout.String(), + StdErr: stderr.String(), + } + } + + return stdout.String(), nil +} diff --git a/tools/simulator_manager/go/simulator_manager.go b/tools/simulator_manager/go/simulator_manager.go new file mode 100644 index 0000000..9e78656 --- /dev/null +++ b/tools/simulator_manager/go/simulator_manager.go @@ -0,0 +1,823 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "sync" + "syscall" + "time" +) + +type SimulatorManagerError struct { + kind string + udid SimulatorUDID + message string +} + +func (e *SimulatorManagerError) Error() string { + if e.message != "" { + return e.message + } + return e.kind +} + +var ( + ErrAlreadyLeased = &SimulatorManagerError{kind: "alreadyLeased"} + ErrNoLease = &SimulatorManagerError{kind: "noLease"} + ErrLeaserExited = &SimulatorManagerError{kind: "leaserExited"} +) + +func NewAlreadyLeasedError(udid SimulatorUDID) error { + return &SimulatorManagerError{kind: "alreadyLeased", udid: udid} +} + +type simulatorLease struct { + udid SimulatorUDID + config SimulatorConfig + exclusive bool + slotIndex int + leaserStartTime *uint64 +} + +type simulatorSlot struct { + kind string + udid SimulatorUDID + exclusive bool + task chan taskResult + cancel context.CancelFunc +} + +const ( + slotEmpty = "empty" + slotPendingCreation = "pendingCreation" + slotActive = "active" + slotPendingDeletion = "pendingDeletion" + slotDeleting = "deleting" +) + +func (s simulatorSlot) sortOrder() int { + switch s.kind { + case slotActive: + return 0 + case slotPendingCreation: + return 1 + case slotPendingDeletion: + return 2 + case slotEmpty: + return 3 + case slotDeleting: + return 4 + default: + return 5 + } +} + +type SimulatorManager struct { + simulatorControl SimulatorControl + + mu sync.Mutex + simulatorSlots map[SimulatorConfig][]simulatorSlot + referenceCount map[SimulatorUDID]int + leases map[int32]simulatorLease + leaserExitWatches map[int32]context.CancelFunc + + getBaseSimulatorTasks map[SimulatorConfig]chan taskResult + + deleteIdleAfter uint16 + deleteRecentlyUsedIdleAfter uint16 + deleteOnPIDExit bool + + leaseStore LeaseStore + + recentlyLeased *LRUSet[SimulatorConfig] + + startupProcessPaths []string + postBoot *string + childProcessCancel []context.CancelFunc +} + +func NewSimulatorManager( + simulatorControl SimulatorControl, + deleteRecentlyUsedIdleAfter uint16, + deleteIdleAfter uint16, + recentlyUsedCapacity int, + deleteOnPIDExit bool, + startupProcesses []string, + postBoot *string, + leaseStore LeaseStore, +) *SimulatorManager { + if err := os.Chdir("/tmp"); err != nil { + logger.Warn("Failed to change directory to /tmp", "error", err) + } + + return &SimulatorManager{ + simulatorControl: simulatorControl, + simulatorSlots: make(map[SimulatorConfig][]simulatorSlot), + referenceCount: make(map[SimulatorUDID]int), + leases: make(map[int32]simulatorLease), + leaserExitWatches: make(map[int32]context.CancelFunc), + getBaseSimulatorTasks: make(map[SimulatorConfig]chan taskResult), + deleteIdleAfter: deleteIdleAfter, + deleteRecentlyUsedIdleAfter: deleteRecentlyUsedIdleAfter, + deleteOnPIDExit: deleteOnPIDExit, + leaseStore: leaseStore, + recentlyLeased: NewLRUSet[SimulatorConfig](recentlyUsedCapacity), + startupProcessPaths: startupProcesses, + postBoot: postBoot, + } +} + +func (sm *SimulatorManager) StartChildProcesses() error { + for _, path := range sm.startupProcessPaths { + ctx, cancel := context.WithCancel(context.Background()) + sm.childProcessCancel = append(sm.childProcessCancel, cancel) + go sm.startChildProcess(ctx, path) + } + return nil +} + +func (sm *SimulatorManager) startChildProcess(ctx context.Context, path string) { + childProcessLogger.Info("Starting child process", "path", path) + + cmd := exec.Command(path) + + outPTY, err := NewPTY() + if err != nil { + childProcessLogger.Error("Failed to create output PTY", "path", path, "error", err) + return + } + + errPTY, err := NewPTY() + if err != nil { + childProcessLogger.Error("Failed to create error PTY", "path", path, "error", err) + return + } + + cmd.Stdout = os.NewFile(uintptr(outPTY.Child), "stdout") + cmd.Stderr = os.NewFile(uintptr(errPTY.Child), "stderr") + + go watchFD(outPTY.Parent, func(line string) { + childProcessLogger.Info(fmt.Sprintf("[%s] %s", path, line)) + }) + + go watchFD(errPTY.Parent, func(line string) { + childProcessLogger.Error(fmt.Sprintf("[%s] %s", path, line)) + }) + + if err := cmd.Start(); err != nil { + childProcessLogger.Error("Failed to start child process", "path", path, "error", err) + return + } + + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + select { + case <-ctx.Done(): + _ = cmd.Process.Kill() + return + case err := <-done: + if err != nil { + childProcessLogger.Warn("Child process exited with error", "path", path, "error", err) + } else { + childProcessLogger.Warn("Child process exited", "path", path) + } + } +} + +func watchFD(fd int, onLine func(string)) { + file := os.NewFile(uintptr(fd), "pipe") + defer file.Close() + + buf := make([]byte, 4096) + var lineBuffer []byte + + for { + n, err := file.Read(buf) + if err != nil { + if err != io.EOF { + logger.Error("Error reading from FD", "error", err) + } + break + } + + lineBuffer = append(lineBuffer, buf[:n]...) + + for { + newlineIdx := -1 + for i, b := range lineBuffer { + if b == '\n' { + newlineIdx = i + break + } + } + + if newlineIdx == -1 { + break + } + + line := string(lineBuffer[:newlineIdx]) + onLine(line) + lineBuffer = lineBuffer[newlineIdx+1:] + } + } +} + +func (sm *SimulatorManager) RestoreLeases() { + if sm.leaseStore == nil { + return + } + + persisted := sm.leaseStore.Load() + if len(persisted) == 0 { + return + } + + sm.mu.Lock() + defer sm.mu.Unlock() + + adopted := 0 + dropped := 0 + + for _, lease := range persisted { + if !sm.leaserSurvived(lease) { + dropped++ + continue + } + + if !sm.canAdopt(lease) { + logger.Error("Not restoring conflicting lease", "pid", lease.PID, "udid", lease.UDID) + dropped++ + continue + } + + sm.leases[lease.PID] = simulatorLease{ + udid: lease.UDID, + config: lease.Config, + exclusive: lease.Exclusive, + slotIndex: lease.SlotIndex, + leaserStartTime: lease.LeaserStartTime, + } + + slots := sm.simulatorSlots[lease.Config] + for len(slots) <= lease.SlotIndex { + slots = append(slots, simulatorSlot{kind: slotEmpty}) + } + slots[lease.SlotIndex] = simulatorSlot{ + kind: slotActive, + udid: lease.UDID, + exclusive: lease.Exclusive, + } + sm.simulatorSlots[lease.Config] = slots + + sm.incrementReferenceCount(lease.UDID) + + if sm.deleteOnPIDExit { + sm.registerReleaseOnExit(lease.PID) + } + + adopted++ + } + + logger.Info("Restored leases from previous simulator manager", "adopted", adopted, "dropped", dropped) + + sm.persistLeases() +} + +func (sm *SimulatorManager) leaserSurvived(lease PersistedLease) bool { + if !processIsRunning(lease.PID) { + return false + } + + if lease.LeaserStartTime == nil { + return true + } + + currentStart := processStartTime(lease.PID) + if currentStart == nil { + return false + } + + if *currentStart != *lease.LeaserStartTime { + logger.Info("PID is running but started at a different time; treating as exited", "pid", lease.PID) + return false + } + + return true +} + +func (sm *SimulatorManager) canAdopt(lease PersistedLease) bool { + for _, existing := range sm.leases { + if existing.udid == lease.UDID && (existing.exclusive || lease.Exclusive) { + return false + } + + if existing.config == lease.Config && + existing.slotIndex == lease.SlotIndex && + existing.udid != lease.UDID { + return false + } + } + return true +} + +func (sm *SimulatorManager) persistLeases() { + if sm.leaseStore == nil { + return + } + + leases := make([]PersistedLease, 0, len(sm.leases)) + for pid, lease := range sm.leases { + leases = append(leases, PersistedLease{ + PID: pid, + LeaserStartTime: lease.leaserStartTime, + UDID: lease.udid, + Config: lease.config, + Exclusive: lease.exclusive, + SlotIndex: lease.slotIndex, + }) + } + + sm.leaseStore.Save(leases) +} + +func (sm *SimulatorManager) Lease(leaser int32, exclusive bool, config SimulatorConfig) (SimulatorUDID, error) { + sm.mu.Lock() + if existingLease, ok := sm.leases[leaser]; ok { + sm.mu.Unlock() + return "", NewAlreadyLeasedError(existingLease.udid) + } + sm.mu.Unlock() + + logger.Info("Leasing simulator", "exclusive", exclusive, "config", config, "pid", leaser) + + if sm.deleteOnPIDExit && !processIsRunning(leaser) { + logger.Info("PID exited before its lease could be provisioned", "pid", leaser) + return "", ErrLeaserExited + } + + simulator, slotIndex, err := sm.getSimulator(config, exclusive) + if err != nil { + return "", err + } + + sm.mu.Lock() + sm.recentlyLeased.Insert(config) + + sm.leases[leaser] = simulatorLease{ + udid: simulator, + config: config, + exclusive: exclusive, + slotIndex: slotIndex, + leaserStartTime: processStartTime(leaser), + } + sm.persistLeases() + sm.mu.Unlock() + + if sm.deleteOnPIDExit && !processIsRunning(leaser) { + logger.Info("PID exited while simulator was being provisioned; returning to pool", "pid", leaser, "udid", simulator) + _ = sm.Release(leaser) + return "", ErrLeaserExited + } + + logger.Info("Leased simulator", "udid", simulator, "pid", leaser) + + if sm.deleteOnPIDExit { + sm.mu.Lock() + sm.registerReleaseOnExit(leaser) + sm.mu.Unlock() + } + + return simulator, nil +} + +func (sm *SimulatorManager) LiveLeaseCount() int { + sm.mu.Lock() + defer sm.mu.Unlock() + + count := 0 + for pid := range sm.leases { + if processIsRunning(pid) { + count++ + } + } + return count +} + +func (sm *SimulatorManager) Release(leaser int32) error { + sm.mu.Lock() + lease, ok := sm.leases[leaser] + if !ok { + sm.mu.Unlock() + return ErrNoLease + } + delete(sm.leases, leaser) + + logger.Info("Releasing simulator", "udid", lease.udid, "pid", leaser) + + sm.persistLeases() + sm.removeReleaseOnExit(leaser) + sm.mu.Unlock() + + sm.simulatorControl.CleanTempFiles(lease.udid) + + return sm.decrementReferenceCount(lease.udid, lease.config, lease.slotIndex) +} + +func (sm *SimulatorManager) getBase(config SimulatorConfig) (SimulatorUDID, error) { + sm.mu.Lock() + if existingTask, ok := sm.getBaseSimulatorTasks[config]; ok { + sm.mu.Unlock() + result := <-existingTask + return result.udid, result.err + } + + resultChan := make(chan taskResult, 1) + sm.getBaseSimulatorTasks[config] = resultChan + sm.mu.Unlock() + + go func() { + defer func() { + sm.mu.Lock() + delete(sm.getBaseSimulatorTasks, config) + sm.mu.Unlock() + }() + + logger.Info("Creating base simulator", "config", config) + + baseSimulator, err := sm.simulatorControl.CreateBase( + config.BaseDeviceName(), + config, + config.RuntimeIdentifier(), + ) + + if err == nil { + logger.Info("Created base simulator", "config", config, "udid", baseSimulator) + } + + resultChan <- taskResult{udid: baseSimulator, err: err} + }() + + result := <-resultChan + return result.udid, result.err +} + +func (sm *SimulatorManager) incrementReferenceCount(simulator SimulatorUDID) { + count := sm.referenceCount[simulator] + count++ + sm.referenceCount[simulator] = count + + logger.Debug("Reference count increased", "udid", simulator, "count", count) +} + +func (sm *SimulatorManager) decrementReferenceCount(simulator SimulatorUDID, config SimulatorConfig, slotIndex int) error { + sm.mu.Lock() + count, ok := sm.referenceCount[simulator] + if !ok { + sm.mu.Unlock() + return nil + } + + count-- + sm.referenceCount[simulator] = count + + logger.Debug("Reference count decreased", "udid", simulator, "count", count) + + if count > 0 { + sm.mu.Unlock() + return nil + } + sm.mu.Unlock() + + sm.pendingDeletion(simulator, config, slotIndex) + return nil +} + +func (sm *SimulatorManager) getSimulator(config SimulatorConfig, exclusive bool) (SimulatorUDID, int, error) { + sm.mu.Lock() + defer sm.mu.Unlock() + + if _, ok := sm.simulatorSlots[config]; !ok { + sm.simulatorSlots[config] = []simulatorSlot{} + } + + slots := sm.simulatorSlots[config] + type indexedSlot struct { + index int + slot simulatorSlot + } + + sortedSlots := make([]indexedSlot, len(slots)) + for i, slot := range slots { + sortedSlots[i] = indexedSlot{index: i, slot: slot} + } + + // Sort by sort order, then by index + for i := 0; i < len(sortedSlots); i++ { + for j := i + 1; j < len(sortedSlots); j++ { + iOrder := sortedSlots[i].slot.sortOrder() + jOrder := sortedSlots[j].slot.sortOrder() + if iOrder > jOrder || (iOrder == jOrder && sortedSlots[i].index > sortedSlots[j].index) { + sortedSlots[i], sortedSlots[j] = sortedSlots[j], sortedSlots[i] + } + } + } + + for _, is := range sortedSlots { + slot := is.slot + index := is.index + + switch slot.kind { + case slotActive: + if !slot.exclusive && !exclusive { + sm.mu.Unlock() + sim, err := sm.reuseSimulator(slot.udid, config, exclusive, index) + sm.mu.Lock() + return sim, index, err + } + + case slotPendingDeletion: + logger.Info("Turning pending deletion into active simulator", "udid", slot.udid, "exclusive", exclusive) + + sm.simulatorSlots[config][index] = simulatorSlot{ + kind: slotActive, + udid: slot.udid, + exclusive: exclusive, + } + + if slot.cancel != nil { + slot.cancel() + } + + sm.mu.Unlock() + sim, err := sm.reuseSimulator(slot.udid, config, exclusive, index) + sm.mu.Lock() + return sim, index, err + + case slotEmpty: + task, cancel := sm.createCloneTask(config, exclusive, index) + sm.simulatorSlots[config][index] = simulatorSlot{ + kind: slotPendingCreation, + task: task, + exclusive: exclusive, + cancel: cancel, + } + sm.mu.Unlock() + result := <-task + sm.mu.Lock() + return result.udid, index, result.err + + case slotPendingCreation: + if !slot.exclusive && !exclusive { + sm.mu.Unlock() + result := <-slot.task + if result.err == nil { + sm.mu.Lock() + sm.incrementReferenceCount(result.udid) + sm.mu.Unlock() + } + sm.mu.Lock() + return result.udid, index, result.err + } + } + } + + index := len(sm.simulatorSlots[config]) + task, cancel := sm.createCloneTask(config, exclusive, index) + sm.simulatorSlots[config] = append(sm.simulatorSlots[config], simulatorSlot{ + kind: slotPendingCreation, + task: task, + exclusive: exclusive, + cancel: cancel, + }) + sm.mu.Unlock() + result := <-task + sm.mu.Lock() + return result.udid, index, result.err +} + +func (sm *SimulatorManager) reuseSimulator(simulator SimulatorUDID, config SimulatorConfig, exclusive bool, slotIndex int) (SimulatorUDID, error) { + sm.mu.Lock() + sm.incrementReferenceCount(simulator) + sm.mu.Unlock() + + err := sm.simulatorControl.EnsureBooted(simulator, fmt.Sprintf("getSimulator, reused: %s", config.CloneDeviceName(slotIndex))) + if err != nil { + if pe, ok := err.(*ProcessError); ok && pe.ExitCode == 148 { + logger.Warn("Boot of existing simulator failed; deleting and returning new simulator", "udid", simulator, "error", err) + + sm.mu.Lock() + _ = sm.delete(simulator, config, slotIndex, false, "getSimulator, reused: "+config.CloneDeviceName(slotIndex)) + + task, cancel := sm.createCloneTask(config, exclusive, slotIndex) + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotPendingCreation, + task: task, + exclusive: exclusive, + cancel: cancel, + } + sm.mu.Unlock() + + result := <-task + return result.udid, result.err + } + return "", err + } + + return simulator, nil +} + +func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bool, slotIndex int) (chan taskResult, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + resultChan := make(chan taskResult, 1) + + go func() { + defer close(resultChan) + + baseUDID, err := sm.getBase(config) + if err != nil { + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + sm.mu.Unlock() + resultChan <- taskResult{err: err} + return + } + + if ctx.Err() != nil { + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + sm.mu.Unlock() + resultChan <- taskResult{err: ctx.Err()} + return + } + + simulator, err := sm.simulatorControl.Clone( + baseUDID, + config.CloneDeviceName(slotIndex), + config.DeviceType, + config.RuntimeIdentifier(), + sm.postBoot, + ) + + if err != nil { + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + sm.mu.Unlock() + resultChan <- taskResult{err: err} + return + } + + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotActive, + udid: simulator, + exclusive: exclusive, + } + sm.incrementReferenceCount(simulator) + sm.mu.Unlock() + + resultChan <- taskResult{udid: simulator, err: nil} + }() + + return resultChan, cancel +} + +func (sm *SimulatorManager) registerReleaseOnExit(leaser int32) { + ctx, cancel := context.WithCancel(context.Background()) + sm.leaserExitWatches[leaser] = cancel + + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !processIsRunning(leaser) { + logger.Debug("PID exited", "pid", leaser) + _ = sm.Release(leaser) + return + } + } + } + }() +} + +func (sm *SimulatorManager) removeReleaseOnExit(leaser int32) { + if cancel, ok := sm.leaserExitWatches[leaser]; ok { + delete(sm.leaserExitWatches, leaser) + cancel() + } +} + +func (sm *SimulatorManager) pendingDeletion(simulator SimulatorUDID, config SimulatorConfig, slotIndex int) { + if sm.deleteIdleAfter == 0 && sm.deleteRecentlyUsedIdleAfter == 0 { + sm.mu.Lock() + _ = sm.delete(simulator, config, slotIndex, true, "pendingDeletion immediate") + sm.mu.Unlock() + return + } + + ctx, cancel := context.WithCancel(context.Background()) + + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotPendingDeletion, + udid: simulator, + cancel: cancel, + } + sm.mu.Unlock() + + go func() { + logger.Info("Scheduling delete of simulator", "udid", simulator, "idleAfter", sm.deleteIdleAfter, "recentlyUsedIdleAfter", sm.deleteRecentlyUsedIdleAfter) + + now := time.Now() + shortDeadline := now.Add(time.Duration(sm.deleteIdleAfter) * time.Second) + recentlyUsedDeadline := now.Add(time.Duration(sm.deleteRecentlyUsedIdleAfter) * time.Second) + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + var remainingTime time.Duration + sm.mu.Lock() + if sm.recentlyLeased.Contains(config) { + remainingTime = time.Until(recentlyUsedDeadline) + } else { + remainingTime = time.Until(shortDeadline) + } + sm.mu.Unlock() + + if remainingTime <= 0 { + sm.mu.Lock() + slots := sm.simulatorSlots[config] + if slotIndex < len(slots) { + slot := slots[slotIndex] + if slot.kind == slotPendingDeletion && slot.udid == simulator { + _ = sm.delete(simulator, config, slotIndex, true, "pendingDeletion delayed") + } + } + sm.mu.Unlock() + return + } + } + } + }() +} + +func (sm *SimulatorManager) delete(simulator SimulatorUDID, config SimulatorConfig, slotIndex int, cleanUpSlots bool, context string) error { + name := config.CloneDeviceName(slotIndex) + + logger.Info("Deleting simulator", "udid", simulator, "name", name) + + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotDeleting, + udid: simulator, + } + + delete(sm.referenceCount, simulator) + + defer func() { + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + + if cleanUpSlots { + slots := sm.simulatorSlots[config] + for len(slots) > 0 && slots[len(slots)-1].kind == slotEmpty { + slots = slots[:len(slots)-1] + } + sm.simulatorSlots[config] = slots + } + }() + + err := sm.simulatorControl.Delete(simulator, name, context) + if err == nil { + logger.Info("Deleted simulator", "udid", simulator, "name", name) + } + + return err +} + +func (sm *SimulatorManager) Close() { + for _, cancel := range sm.childProcessCancel { + cancel() + } +} + +func processIsRunning(pid int32) bool { + err := syscall.Kill(int(pid), 0) + if err == nil { + return true + } + return err != syscall.ESRCH +} diff --git a/tools/simulator_manager/go/simulator_request_handler.go b/tools/simulator_manager/go/simulator_request_handler.go new file mode 100644 index 0000000..fd8ca3d --- /dev/null +++ b/tools/simulator_manager/go/simulator_request_handler.go @@ -0,0 +1,166 @@ +package main + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +type SimulatorManagerResponse struct { + Status int + Message string +} + +type SimulatorRequestHandler struct { + simulatorManager *SimulatorManager +} + +func NewSimulatorRequestHandler(simulatorManager *SimulatorManager) *SimulatorRequestHandler { + return &SimulatorRequestHandler{ + simulatorManager: simulatorManager, + } +} + +func (h *SimulatorRequestHandler) LiveLeaseCount() int { + return h.simulatorManager.LiveLeaseCount() +} + +func (h *SimulatorRequestHandler) HandleRequest(method string, path string, queryParams url.Values) SimulatorManagerResponse { + pathComponents := strings.Split(strings.Trim(path, "/"), "/") + if len(pathComponents) > 0 && pathComponents[0] == "simulator" { + pathComponents = pathComponents[1:] + } + + switch method { + case http.MethodPost: + return h.handlePost(pathComponents, queryParams) + case http.MethodDelete: + return h.handleDelete(pathComponents, queryParams) + default: + return SimulatorManagerResponse{ + Status: http.StatusMethodNotAllowed, + Message: fmt.Sprintf("Unsupported HTTP method: %s", method), + } + } +} + +func (h *SimulatorRequestHandler) handlePost(pathComponents []string, queryParams url.Values) SimulatorManagerResponse { + if len(pathComponents) < 1 || pathComponents[0] == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify ", + } + } + + leaser, err := strconv.ParseInt(pathComponents[0], 10, 32) + if err != nil { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Leaser PID must be an integer", + } + } + + exclusiveStr := queryParams.Get("exclusive") + if exclusiveStr == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'exclusive' query parameter", + } + } + exclusive := exclusiveStr == "1" + + deviceType := queryParams.Get("deviceType") + if deviceType == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'deviceType' query parameter", + } + } + + osParam := queryParams.Get("os") + if osParam == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'os' query parameter", + } + } + + version := queryParams.Get("version") + if version == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'version' query parameter", + } + } + + config := SimulatorConfig{ + DeviceType: deviceType, + OS: osParam, + Version: version, + } + + udid, err := h.simulatorManager.Lease(int32(leaser), exclusive, config) + if err != nil { + if err == ErrLeaserExited { + return SimulatorManagerResponse{ + Status: http.StatusGone, + Message: fmt.Sprintf("PID %d exited before its simulator was provisioned", leaser), + } + } + + if sme, ok := err.(*SimulatorManagerError); ok && sme.kind == "alreadyLeased" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: fmt.Sprintf("PID %d has already leased another simulator: %s", leaser, sme.udid), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusInternalServerError, + Message: fmt.Sprintf("Internal server error: %v", err), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusCreated, + Message: udid, + } +} + +func (h *SimulatorRequestHandler) handleDelete(pathComponents []string, queryParams url.Values) SimulatorManagerResponse { + if len(pathComponents) < 1 || pathComponents[0] == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify ", + } + } + + leaser, err := strconv.ParseInt(pathComponents[0], 10, 32) + if err != nil { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Leaser PID must be an integer", + } + } + + if err := h.simulatorManager.Release(int32(leaser)); err != nil { + if err == ErrNoLease { + return SimulatorManagerResponse{ + Status: http.StatusNotFound, + Message: fmt.Sprintf("PID %d doesn't have a simulator leased", leaser), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusInternalServerError, + Message: fmt.Sprintf("Internal server error: %v", err), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusOK, + Message: "Success", + } +} diff --git a/tools/simulator_manager/start.sh b/tools/simulator_manager/start.sh index be2d11d..59bda59 100755 --- a/tools/simulator_manager/start.sh +++ b/tools/simulator_manager/start.sh @@ -204,7 +204,7 @@ if [[ -z "$version" ]]; then # --- end runfiles.bash initialization v3 --- simulator_manager="$( - rlocation _main/tools/simulator_manager/simulator_manager + rlocation _main/tools/simulator_manager/go/go_/go )" bazel_prepare_simulator="$( rlocation _main/tools/simulator_manager/prepare_simulator From db4ff148799d3b976eb9c2bb16d064084194b967 Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Fri, 21 Aug 2026 10:19:29 +0200 Subject: [PATCH 10/11] more --- MODULE.bazel | 2 +- go.mod | 5 + go.sum | 4 + tools/simulator_manager/go/BUILD.bazel | 21 +- .../go/fake_simulator_control_test.go | 139 ++++++++ .../simulator_manager/go/http_server_test.go | 75 ++++ .../simulator_manager/go/lease_store_test.go | 84 +++++ tools/simulator_manager/go/lru_set.go | 6 +- tools/simulator_manager/go/lru_set_test.go | 60 ++++ tools/simulator_manager/go/main_test.go | 26 ++ .../simulator_manager/go/simulator_control.go | 59 ++- .../go/simulator_control_test.go | 49 +++ .../simulator_manager/go/simulator_manager.go | 42 ++- .../go/simulator_manager_test.go | 337 ++++++++++++++++++ .../go/simulator_request_handler_test.go | 147 ++++++++ 15 files changed, 1013 insertions(+), 43 deletions(-) create mode 100644 tools/simulator_manager/go/fake_simulator_control_test.go create mode 100644 tools/simulator_manager/go/http_server_test.go create mode 100644 tools/simulator_manager/go/lease_store_test.go create mode 100644 tools/simulator_manager/go/lru_set_test.go create mode 100644 tools/simulator_manager/go/main_test.go create mode 100644 tools/simulator_manager/go/simulator_control_test.go create mode 100644 tools/simulator_manager/go/simulator_manager_test.go create mode 100644 tools/simulator_manager/go/simulator_request_handler_test.go diff --git a/MODULE.bazel b/MODULE.bazel index 3b82a80..c402d9a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -99,7 +99,7 @@ go_sdk.download(version = GO_VERSION) go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") -use_repo(go_deps, "com_github_google_go_cmp", "org_golang_x_sys") +use_repo(go_deps, "com_github_google_go_cmp", "org_golang_x_sys", "com_github_stretchr_testify") bazel_dep(name = "rules_java", version = "9.6.1") diff --git a/go.mod b/go.mod index 202b1bd..a5f27dc 100644 --- a/go.mod +++ b/go.mod @@ -5,3 +5,8 @@ go 1.25.0 require github.com/google/go-cmp v0.6.0 require golang.org/x/sys v0.47.0 + +require ( + github.com/stretchr/testify v1.12.1 + go.yaml.in/yaml/v3 v3.0.5 // indirect +) diff --git a/go.sum b/go.sum index e028c78..375b797 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/tools/simulator_manager/go/BUILD.bazel b/tools/simulator_manager/go/BUILD.bazel index af7e2be..66174e4 100644 --- a/tools/simulator_manager/go/BUILD.bazel +++ b/tools/simulator_manager/go/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") go_library( name = "go_lib", @@ -23,3 +23,22 @@ go_binary( embed = [":go_lib"], visibility = ["//visibility:public"], ) + +go_test( + name = "go_test", + srcs = [ + "fake_simulator_control_test.go", + "http_server_test.go", + "lease_store_test.go", + "lru_set_test.go", + "main_test.go", + "simulator_control_test.go", + "simulator_manager_test.go", + "simulator_request_handler_test.go", + ], + embed = [":go_lib"], + deps = [ + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/tools/simulator_manager/go/fake_simulator_control_test.go b/tools/simulator_manager/go/fake_simulator_control_test.go new file mode 100644 index 0000000..4a15b6d --- /dev/null +++ b/tools/simulator_manager/go/fake_simulator_control_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "fmt" + "sync" +) + +// fakeSimulatorControl is an in-memory SimulatorControl used to test +// SimulatorManager's lease/slot/reference-count logic without shelling out to +// xcrun simctl. +type fakeSimulatorControl struct { + mu sync.Mutex + + udidCounter int + + createBaseCalls int + + cloneCalls []string // clone device names, in call order + // cloneGate, if non-nil, blocks the first Clone call until the test sends + // (or closes) it -- used to deterministically exercise in-flight clone + // sharing between concurrent Lease calls. + cloneGate chan struct{} + cloneGateOnce sync.Once + cloneGateFired bool + + ensureBootedCalls []SimulatorUDID + // ensureBootedErrs are one-shot: each error is returned exactly once for + // the matching UDID, then cleared, so a retry can succeed. + ensureBootedErrs map[SimulatorUDID]error + + deleteCalls []SimulatorUDID + deleteErrs map[SimulatorUDID]error + + cleanTempFilesCalls []SimulatorUDID +} + +func newFakeSimulatorControl() *fakeSimulatorControl { + return &fakeSimulatorControl{ + ensureBootedErrs: make(map[SimulatorUDID]error), + deleteErrs: make(map[SimulatorUDID]error), + } +} + +func (f *fakeSimulatorControl) nextUDID() SimulatorUDID { + f.udidCounter++ + return fmt.Sprintf("udid-%d", f.udidCounter) +} + +func (f *fakeSimulatorControl) CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { + f.mu.Lock() + f.createBaseCalls++ + udid := f.nextUDID() + f.mu.Unlock() + return udid, nil +} + +func (f *fakeSimulatorControl) Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { + f.mu.Lock() + f.cloneCalls = append(f.cloneCalls, name) + gate := f.cloneGate + alreadyFired := f.cloneGateFired + f.mu.Unlock() + + if gate != nil && !alreadyFired { + f.cloneGateOnce.Do(func() { + <-gate + f.mu.Lock() + f.cloneGateFired = true + f.mu.Unlock() + }) + } + + f.mu.Lock() + udid := f.nextUDID() + f.mu.Unlock() + return udid, nil +} + +func (f *fakeSimulatorControl) EnsureBooted(simulator SimulatorUDID, context string) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.ensureBootedCalls = append(f.ensureBootedCalls, simulator) + if err, ok := f.ensureBootedErrs[simulator]; ok { + delete(f.ensureBootedErrs, simulator) + return err + } + return nil +} + +func (f *fakeSimulatorControl) CleanTempFiles(simulator SimulatorUDID) { + f.mu.Lock() + defer f.mu.Unlock() + f.cleanTempFilesCalls = append(f.cleanTempFilesCalls, simulator) +} + +func (f *fakeSimulatorControl) Delete(simulator SimulatorUDID, name string, context string) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.deleteCalls = append(f.deleteCalls, simulator) + return f.deleteErrs[simulator] +} + +func (f *fakeSimulatorControl) GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { + // SimulatorManager never calls this directly -- it's only used internally + // by RealSimulatorControl's own createBase/clone rediscovery logic, which + // this fake doesn't need to replicate. + return "", nil +} + +// fakeLeaseStore is an in-memory LeaseStore for testing RestoreLeases and +// persistence side effects without touching disk. +type fakeLeaseStore struct { + mu sync.Mutex + loaded []PersistedLease + saved [][]PersistedLease +} + +func (f *fakeLeaseStore) Load() []PersistedLease { + f.mu.Lock() + defer f.mu.Unlock() + return f.loaded +} + +func (f *fakeLeaseStore) Save(leases []PersistedLease) { + f.mu.Lock() + defer f.mu.Unlock() + f.saved = append(f.saved, leases) +} + +func (f *fakeLeaseStore) lastSaved() []PersistedLease { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.saved) == 0 { + return nil + } + return f.saved[len(f.saved)-1] +} diff --git a/tools/simulator_manager/go/http_server_test.go b/tools/simulator_manager/go/http_server_test.go new file mode 100644 index 0000000..a0ea240 --- /dev/null +++ b/tools/simulator_manager/go/http_server_test.go @@ -0,0 +1,75 @@ +package main + +// These tests exercise HTTPServer's HTTP handler methods directly via +// net/http/httptest, bypassing Run(). Run() itself (unix socket setup, PID +// file management, OS signal handling) and handleShutdownRequest (which +// calls os.Exit) are integration-level, OS-facing plumbing that would +// require killing or replacing the test process itself to exercise safely, +// so they're intentionally left uncovered here. + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHandleVersionRequest(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1.2.3") + + req := httptest.NewRequest(http.MethodGet, "/version", nil) + rec := httptest.NewRecorder() + + server.handleVersionRequest(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "v1.2.3\n", rec.Body.String()) +} + +func TestHandleLeasesRequest(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1") + + req := httptest.NewRequest(http.MethodGet, "/leases", nil) + rec := httptest.NewRecorder() + + server.handleLeasesRequest(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "0\n", rec.Body.String()) +} + +func TestHandleSimulatorRequest_RoutesToHandler(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1") + + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + req := httptest.NewRequest(http.MethodPost, "/simulator/1?"+params.Encode(), nil) + rec := httptest.NewRecorder() + + server.handleSimulatorRequest(rec, req) + + require.Equal(t, http.StatusCreated, rec.Code) + assert.NotEmpty(t, rec.Body.String()) +} + +func TestHandleSimulatorRequest_BadRequestSurfacesMessage(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1") + + req := httptest.NewRequest(http.MethodDelete, "/simulator/1", nil) + rec := httptest.NewRecorder() + + server.handleSimulatorRequest(rec, req) + + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Contains(t, rec.Body.String(), "doesn't have a simulator leased") +} diff --git a/tools/simulator_manager/go/lease_store_test.go b/tools/simulator_manager/go/lease_store_test.go new file mode 100644 index 0000000..65b80f3 --- /dev/null +++ b/tools/simulator_manager/go/lease_store_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFileLeaseStore_LoadNonexistentFile(t *testing.T) { + store := NewFileLeaseStore(filepath.Join(t.TempDir(), "does-not-exist.json")) + assert.Nil(t, store.Load()) +} + +func TestFileLeaseStore_SaveAndLoadRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "leases.json") + store := NewFileLeaseStore(path) + + startTime := uint64(123) + leases := []PersistedLease{ + { + PID: 42, + LeaserStartTime: &startTime, + UDID: "udid-1", + Config: SimulatorConfig{DeviceType: "iPhone", OS: "iOS", Version: "17.0"}, + Exclusive: true, + SlotIndex: 0, + }, + { + PID: 43, + UDID: "udid-2", + Config: SimulatorConfig{DeviceType: "iPad", OS: "iOS", Version: "17.0"}, + Exclusive: false, + SlotIndex: 1, + }, + } + + store.Save(leases) + + loaded := store.Load() + require.Equal(t, leases, loaded) +} + +func TestFileLeaseStore_SaveIsAtomic(t *testing.T) { + path := filepath.Join(t.TempDir(), "leases.json") + store := NewFileLeaseStore(path) + + store.Save([]PersistedLease{{PID: 1, UDID: "udid-1"}}) + store.Save([]PersistedLease{{PID: 2, UDID: "udid-2"}}) + + // The temp file used for the atomic rename should never be left behind. + _, err := os.Stat(path + ".tmp") + assert.True(t, os.IsNotExist(err)) + + loaded := store.Load() + require.Len(t, loaded, 1) + assert.Equal(t, int32(2), loaded[0].PID) +} + +func TestFileLeaseStore_LoadCorruptFileReturnsNilInsteadOfCrashing(t *testing.T) { + path := filepath.Join(t.TempDir(), "leases.json") + require.NoError(t, os.WriteFile(path, []byte("not valid json"), 0644)) + + store := NewFileLeaseStore(path) + assert.Nil(t, store.Load()) +} + +func TestProcessStartTime_CurrentProcessIsStable(t *testing.T) { + pid := int32(os.Getpid()) + + first := processStartTime(pid) + require.NotNil(t, first, "should be able to read the current process's own start time") + + second := processStartTime(pid) + require.NotNil(t, second) + + assert.Equal(t, *first, *second, "the start time of a still-running process must not change between calls") +} + +func TestProcessStartTime_ExitedProcessReturnsNil(t *testing.T) { + assert.Nil(t, processStartTime(spawnDeadPID(t))) +} diff --git a/tools/simulator_manager/go/lru_set.go b/tools/simulator_manager/go/lru_set.go index 45ba636..4f7c9a4 100644 --- a/tools/simulator_manager/go/lru_set.go +++ b/tools/simulator_manager/go/lru_set.go @@ -25,11 +25,11 @@ func NewLRUSet[T comparable](capacity int) *LRUSet[T] { func (s *LRUSet[T]) Insert(element T) *T { var evicted *T if _, exists := s.storage[element]; exists { - // Remove from current position in order + // Remove from current position in order. Nothing is evicted here -- + // the element is already in the set, just moving to the + // most-recently-used end. for i, e := range s.order { if e == element { - old := s.order[i] - evicted = &old s.order = append(s.order[:i], s.order[i+1:]...) break } diff --git a/tools/simulator_manager/go/lru_set_test.go b/tools/simulator_manager/go/lru_set_test.go new file mode 100644 index 0000000..dabdc24 --- /dev/null +++ b/tools/simulator_manager/go/lru_set_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLRUSet_InsertAndContains(t *testing.T) { + s := NewLRUSet[string](2) + + assert.Nil(t, s.Insert("a")) + assert.True(t, s.Contains("a")) + assert.False(t, s.Contains("b")) +} + +func TestLRUSet_EvictsLeastRecentlyUsed(t *testing.T) { + s := NewLRUSet[string](2) + + s.Insert("a") + s.Insert("b") + + evicted := s.Insert("c") + assert.NotNil(t, evicted) + assert.Equal(t, "a", *evicted) + assert.False(t, s.Contains("a")) + assert.True(t, s.Contains("b")) + assert.True(t, s.Contains("c")) +} + +func TestLRUSet_ReinsertingExistingElementRefreshesItsPosition(t *testing.T) { + s := NewLRUSet[string](2) + + s.Insert("a") + s.Insert("b") + + // Touching "a" again should make "b" the least-recently-used one instead. + evicted := s.Insert("a") + assert.Nil(t, evicted, "re-inserting an existing element should not evict anything") + + evicted = s.Insert("c") + assert.NotNil(t, evicted) + assert.Equal(t, "b", *evicted, "\"b\" should have become least-recently-used after \"a\" was touched") + assert.True(t, s.Contains("a")) + assert.True(t, s.Contains("c")) +} + +func TestLRUSet_Elements_ReflectsInsertionOrder(t *testing.T) { + s := NewLRUSet[string](3) + s.Insert("a") + s.Insert("b") + s.Insert("c") + + assert.Equal(t, []string{"a", "b", "c"}, s.Elements()) +} + +func TestLRUSet_CapacityMustBePositive(t *testing.T) { + assert.Panics(t, func() { NewLRUSet[string](0) }) + assert.Panics(t, func() { NewLRUSet[string](-1) }) +} diff --git a/tools/simulator_manager/go/main_test.go b/tools/simulator_manager/go/main_test.go new file mode 100644 index 0000000..8f23f96 --- /dev/null +++ b/tools/simulator_manager/go/main_test.go @@ -0,0 +1,26 @@ +package main + +// main() itself isn't covered here: it parses process-global flags and calls +// os.Exit on invalid input, neither of which can be exercised safely from +// within the test binary's own process. + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestArrayFlags_SetAppendsInOrder(t *testing.T) { + var flags arrayFlags + + assert.NoError(t, flags.Set("a")) + assert.NoError(t, flags.Set("b")) + + assert.Equal(t, arrayFlags{"a", "b"}, flags) + assert.Equal(t, "a, b", flags.String()) +} + +func TestArrayFlags_StringOnEmpty(t *testing.T) { + var flags arrayFlags + assert.Equal(t, "", flags.String()) +} diff --git a/tools/simulator_manager/go/simulator_control.go b/tools/simulator_manager/go/simulator_control.go index 7c15165..b06bf0a 100644 --- a/tools/simulator_manager/go/simulator_control.go +++ b/tools/simulator_manager/go/simulator_control.go @@ -88,10 +88,10 @@ type SimulatorControl interface { } type RealSimulatorControl struct { - createBaseTasks map[string]chan taskResult + createBaseTasks map[string]*resultBroadcaster createBaseTasksLock sync.Mutex - cloneTasks map[string]chan taskResult + cloneTasks map[string]*resultBroadcaster cloneTasksLock sync.Mutex deleteAndExistenceMutexes map[string]*deleteOrExistenceMutexEntry @@ -103,6 +103,33 @@ type taskResult struct { err error } +// resultBroadcaster lets any number of callers await the same eventual +// taskResult without consuming it. A plain channel can't do this safely: if +// two goroutines both receive from the one channel used to coalesce a single +// in-flight CreateBase/Clone request, only the first gets the real value -- +// the second gets the channel's zero value (an empty UDID with a nil error, +// i.e. a phantom successful lease) once the channel is drained and closed. +// Waiting on a channel that's closed once, then reading a result set before +// that close, delivers the same value to every waiter instead. +type resultBroadcaster struct { + done chan struct{} + result taskResult +} + +func newResultBroadcaster() *resultBroadcaster { + return &resultBroadcaster{done: make(chan struct{})} +} + +func (b *resultBroadcaster) complete(result taskResult) { + b.result = result + close(b.done) +} + +func (b *resultBroadcaster) wait() taskResult { + <-b.done + return b.result +} + type deleteOrExistenceMutexEntry struct { mutex *SimulatorDeleteOrExistenceMutex count int @@ -110,22 +137,22 @@ type deleteOrExistenceMutexEntry struct { func NewRealSimulatorControl() *RealSimulatorControl { return &RealSimulatorControl{ - createBaseTasks: make(map[string]chan taskResult), - cloneTasks: make(map[string]chan taskResult), + createBaseTasks: make(map[string]*resultBroadcaster), + cloneTasks: make(map[string]*resultBroadcaster), deleteAndExistenceMutexes: make(map[string]*deleteOrExistenceMutexEntry), } } func (r *RealSimulatorControl) CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { r.createBaseTasksLock.Lock() - if existingTask, ok := r.createBaseTasks[name]; ok { + if existing, ok := r.createBaseTasks[name]; ok { r.createBaseTasksLock.Unlock() - result := <-existingTask + result := existing.wait() return result.udid, result.err } - resultChan := make(chan taskResult, 1) - r.createBaseTasks[name] = resultChan + broadcaster := newResultBroadcaster() + r.createBaseTasks[name] = broadcaster r.createBaseTasksLock.Unlock() go func() { @@ -136,10 +163,10 @@ func (r *RealSimulatorControl) CreateBase(name string, config SimulatorConfig, r }() udid, err := r.createBaseImpl(name, config, runtimeIdentifier) - resultChan <- taskResult{udid: udid, err: err} + broadcaster.complete(taskResult{udid: udid, err: err}) }() - result := <-resultChan + result := broadcaster.wait() return result.udid, result.err } @@ -186,14 +213,14 @@ func (r *RealSimulatorControl) createBaseImpl(name string, config SimulatorConfi func (r *RealSimulatorControl) Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { r.cloneTasksLock.Lock() - if existingTask, ok := r.cloneTasks[name]; ok { + if existing, ok := r.cloneTasks[name]; ok { r.cloneTasksLock.Unlock() - result := <-existingTask + result := existing.wait() return result.udid, result.err } - resultChan := make(chan taskResult, 1) - r.cloneTasks[name] = resultChan + broadcaster := newResultBroadcaster() + r.cloneTasks[name] = broadcaster r.cloneTasksLock.Unlock() go func() { @@ -204,10 +231,10 @@ func (r *RealSimulatorControl) Clone(baseSimulator SimulatorUDID, name string, d }() udid, err := r.cloneImpl(baseSimulator, name, deviceType, runtimeIdentifier, postBoot) - resultChan <- taskResult{udid: udid, err: err} + broadcaster.complete(taskResult{udid: udid, err: err}) }() - result := <-resultChan + result := broadcaster.wait() return result.udid, result.err } diff --git a/tools/simulator_manager/go/simulator_control_test.go b/tools/simulator_manager/go/simulator_control_test.go new file mode 100644 index 0000000..3985914 --- /dev/null +++ b/tools/simulator_manager/go/simulator_control_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSimulatorConfig_Naming(t *testing.T) { + cfg := SimulatorConfig{DeviceType: "iPhone15,3", OS: "iOS", Version: "17.4"} + + assert.Equal(t, "iPhone15,3 (iOS 17.4)", cfg.String()) + assert.Equal(t, "EXAMPLE_BAZEL_BASE_iPhone15,3_17.4", cfg.BaseDeviceName()) + assert.Equal(t, "EXAMPLE_BAZEL_CLONE_iPhone15,3_17.4_0", cfg.CloneDeviceName(0)) + assert.Equal(t, "EXAMPLE_BAZEL_CLONE_iPhone15,3_17.4_3", cfg.CloneDeviceName(3)) + assert.Equal(t, "com.apple.CoreSimulator.SimRuntime.iOS-17-4", cfg.RuntimeIdentifier()) +} + +func TestProcessError_Error(t *testing.T) { + withoutContext := &ProcessError{Command: "xcrun simctl delete X", ExitCode: 1, StdOut: "out", StdErr: "err"} + assert.Equal(t, "\"xcrun simctl delete X\" failed with exit code 1:\nouterr", withoutContext.Error()) + + withContext := &ProcessError{Command: "xcrun simctl delete X", Context: "reaper", ExitCode: 2, StdOut: "", StdErr: "boom"} + assert.Equal(t, "\"xcrun simctl delete X\" (reaper) failed with exit code 2:\nboom", withContext.Error()) +} + +func TestSubprocess_CapturesStdout(t *testing.T) { + out, err := subprocess("/bin/echo", []string{"hello", "world"}, nil, "") + assert := assert.New(t) + assert.NoError(err) + assert.Equal("hello world\n", out) +} + +func TestSubprocess_FailureReturnsProcessErrorWithExitCode(t *testing.T) { + _, err := subprocess("/usr/bin/false", nil, nil, "test-context") + + pe, ok := err.(*ProcessError) + assert := assert.New(t) + assert.True(ok, "expected a *ProcessError, got %T", err) + assert.Equal(1, pe.ExitCode) + assert.Equal("test-context", pe.Context) +} + +func TestSubprocess_PassesEnvironment(t *testing.T) { + out, err := subprocess("/bin/sh", []string{"-c", "echo $SIMULATOR_UDID"}, map[string]string{"SIMULATOR_UDID": "udid-123"}, "") + assert := assert.New(t) + assert.NoError(err) + assert.Equal("udid-123\n", out) +} diff --git a/tools/simulator_manager/go/simulator_manager.go b/tools/simulator_manager/go/simulator_manager.go index 9e78656..20c5c91 100644 --- a/tools/simulator_manager/go/simulator_manager.go +++ b/tools/simulator_manager/go/simulator_manager.go @@ -46,7 +46,7 @@ type simulatorSlot struct { kind string udid SimulatorUDID exclusive bool - task chan taskResult + task *resultBroadcaster cancel context.CancelFunc } @@ -84,7 +84,7 @@ type SimulatorManager struct { leases map[int32]simulatorLease leaserExitWatches map[int32]context.CancelFunc - getBaseSimulatorTasks map[SimulatorConfig]chan taskResult + getBaseSimulatorTasks map[SimulatorConfig]*resultBroadcaster deleteIdleAfter uint16 deleteRecentlyUsedIdleAfter uint16 @@ -119,7 +119,7 @@ func NewSimulatorManager( referenceCount: make(map[SimulatorUDID]int), leases: make(map[int32]simulatorLease), leaserExitWatches: make(map[int32]context.CancelFunc), - getBaseSimulatorTasks: make(map[SimulatorConfig]chan taskResult), + getBaseSimulatorTasks: make(map[SimulatorConfig]*resultBroadcaster), deleteIdleAfter: deleteIdleAfter, deleteRecentlyUsedIdleAfter: deleteRecentlyUsedIdleAfter, deleteOnPIDExit: deleteOnPIDExit, @@ -431,14 +431,14 @@ func (sm *SimulatorManager) Release(leaser int32) error { func (sm *SimulatorManager) getBase(config SimulatorConfig) (SimulatorUDID, error) { sm.mu.Lock() - if existingTask, ok := sm.getBaseSimulatorTasks[config]; ok { + if existing, ok := sm.getBaseSimulatorTasks[config]; ok { sm.mu.Unlock() - result := <-existingTask + result := existing.wait() return result.udid, result.err } - resultChan := make(chan taskResult, 1) - sm.getBaseSimulatorTasks[config] = resultChan + broadcaster := newResultBroadcaster() + sm.getBaseSimulatorTasks[config] = broadcaster sm.mu.Unlock() go func() { @@ -460,10 +460,10 @@ func (sm *SimulatorManager) getBase(config SimulatorConfig) (SimulatorUDID, erro logger.Info("Created base simulator", "config", config, "udid", baseSimulator) } - resultChan <- taskResult{udid: baseSimulator, err: err} + broadcaster.complete(taskResult{udid: baseSimulator, err: err}) }() - result := <-resultChan + result := broadcaster.wait() return result.udid, result.err } @@ -568,14 +568,14 @@ func (sm *SimulatorManager) getSimulator(config SimulatorConfig, exclusive bool) cancel: cancel, } sm.mu.Unlock() - result := <-task + result := task.wait() sm.mu.Lock() return result.udid, index, result.err case slotPendingCreation: if !slot.exclusive && !exclusive { sm.mu.Unlock() - result := <-slot.task + result := slot.task.wait() if result.err == nil { sm.mu.Lock() sm.incrementReferenceCount(result.udid) @@ -596,7 +596,7 @@ func (sm *SimulatorManager) getSimulator(config SimulatorConfig, exclusive bool) cancel: cancel, }) sm.mu.Unlock() - result := <-task + result := task.wait() sm.mu.Lock() return result.udid, index, result.err } @@ -623,7 +623,7 @@ func (sm *SimulatorManager) reuseSimulator(simulator SimulatorUDID, config Simul } sm.mu.Unlock() - result := <-task + result := task.wait() return result.udid, result.err } return "", err @@ -632,19 +632,17 @@ func (sm *SimulatorManager) reuseSimulator(simulator SimulatorUDID, config Simul return simulator, nil } -func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bool, slotIndex int) (chan taskResult, context.CancelFunc) { +func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bool, slotIndex int) (*resultBroadcaster, context.CancelFunc) { ctx, cancel := context.WithCancel(context.Background()) - resultChan := make(chan taskResult, 1) + broadcaster := newResultBroadcaster() go func() { - defer close(resultChan) - baseUDID, err := sm.getBase(config) if err != nil { sm.mu.Lock() sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} sm.mu.Unlock() - resultChan <- taskResult{err: err} + broadcaster.complete(taskResult{err: err}) return } @@ -652,7 +650,7 @@ func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bo sm.mu.Lock() sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} sm.mu.Unlock() - resultChan <- taskResult{err: ctx.Err()} + broadcaster.complete(taskResult{err: ctx.Err()}) return } @@ -668,7 +666,7 @@ func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bo sm.mu.Lock() sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} sm.mu.Unlock() - resultChan <- taskResult{err: err} + broadcaster.complete(taskResult{err: err}) return } @@ -681,10 +679,10 @@ func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bo sm.incrementReferenceCount(simulator) sm.mu.Unlock() - resultChan <- taskResult{udid: simulator, err: nil} + broadcaster.complete(taskResult{udid: simulator, err: nil}) }() - return resultChan, cancel + return broadcaster, cancel } func (sm *SimulatorManager) registerReleaseOnExit(leaser int32) { diff --git a/tools/simulator_manager/go/simulator_manager_test.go b/tools/simulator_manager/go/simulator_manager_test.go new file mode 100644 index 0000000..20c4144 --- /dev/null +++ b/tools/simulator_manager/go/simulator_manager_test.go @@ -0,0 +1,337 @@ +package main + +import ( + "os" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestManager(control SimulatorControl, deleteIdleAfter uint16, deleteRecentlyUsedIdleAfter uint16, deleteOnPIDExit bool, leaseStore LeaseStore) *SimulatorManager { + return NewSimulatorManager(control, deleteRecentlyUsedIdleAfter, deleteIdleAfter, 1, deleteOnPIDExit, nil, nil, leaseStore) +} + +func testConfig(deviceType string) SimulatorConfig { + return SimulatorConfig{DeviceType: deviceType, OS: "iOS", Version: "17.0"} +} + +// spawnDeadPID starts and waits for a trivial subprocess to exit, then +// returns its PID. Once a process is reaped, the kernel guarantees +// kill(pid, 0) reports ESRCH for that PID (barring an extremely unlikely +// immediate reuse), making this a reliable "definitely not running" PID. +func spawnDeadPID(t *testing.T) int32 { + t.Helper() + cmd := exec.Command("/usr/bin/true") + require.NoError(t, cmd.Run()) + return int32(cmd.Process.Pid) +} + +// spawnLiveProcess starts a long-lived subprocess, killing it automatically +// at test cleanup, and returns its PID. +func spawnLiveProcess(t *testing.T) (*exec.Cmd, int32) { + t.Helper() + cmd := exec.Command("/bin/sleep", "30") + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + return cmd, int32(cmd.Process.Pid) +} + +func TestLease_NonExclusive_SharesDevice(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + + udid2, err := sm.Lease(2, false, cfg) + require.NoError(t, err) + + assert.Equal(t, udid1, udid2, "two non-exclusive leases for the same config should share one device") + assert.Len(t, control.cloneCalls, 1, "only one clone should have been created") +} + +func TestLease_Exclusive_NeverShares(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, true, cfg) + require.NoError(t, err) + + udid2, err := sm.Lease(2, true, cfg) + require.NoError(t, err) + + assert.NotEqual(t, udid1, udid2, "exclusive leases must never share a device") + assert.Len(t, control.cloneCalls, 2) +} + +func TestLease_ExclusiveDoesNotShareWithNonExclusive(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + nonExclusive, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + + exclusive, err := sm.Lease(2, true, cfg) + require.NoError(t, err) + + assert.NotEqual(t, nonExclusive, exclusive) +} + +func TestLease_SamePID_AlreadyLeased(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + + _, err = sm.Lease(1, false, cfg) + require.Error(t, err) + + sme, ok := err.(*SimulatorManagerError) + require.True(t, ok, "expected a *SimulatorManagerError, got %T", err) + assert.Equal(t, "alreadyLeased", sme.kind) + assert.Equal(t, udid, sme.udid) +} + +func TestRelease_NoLease(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + + err := sm.Release(1) + assert.Equal(t, ErrNoLease, err) +} + +func TestRelease_DeletesImmediatelyWhenIdleAfterIsZero(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + cfg := testConfig("iPhone") + + udid, err := sm.Lease(1, true, cfg) + require.NoError(t, err) + + require.NoError(t, sm.Release(1)) + + assert.Equal(t, []SimulatorUDID{udid}, control.deleteCalls) + assert.Equal(t, []SimulatorUDID{udid}, control.cleanTempFilesCalls) +} + +func TestRelease_SharedDevice_OnlyDeletesAfterLastRelease(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + cfg := testConfig("iPhone") + + udid, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + _, err = sm.Lease(2, false, cfg) + require.NoError(t, err) + + require.NoError(t, sm.Release(1)) + assert.Empty(t, control.deleteCalls, "device is still referenced by PID 2") + + require.NoError(t, sm.Release(2)) + assert.Equal(t, []SimulatorUDID{udid}, control.deleteCalls, "device should be deleted once unreferenced") +} + +func TestPendingDeletion_Resurrection(t *testing.T) { + control := newFakeSimulatorControl() + // A long idle timeout means the background deletion task has no chance to + // fire before this test's synchronous re-lease below. + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + require.NoError(t, sm.Release(1)) + assert.Empty(t, control.deleteCalls, "device should be pending deletion, not deleted yet") + + udid2, err := sm.Lease(2, false, cfg) + require.NoError(t, err) + + assert.Equal(t, udid1, udid2, "a new lease should resurrect the pending-deletion device") + assert.Len(t, control.cloneCalls, 1, "resurrection must not create a second device") +} + +func TestReuseSimulator_InvalidDeviceIsRecreated(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + require.NoError(t, sm.Release(1)) + + // Simulate the device having gone corrupt/invalid in the interim. + control.ensureBootedErrs[udid1] = &ProcessError{ExitCode: 148} + + udid2, err := sm.Lease(2, false, cfg) + require.NoError(t, err) + + assert.NotEqual(t, udid1, udid2, "an invalid device must be replaced, not reused") + assert.Contains(t, control.deleteCalls, udid1, "the corrupt device should have been deleted") + assert.Len(t, control.cloneCalls, 2, "one original clone plus one replacement") +} + +func TestLease_ConcurrentNonExclusiveRequests_ShareOneInFlightClone(t *testing.T) { + control := newFakeSimulatorControl() + control.cloneGate = make(chan struct{}) + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + type leaseResult struct { + udid SimulatorUDID + err error + } + results := make(chan leaseResult, 2) + + go func() { + udid, err := sm.Lease(1, false, cfg) + results <- leaseResult{udid, err} + }() + + // Wait for the first request to actually start cloning (and block on the + // gate) before issuing the second, so this deterministically exercises + // the "join an in-flight clone" path rather than racing two fresh clones. + require.Eventually(t, func() bool { + control.mu.Lock() + defer control.mu.Unlock() + return len(control.cloneCalls) == 1 + }, 2*time.Second, 5*time.Millisecond) + + go func() { + udid, err := sm.Lease(2, false, cfg) + results <- leaseResult{udid, err} + }() + + // Give the second request a moment to reach the shared pendingCreation + // slot before unblocking the clone, then let it complete. + time.Sleep(50 * time.Millisecond) + close(control.cloneGate) + + first := <-results + second := <-results + require.NoError(t, first.err) + require.NoError(t, second.err) + + assert.Equal(t, first.udid, second.udid, "both leasers should get the same, singly-cloned device") + assert.Len(t, control.cloneCalls, 1, "only one clone should ever have been started") +} + +func TestLease_LeaserAlreadyExited(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, true, nil) + deadPID := spawnDeadPID(t) + + _, err := sm.Lease(deadPID, false, testConfig("iPhone")) + + assert.Equal(t, ErrLeaserExited, err) + assert.Empty(t, control.cloneCalls, "must not provision a device for a leaser that's already gone") +} + +func TestDeleteOnPIDExit_AutoReleasesWhenProcessDies(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, true, nil) + cmd, pid := spawnLiveProcess(t) + + udid, err := sm.Lease(pid, true, testConfig("iPhone")) + require.NoError(t, err) + + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + + require.Eventually(t, func() bool { + return sm.Release(pid) == ErrNoLease + }, 3*time.Second, 50*time.Millisecond, "manager should auto-release once the leasing process exits") + + assert.Contains(t, control.deleteCalls, udid) +} + +func TestLiveLeaseCount(t *testing.T) { + control := newFakeSimulatorControl() + // deleteOnPIDExit disabled so a lease for an already-dead PID isn't + // rejected or auto-released -- this test wants both a live and a dead + // lease to coexist in order to prove LiveLeaseCount filters correctly. + sm := newTestManager(control, 60, 60, false, nil) + + _, livePID := spawnLiveProcess(t) + deadPID := spawnDeadPID(t) + + _, err := sm.Lease(livePID, true, testConfig("iPhone-live")) + require.NoError(t, err) + _, err = sm.Lease(deadPID, true, testConfig("iPhone-dead")) + require.NoError(t, err) + + assert.Equal(t, 1, sm.LiveLeaseCount()) +} + +func TestRestoreLeases_AdoptsRunningLease_DropsExitedLease(t *testing.T) { + control := newFakeSimulatorControl() + deadPID := spawnDeadPID(t) + // The test process itself is guaranteed to still be running. + livePID := int32(os.Getpid()) + + store := &fakeLeaseStore{ + loaded: []PersistedLease{ + { + PID: livePID, + UDID: "udid-running", + Config: testConfig("running"), + Exclusive: true, + SlotIndex: 0, + }, + { + PID: deadPID, + UDID: "udid-exited", + Config: testConfig("exited"), + Exclusive: true, + SlotIndex: 0, + }, + }, + } + + sm := newTestManager(control, 60, 60, false, store) + sm.RestoreLeases() + + assert.NoError(t, sm.Release(livePID), "the lease for a still-running process should have been adopted") + assert.Equal(t, ErrNoLease, sm.Release(deadPID), "the lease for an exited process should have been dropped") +} + +func TestRestoreLeases_ConflictingExclusiveLeases_KeepsFirstOnly(t *testing.T) { + control := newFakeSimulatorControl() + _, pid1 := spawnLiveProcess(t) + _, pid2 := spawnLiveProcess(t) + + cfg := testConfig("iPhone") + store := &fakeLeaseStore{ + loaded: []PersistedLease{ + {PID: pid1, UDID: "udid-shared", Config: cfg, Exclusive: true, SlotIndex: 0}, + {PID: pid2, UDID: "udid-shared", Config: cfg, Exclusive: true, SlotIndex: 0}, + }, + } + + sm := newTestManager(control, 60, 60, false, store) + sm.RestoreLeases() + + firstErr := sm.Release(pid1) + secondErr := sm.Release(pid2) + + // Restoration walks the persisted list in order, so the first entry wins + // and the second is rejected as conflicting -- exactly one adopted. + adopted := 0 + if firstErr == nil { + adopted++ + } + if secondErr == nil { + adopted++ + } + assert.Equal(t, 1, adopted, "exactly one of the two conflicting leases should have been adopted") +} diff --git a/tools/simulator_manager/go/simulator_request_handler_test.go b/tools/simulator_manager/go/simulator_request_handler_test.go new file mode 100644 index 0000000..b5de461 --- /dev/null +++ b/tools/simulator_manager/go/simulator_request_handler_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "net/http" + "net/url" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestRequestHandler() (*SimulatorRequestHandler, *fakeSimulatorControl) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + return NewSimulatorRequestHandler(sm), control +} + +func TestHandleRequest_UnsupportedMethod(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodGet, "/simulator/1", url.Values{}) + + assert.Equal(t, http.StatusMethodNotAllowed, resp.Status) +} + +func TestHandlePost_MissingPID(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodPost, "/simulator/", url.Values{}) + + assert.Equal(t, http.StatusBadRequest, resp.Status) + assert.Contains(t, resp.Message, "leaser PID") +} + +func TestHandlePost_NonIntegerPID(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodPost, "/simulator/not-a-pid", url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}}) + + assert.Equal(t, http.StatusBadRequest, resp.Status) +} + +func TestHandlePost_MissingQueryParams(t *testing.T) { + h, _ := newTestRequestHandler() + + cases := []struct { + name string + params url.Values + }{ + {"missing exclusive", url.Values{"deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}}}, + {"missing deviceType", url.Values{"exclusive": {"1"}, "os": {"iOS"}, "version": {"17.0"}}}, + {"missing os", url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "version": {"17.0"}}}, + {"missing version", url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}}}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resp := h.HandleRequest(http.MethodPost, "/simulator/1", c.params) + assert.Equal(t, http.StatusBadRequest, resp.Status) + }) + } +} + +func TestHandlePost_Success(t *testing.T) { + h, control := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodPost, "/simulator/1", url.Values{ + "exclusive": {"1"}, + "deviceType": {"iPhone"}, + "os": {"iOS"}, + "version": {"17.0"}, + }) + + require.Equal(t, http.StatusCreated, resp.Status) + assert.Equal(t, control.cloneCalls[0], "EXAMPLE_BAZEL_CLONE_iPhone_17.0_0") + assert.NotEmpty(t, resp.Message, "response body should carry the leased UDID") +} + +func TestHandlePost_NonExclusiveWhenNotSetTo1(t *testing.T) { + h, control := newTestRequestHandler() + + // Anything other than exactly "1" for `exclusive` should be treated as + // non-exclusive, per the query-param contract. + _ = h.HandleRequest(http.MethodPost, "/simulator/1", url.Values{ + "exclusive": {"0"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}, + }) + first := h.HandleRequest(http.MethodPost, "/simulator/2", url.Values{ + "exclusive": {"0"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}, + }) + + require.Equal(t, http.StatusCreated, first.Status) + assert.Len(t, control.cloneCalls, 1, "both non-exclusive leases should share the same device") +} + +func TestHandlePost_AlreadyLeased(t *testing.T) { + h, _ := newTestRequestHandler() + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + + first := h.HandleRequest(http.MethodPost, "/simulator/1", params) + require.Equal(t, http.StatusCreated, first.Status) + + second := h.HandleRequest(http.MethodPost, "/simulator/1", params) + assert.Equal(t, http.StatusBadRequest, second.Status) + assert.Contains(t, second.Message, "already leased") +} + +func TestHandleDelete_MissingPID(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodDelete, "/simulator/", url.Values{}) + + assert.Equal(t, http.StatusBadRequest, resp.Status) +} + +func TestHandleDelete_NoLease(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodDelete, "/simulator/1", url.Values{}) + + assert.Equal(t, http.StatusNotFound, resp.Status) + assert.Contains(t, resp.Message, "doesn't have a simulator leased") +} + +func TestHandleDelete_Success(t *testing.T) { + h, _ := newTestRequestHandler() + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + + require.Equal(t, http.StatusCreated, h.HandleRequest(http.MethodPost, "/simulator/1", params).Status) + + resp := h.HandleRequest(http.MethodDelete, "/simulator/1", url.Values{}) + assert.Equal(t, http.StatusOK, resp.Status) +} + +func TestLiveLeaseCount_Passthrough(t *testing.T) { + h, _ := newTestRequestHandler() + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + + assert.Equal(t, 0, h.LiveLeaseCount()) + + // This test process's own PID is guaranteed to be running. + pid := strconv.Itoa(os.Getpid()) + require.Equal(t, http.StatusCreated, h.HandleRequest(http.MethodPost, "/simulator/"+pid, params).Status) + + assert.Equal(t, 1, h.LiveLeaseCount()) +} From bacfd647519b458b368249d3fdceec7c694165fe Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Fri, 21 Aug 2026 10:22:21 +0200 Subject: [PATCH 11/11] more --- .../go/fake_simulator_control_test.go | 25 ++++++++++ .../simulator_manager/go/simulator_control.go | 35 ++++++++++++- .../simulator_manager/go/simulator_manager.go | 41 ++++++++++++++++ .../go/simulator_manager_test.go | 49 +++++++++++++++++++ 4 files changed, 148 insertions(+), 2 deletions(-) diff --git a/tools/simulator_manager/go/fake_simulator_control_test.go b/tools/simulator_manager/go/fake_simulator_control_test.go index 4a15b6d..ae1c148 100644 --- a/tools/simulator_manager/go/fake_simulator_control_test.go +++ b/tools/simulator_manager/go/fake_simulator_control_test.go @@ -32,12 +32,20 @@ type fakeSimulatorControl struct { deleteErrs map[SimulatorUDID]error cleanTempFilesCalls []SimulatorUDID + + // runningOverride, keyed by clone name, is returned by RunningSimulators + // for that name. The fake doesn't model real booted state on its own + // (there's nothing to derive it from), so this defaults to empty -- + // tests that want to exercise the "duplicate running simulator" assertion + // set an override explicitly. + runningOverride map[string][]SimCtlDevice } func newFakeSimulatorControl() *fakeSimulatorControl { return &fakeSimulatorControl{ ensureBootedErrs: make(map[SimulatorUDID]error), deleteErrs: make(map[SimulatorUDID]error), + runningOverride: make(map[string][]SimCtlDevice), } } @@ -109,6 +117,23 @@ func (f *fakeSimulatorControl) GetExisting(name string, deviceType string, runti return "", nil } +func (f *fakeSimulatorControl) RunningSimulators(name string) ([]SimCtlDevice, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.runningOverride[name], nil +} + +// erroringRunningSimulatorsControl wraps a fakeSimulatorControl to make +// RunningSimulators always fail, for testing that the lease-time assertion +// treats a failure to perform the check as non-fatal to the lease itself. +type erroringRunningSimulatorsControl struct { + *fakeSimulatorControl +} + +func (c *erroringRunningSimulatorsControl) RunningSimulators(name string) ([]SimCtlDevice, error) { + return nil, fmt.Errorf("simctl unavailable") +} + // fakeLeaseStore is an in-memory LeaseStore for testing RestoreLeases and // persistence side effects without touching disk. type fakeLeaseStore struct { diff --git a/tools/simulator_manager/go/simulator_control.go b/tools/simulator_manager/go/simulator_control.go index b06bf0a..38c53e1 100644 --- a/tools/simulator_manager/go/simulator_control.go +++ b/tools/simulator_manager/go/simulator_control.go @@ -40,8 +40,9 @@ type SimCtlDevices struct { } type SimCtlDevice struct { - Name string `json:"name"` - UDID string `json:"udid"` + Name string `json:"name"` + UDID string `json:"udid"` + State string `json:"state"` } type ProcessError struct { @@ -85,6 +86,13 @@ type SimulatorControl interface { Delete(simulator SimulatorUDID, name string, context string) error GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) + + // RunningSimulators returns every currently-booted simulator named `name`, + // across all runtimes. Used as a lease-time sanity check: a shared device + // should only ever have one real, booted simulator behind it, so finding + // more than one indicates the sharing/reuse logic let a duplicate device + // come into existence. + RunningSimulators(name string) ([]SimCtlDevice, error) } type RealSimulatorControl struct { @@ -321,6 +329,29 @@ func (r *RealSimulatorControl) Delete(simulator SimulatorUDID, name string, cont }) } +func (r *RealSimulatorControl) RunningSimulators(name string) ([]SimCtlDevice, error) { + output, err := simctl([]string{"list", "devices", "-j"}, "assertAtMostOneRunning") + if err != nil { + return nil, err + } + + var devices SimCtlDevices + if err := json.Unmarshal([]byte(output), &devices); err != nil { + simulatorControlLogger.Error("Failed to decode 'simctl list devices -j'", "error", err, "output", output) + return nil, fmt.Errorf("failed to decode output: %w - %s", err, output) + } + + var running []SimCtlDevice + for _, deviceList := range devices.Devices { + for _, device := range deviceList { + if device.Name == name && device.State == "Booted" { + running = append(running, device) + } + } + } + return running, nil +} + func (r *RealSimulatorControl) GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { var result string err := r.deleteAndExistenceMutex(name, func(mutex *SimulatorDeleteOrExistenceMutex) error { diff --git a/tools/simulator_manager/go/simulator_manager.go b/tools/simulator_manager/go/simulator_manager.go index 20c5c91..951742c 100644 --- a/tools/simulator_manager/go/simulator_manager.go +++ b/tools/simulator_manager/go/simulator_manager.go @@ -6,6 +6,7 @@ import ( "io" "os" "os/exec" + "strings" "sync" "syscall" "time" @@ -366,6 +367,10 @@ func (sm *SimulatorManager) Lease(leaser int32, exclusive bool, config Simulator return "", err } + if err := sm.assertAtMostOneRunning(config, slotIndex); err != nil { + return "", err + } + sm.mu.Lock() sm.recentlyLeased.Insert(config) @@ -396,6 +401,42 @@ func (sm *SimulatorManager) Lease(leaser int32, exclusive bool, config Simulator return simulator, nil } +// assertAtMostOneRunning is a lease-time sanity check, not a recoverable +// error path: a device backing one slot should only ever have one real, +// booted simulator behind it. If simctl reports more than one booted +// simulator sharing this slot's name, the reuse/sharing logic has let a +// duplicate device come into existence -- exactly the failure mode that +// causes leaked, unreferenced launchd_sim processes. Failing loudly here +// catches that at the moment it happens, rather than relying solely on +// after-the-fact reaping. +// +// A failure to even perform the check (simctl itself erroring) is logged and +// ignored rather than failing the lease -- the check is a diagnostic, and +// its own failure shouldn't block a lease that's otherwise fine. +func (sm *SimulatorManager) assertAtMostOneRunning(config SimulatorConfig, slotIndex int) error { + name := config.CloneDeviceName(slotIndex) + + running, err := sm.simulatorControl.RunningSimulators(name) + if err != nil { + logger.Warn("Failed to list running simulators for lease assertion", "name", name, "error", err) + return nil + } + + if len(running) <= 1 { + return nil + } + + udids := make([]string, len(running)) + for i, device := range running { + udids[i] = device.UDID + } + + return fmt.Errorf( + "assertion failed: %d simulators named %q are running (expected at most 1): %s", + len(running), name, strings.Join(udids, ", "), + ) +} + func (sm *SimulatorManager) LiveLeaseCount() int { sm.mu.Lock() defer sm.mu.Unlock() diff --git a/tools/simulator_manager/go/simulator_manager_test.go b/tools/simulator_manager/go/simulator_manager_test.go index 20c4144..c9ee2d4 100644 --- a/tools/simulator_manager/go/simulator_manager_test.go +++ b/tools/simulator_manager/go/simulator_manager_test.go @@ -255,6 +255,55 @@ func TestDeleteOnPIDExit_AutoReleasesWhenProcessDies(t *testing.T) { assert.Contains(t, control.deleteCalls, udid) } +func TestLease_FailsAssertionWhenMoreThanOneRunningSimulatorShareTheSlot(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + // Simulate the exact failure mode the assertion exists to catch: two real + // booted devices ended up sharing what should be one slot's name. + name := cfg.CloneDeviceName(0) + control.runningOverride[name] = []SimCtlDevice{ + {Name: name, UDID: "udid-a", State: "Booted"}, + {Name: name, UDID: "udid-b", State: "Booted"}, + } + + _, err := sm.Lease(1, false, cfg) + + require.Error(t, err) + assert.Contains(t, err.Error(), "assertion failed") + assert.Contains(t, err.Error(), "udid-a") + assert.Contains(t, err.Error(), "udid-b") + assert.Equal(t, ErrNoLease, sm.Release(1), "a lease that failed the assertion must not have been recorded") + assert.Equal(t, 0, sm.LiveLeaseCount()) +} + +func TestLease_SucceedsWhenAtMostOneRunningSimulatorForTheSlot(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + name := cfg.CloneDeviceName(0) + control.runningOverride[name] = []SimCtlDevice{ + {Name: name, UDID: "udid-a", State: "Booted"}, + } + + udid, err := sm.Lease(1, false, cfg) + + require.NoError(t, err) + assert.NotEmpty(t, udid) +} + +func TestLease_IgnoresRunningSimulatorsListErrorRatherThanFailingTheLease(t *testing.T) { + control := &erroringRunningSimulatorsControl{fakeSimulatorControl: newFakeSimulatorControl()} + sm := newTestManager(control, 60, 60, false, nil) + + udid, err := sm.Lease(1, false, testConfig("iPhone")) + + require.NoError(t, err, "a failure to run the diagnostic check itself must not fail the lease") + assert.NotEmpty(t, udid) +} + func TestLiveLeaseCount(t *testing.T) { control := newFakeSimulatorControl() // deleteOnPIDExit disabled so a lease for an already-dead PID isn't