diff --git a/MODULE.bazel b/MODULE.bazel index a3ee6a5..c402d9a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -99,7 +99,7 @@ go_sdk.download(version = GO_VERSION) go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") -use_repo(go_deps, "com_github_google_go_cmp") +use_repo(go_deps, "com_github_google_go_cmp", "org_golang_x_sys", "com_github_stretchr_testify") bazel_dep(name = "rules_java", version = "9.6.1") diff --git a/go.mod b/go.mod index 49d14c0..a5f27dc 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,12 @@ module github.com/EngFlow/example -go 1.21.2 +go 1.25.0 require github.com/google/go-cmp v0.6.0 + +require golang.org/x/sys v0.47.0 + +require ( + github.com/stretchr/testify v1.12.1 + go.yaml.in/yaml/v3 v3.0.5 // indirect +) diff --git a/go.sum b/go.sum index 5a8d551..375b797 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/tools/simulator_manager/BUILD b/tools/simulator_manager/BUILD index 54fda20..115cc17 100644 --- a/tools/simulator_manager/BUILD +++ b/tools/simulator_manager/BUILD @@ -49,7 +49,7 @@ sh_binary( data = [ ":install_post_boot_script", ":prepare_simulator", - ":simulator_manager", + "//tools/simulator_manager/go", ], visibility = ["//visibility:public"], deps = ["@bazel_tools//tools/bash/runfiles"], diff --git a/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md b/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md new file mode 100644 index 0000000..4747b3a --- /dev/null +++ b/tools/simulator_manager/Sources/LEASE_LIFECYCLE.md @@ -0,0 +1,279 @@ +# The lifecycle of a simulator lease + +This is a reasoning walkthrough of everything that happens to one lease, from +`POST /simulator/` to the device eventually being deleted (or handed to +someone else). It's meant to make the state machine in `SimulatorManager.swift` +easy to hold in your head, not to restate the architecture already covered in +`README.md`. + +## The actors + +- **A slot** (`SimulatorSlot`) is one device-shaped bucket for a `SimulatorConfig`. + It's `empty`, `pendingCreation`, `active`, `pendingDeletion`, or `deleting`. +- **A lease** (`SimulatorLease` in memory, `PersistedLease` on disk) is the + daemon's record that PID `p` is holding device `udid` in slot `i`. +- **A reference count** (`referenceCount[udid]`) is how many leases currently + point at that device. Non-exclusive leases for the same config can share one + device, so this can be > 1; exclusive leases never share, so it's always + exactly 1 while active. + +A lease and a slot are related but not the same thing: the slot is about the +*device*, the lease is about *who's holding it*. A non-exclusive device can +outlive any single lease on it (reused by the next leaser), and a lease always +maps to exactly one slot at a time. + +## Phase 1 — request arrives + +`SimulatorRequestHandler` parses the HTTP request and calls +`SimulatorManager.lease(to:exclusive:config:)`. First check, before anything +else: does this PID already have a lease? (`leases[leaser]` in +`SimulatorManager.swift:305`). One process, one simulator — this isn't a +counter, it's a hard rule enforced synchronously (no `await` before the +check), so two concurrent requests from the same PID can't both slip through. + +## Phase 2 — is the leaser even still there? + +Before doing anything expensive, the manager checks `processIsRunning(leaser)` +(only when `deleteOnPIDExit` is set — otherwise the caller owns the PID's +lifetime and it need not be a real process at all, e.g. in tests). This looks +redundant — surely the caller wouldn't ask for a lease if it's already dead — +but the point isn't "is it dead right now," it's "is it dead *before we commit +to a multi-minute operation*." Provisioning is the expensive part; checking +liveness is nearly free. This is the cheap half of a two-part guard; the +expensive half is Phase 4. + +## Phase 3 — finding a device: the slot state machine + +This is the core of `getSimulator()`. Slots for the requested config are +sorted by `SimulatorSlot.sortOrder` and walked in order, taking the first one +that matches. The order (after the fix earlier in this session) is: + +1. **`active`, non-exclusive, and the request is non-exclusive** — reuse it + directly. This is the fast path: no provisioning at all, just + `ensureBooted` as a sanity check. +2. **`pendingDeletion`** — a device that's fully created and booted, just + waiting out its idle timer. Reusing it is just as cheap as case 1; the + *only* reason it wasn't in case 1 is that its reference count already hit + zero once. Grabbing it cancels the scheduled deletion task and reassigns + its exclusivity to whatever the new lease wants — which is fine, since + "pending deletion" means nobody currently holds it, so there's no + conflicting owner to worry about. +3. **`empty`** — nobody's even started building a device for this slot index. + Start a fresh clone. +4. **`pendingCreation`, non-exclusive, and the request is non-exclusive** — + someone else is already cloning a device for this config; wait on their + task and count as a second (or third...) leaser of it once it's ready. + +The interesting design choice is putting `empty` *ahead of* `pendingCreation`. +It means a new non-exclusive request would rather kick off a second, +independent clone than wait behind someone else's in-flight one, even though +the in-flight one might finish sooner. My read: this trades a bit of +redundant provisioning work for not making a request's latency depend on +someone else's request. It also means the pool tends to grow to "however many +concurrent first-requests there were for a config" before it starts +consolidating onto reuse — which matches the doc's framing of `pendingCreation` +sharing as a bonus for latecomers, not the primary mechanism. + +Exclusive requests only ever match case 3 (fresh empty slot) or fall through +to appending a brand new slot — they never share `active`, `pendingCreation`, +or (implicitly, since case 2 always wins first if reached) another lease's +slot. `pendingDeletion` is the one case exclusive requests *do* match, and +that's safe precisely because "pending deletion" implies zero current owners. + +If nothing matches, a new slot is appended at the end of the array — the pool +for a config only grows on demand, never pre-allocated. + +## Phase 4 — provisioning, and when the reference count actually increments + +This part is subtle because of the actor-isolation rule spelled out in a +comment above `getSimulator`: slot state must be updated *before* the first +`await`, or a concurrent call could observe a stale slot. That rule shapes +where reference counting happens too — it can't just happen "whenever," it has +to happen at the exact point where the device becomes claimed, before any +suspension point could let the device get swept into deletion by someone else +finishing first. Three different code paths each own this responsibility: + +- **Reuse (`reuseSimulator`)**: increments immediately, *then* awaits + `ensureBooted`. If boot fails with "invalid device" (exit 148), it deletes + the corpse and clones a replacement — the increment from the original + attempt is moot because `delete()` wipes the reference count entry outright, + and the replacement gets its own fresh increment via `createCloneTask`. +- **Fresh clone (`createCloneTask`)**: increments right after the clone + finishes, before returning, on the theory (per the comment in the code) + that whoever created the device should be the one to count it, and anyone + who later shares that same task must increment separately. +- **Joining an in-flight `pendingCreation`**: increments after awaiting the + shared task's result — this is the "anyone who shares it counts separately" + half of the previous point. + +So for a config with 3 non-exclusive leases sharing one device, there are 3 +separate `incrementReferenceCount` calls across possibly all three code paths, +never a batch increment. That's what makes "decrement to zero → pendingDeletion" +in Phase 6 a safe signal: it really does mean the last leaser let go. + +Once a device is in hand, the lease is recorded in `leases[leaser]` and +persisted to disk (`persistLeases()`) *before* the second liveness check. Then +liveness is checked again — the expensive half of the Phase 2 guard. Cloning +and booting can take a real amount of wall-clock time (comment says +"minutes"), long enough for a test harness's timeout to have killed the leaser +already. If that happened, the manager doesn't hand back a UDID nobody will +ever release — it calls `release(for: leaser)` on itself immediately and +reports `leaserExited`. Recording the lease first (even though it's about to +be released) is what lets `release()` work at all; it operates purely on +`leases[leaser]`, it has no other way to find the device. + +Only after surviving both liveness checks does the manager register a process- +exit watcher (`registerReleaseOnExit`) — no point watching a PID you're not +going to hand anything to. + +## Phase 5 — active + +The device sits in `leases[leaser]` and as `.active(udid, exclusive)` in its +slot, reference-counted. Nothing happens to it on its own; it just waits for +one of the release triggers below. Multiple non-exclusive leases can be +pointing at the same UDID simultaneously, each with their own entry in +`leases`, all decrementing the same shared `referenceCount[udid]` independently +whenever *they* release. + +## Phase 6 — release: three ways in + +1. **Explicit `DELETE /simulator/`** → `SimulatorRequestHandler` → + `SimulatorManager.release(for:)`. +2. **The leasing process exits** without ever calling DELETE — caught by the + `DispatchSourceProcess` registered in Phase 4, which calls `release(for:)` + itself. This is what makes leases safe against a build tool that + `kill -9`s a test that forgot to clean up. +3. **The daemon is replaced and the leaser didn't survive the handover** — + handled entirely in `restoreLeases()` at startup, not in `release()` at all; + see Phase 8. The lease is just never adopted, so no explicit release call + ever happens for it, and the device is later picked up implicitly by name. + +`release()` itself is small and ordered deliberately: + +1. Pop the lease out of `leases` (also the point where a nonexistent lease + surfaces as `SimulatorManagerError.noLease` — expected after a daemon + restart dropped a lease the caller doesn't know is gone). +2. Persist the *now-shorter* lease list — before touching the device, so a + daemon replaced mid-release doesn't try to re-adopt a lease whose device is + about to be torn down. +3. Cancel the exit watcher (no longer relevant). +4. Clean temp files on the device. +5. Decrement the reference count, which is where the real branch happens. + +## Phase 7 — idle, and the reference count reaching zero + +`decrementReferenceCount` only does something interesting when the count hits +zero: it hands off to `pendingDeletion()`. If both idle timers are configured +as zero, deletion is immediate — no grace period at all. Otherwise a task is +scheduled that polls once a second, and *at every tick* re-decides which +deadline applies by checking `recentlyLeased.contains(config)` fresh. That's a +live check, not a snapshot taken when the task started — a config that gets +leased again elsewhere while this device is winding down flips it from the +short deadline to the long one (or the reverse, if it falls out of the +LRU set's capacity) mid-wait. It's a genuinely dynamic decision, not "pick a +deadline once." + +Two ways out of `pendingDeletion`: + +- **Resurrection**: a new lease for the same config finds this slot before the + timer expires (Phase 3, case 2) and turns it back into `active`. The + scheduled task gets cancelled, and because `Task.sleep` throws on + cancellation, the task body simply unwinds without ever calling `delete()`. +- **Timeout**: the deadline is reached, the task double-checks the slot still + holds *this exact device* (guarding against a resurrection racing the + cancellation), and calls `delete()` for real — `simctl delete`, slot to + `.deleting` then `.empty`, and (if `cleanUpSlots: true`) trims trailing empty + slots off the end of the array so the pool doesn't grow unboundedly with + dead slot indices. + +## Phase 8 — surviving a daemon restart + +This is the one part of the lifecycle that doesn't originate from an HTTP +request at all — it happens once, at startup, before the server even binds. +`start.sh` replaces the daemon on every new version, which would otherwise +silently orphan every in-flight lease. The fix is that every lease mutation in +Phases 4, 6, and 7 already persists the *entire* current lease set to disk +(atomically), so `restoreLeases()` just has to read that file back and decide, +per lease, whether to trust it: + +1. **Did the leasing process survive?** Not just "is this PID alive" — PIDs + recycle, so it also compares the process's actual start time + (`kinfo_proc`) against what was recorded at lease time. A live PID with a + *different* start time means a different process entirely; treated as + exited. +2. **Does adopting it contradict something already restored?** Two leases + can't share one exclusive device, and one slot can't hold two devices. A + file that claims otherwise is treated as corrupt rather than trusted. + +A lease that passes both checks gets its slot rebuilt as `.active`, its +reference count bumped, and (if configured) a fresh exit watcher — from that +point on it's indistinguishable from a lease that was granted moments ago by +this same daemon. A lease that fails either check is simply dropped — not +released, not cleaned up, just forgotten. Its device isn't touched at all. +`createBase`/`clone` look up devices by name before creating new ones, so an +orphaned device gets rediscovered and correctly reference-counted the next +time something happens to lease that same config — but if nothing ever leases +that exact config again, this lazy reconciliation never fires. See Phase 9 +for the backstop that catches that case. + +## Phase 9 — the orphan reaper + +Every cleanup path above is triggered by an event: a release call, a PID +exit, an idle timer, or "the same config got leased again." Each of those can +miss a device — Phase 8 above is one way (a lease that never gets re-leased), +and a `simctl delete` call inside `delete()` failing silently (it's called as +`try? await delete(...)`, and the slot is reset to `.empty` in a `defer` +regardless of whether the delete actually succeeded) is another. Either way, +nothing else ever looks for that device again. + +`delete()` failing isn't necessarily transient, either: `simctl delete` isn't +reliable against a still-booted device, and nothing in the original lifecycle +ever shuts a clone down before deleting it (a clone goes straight from +"active" to "delete this" — there's no shutdown step in between). A device +that's genuinely wedged (unresponsive, the same profile as a `launchd_sim` +that's been running for days) will fail both `shutdown` and `delete` for the +same reason every time, not just once. + +`startReaper(interval:)` runs a sweep on a timer, independent of any lease +event, that reconciles against reality instead of the event stream: it lists +every simulator named with the manager's clone prefix +(`SimulatorConfig.cloneDeviceName`, so base simulators and anything not +created by this manager are never touched) and deletes any whose UDID has no +entry in `referenceCount` — the same dictionary key presence check works here +as everywhere else, since an entry exists from the moment a device is claimed +until `delete()` removes it, including through the idle-timer grace period. + +To avoid deleting a clone that's mid-creation (it exists on disk for a moment +before `createCloneTask` resumes and records it in `referenceCount`), a +device must show up as unknown on two consecutive sweeps before it's reaped. +Deletion goes straight through `simulatorControl`, not through `delete()`: +an orphan has no slot pointing at it, so there's nothing in `simulatorSlots` +for `delete()`'s bookkeeping to update. + +`SimulatorControl.delete()` itself now shuts the device down before each of +its own delete attempts (see `shutdownSimulator` in `SimulatorControl.swift`), +so most orphans clean up on the reaper's first attempt. For the wedged case +where that still isn't enough, the reaper tracks how many sweeps in a row a +given UDID has failed to delete (`orphanDeleteFailureCounts`). After three +consecutive failures, `reapOrphan` treats the device as stuck rather than +unlucky and calls `SimulatorControl.forceKillLaunchdSim(for:)`, which finds +the device's `launchd_sim` process by `pgrep -f ` (the process's own +command line embeds its data path, so the UDID is a safe, specific pattern), +double-checks the process name before signaling it, and sends it `SIGKILL` +directly — bypassing `simctl` entirely. One more delete attempt follows; if +that still fails, the device is logged loudly as needing manual cleanup +rather than retried silently forever. + +## Where this got subtle + +- The "mutate slot before the first `await`" rule is easy to state and easy to + violate by accident; it's the reason reference-count bookkeeping is smeared + across three different functions instead of living in one place. +- `pendingDeletion` slots deliberately don't remember their old exclusivity + flag — and don't need to, because reaching that state already implies zero + current owners. +- The dynamic (not snapshotted) recheck of `recentlyLeased` inside the idle- + deletion poll loop means the *duration* of a device's grace period can + change while it's already ticking. +- Persistence is a side effect of every state change, not a periodic snapshot + or a shutdown hook — which is what makes it safe against `kill -9`. diff --git a/tools/simulator_manager/Sources/LRUSet.swift b/tools/simulator_manager/Sources/LRUSet.swift index 972c08c..1f903fd 100644 --- a/tools/simulator_manager/Sources/LRUSet.swift +++ b/tools/simulator_manager/Sources/LRUSet.swift @@ -14,28 +14,28 @@ struct LRUSet { self.capacity = capacity } - // Returns an element that was evicted from the set. + // Returns an element that was evicted from the set, or nil if nothing was evicted + // (including when `element` was already present and just moved to most-recently-used). mutating func insert(_ element: Element) -> Element? { - let evicted: Element? if storage.contains(element) { if let index = order.firstIndex(of: element) { - evicted = order.remove(at: index) - } else { - evicted = nil + order.remove(at: index) } order.append(element) - } else { - if order.count >= capacity, let oldest = order.first { - order.removeFirst() - evicted = storage.remove(oldest) - } else { - evicted = nil - } + return nil + } - order.append(element) - storage.insert(element) + let evicted: Element? + if order.count >= capacity, let oldest = order.first { + order.removeFirst() + evicted = storage.remove(oldest) + } else { + evicted = nil } + + order.append(element) + storage.insert(element) return evicted } diff --git a/tools/simulator_manager/Sources/Main.swift b/tools/simulator_manager/Sources/Main.swift index e3e5dc5..add0094 100644 --- a/tools/simulator_manager/Sources/Main.swift +++ b/tools/simulator_manager/Sources/Main.swift @@ -52,6 +52,15 @@ struct Main: AsyncParsableCommand { ) var leasePath: String? + @Option( + help: """ + Seconds between sweeps of the orphan reaper, which deletes clone simulators the \ + manager has lost track of (e.g. left behind by a prior daemon's restart, or a \ + failed deletion). 0 disables the reaper. + """ + ) + var reapIntervalSeconds: UInt16 = 300 + func validate() throws { guard recentlyUsedCapacity > 0 else { throw ValidationError( @@ -82,6 +91,8 @@ struct Main: AsyncParsableCommand { // daemon inherited. await simulatorManager.restoreLeases() + await simulatorManager.startReaper(interval: .seconds(Int(reapIntervalSeconds))) + try await simulatorManager.startChildProcesses() try await HTTPServer( diff --git a/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md new file mode 100644 index 0000000..46bdcb3 --- /dev/null +++ b/tools/simulator_manager/Sources/ORPHAN_SIMULATOR_FIX.md @@ -0,0 +1,205 @@ +# Orphaned simulator fix + +Customers reported resource contention caused by `launchd_sim` processes that +had been running for days, tied to CoreSimulator devices the daemon no longer +knew about: + +```text +PID PPID RSS ELAPSED COMMAND +78247 1 9728 05-23:05:41 launchd_sim .../Devices/B9A13CD5-.../data/var/run/launchd_bootstrap.plist +78293 1 11600 05-23:05:40 launchd_sim .../Devices/88362C6D-.../... +78337 1 9968 05-23:05:40 launchd_sim .../Devices/B8CD72C0-.../... +``` + +This document explains the two root causes and the fix. See +[`LEASE_LIFECYCLE.md`](LEASE_LIFECYCLE.md) for the full lease state machine +this fix adds to (Phase 9). + +## Root causes + +Every existing cleanup path in `SimulatorManager.swift` is event-driven: an +explicit `DELETE /simulator/`, a PID-exit watcher, an idle timer, or a +device getting rediscovered by name the next time its config is leased +again. There was no path that checked the manager's bookkeeping against what +CoreSimulator actually had running. Two specific gaps let devices escape all +of those events at once. + +### 1. Daemon restarts drop leases without touching their devices + +`start.sh` replaces the daemon on every version change. On startup, +`restoreLeases()` (`SimulatorManager.swift:166`) re-adopts a persisted lease +only if its PID is still alive *and* its recorded process start time matches +(to rule out PID reuse). A lease that fails either check is dropped — +intentionally, per the existing code comment: "not released, not cleaned up, +just forgotten." The device itself is never shut down or deleted. + +The intended recovery path was that `createBase`/`clone` look up devices by +name, so an orphaned device would get picked up and reference-counted again +the next time something leased that same `SimulatorConfig`. If that exact +config was never requested again, nothing ever looked for the device again — +it, and its `launchd_sim`, ran forever. + +### 2. Failed `simctl delete` calls were swallowed silently + +In `delete()` (`SimulatorManager.swift:758-792`), the slot was reset to +`.empty` and the reference-count entry removed in a `defer` — regardless of +whether the `simctl delete` call inside it actually succeeded. Both call +sites invoked this as `try? await delete(...)`, discarding any error with no +retry. If `simctl delete` failed for any reason (CoreSimulator daemon busy, a +lingering child process, disk I/O), the manager's bookkeeping said "deleted" +while the real device kept running — and since its slot was now empty, the +manager would never look at that UDID again. + +Both gaps produce the same outcome: a real, running simulator with zero +corresponding state in the manager. + +## The fix + +### Retry `simctl delete` before giving up + +`SimulatorControl.swift`, `SimulatorDeleteOrExistenceMutex.unlockedDelete` +now retries the underlying `simctl delete` up to 3 times, 2 seconds apart, +before rethrowing. This doesn't touch `SimulatorManager`'s actor-isolated +slot state machine, so it carries no risk to the "mutate slot before the +first `await`" invariant documented above `getSimulator()`. It simply makes +the existing call sites succeed more often instead of falling straight +through to `try?`. + +### Periodic orphan reaper + +The real fix: a sweep that reconciles the manager's state against reality, +independent of any lease event, so it catches a device regardless of *how* +it became untracked. + +- **`SimulatorControl.listManagedClones()`** (new protocol method) lists + every simulator whose name starts with `EXAMPLE_BAZEL_CLONE_` — the prefix + this manager already uses for every clone it creates + (`SimulatorConfig.cloneDeviceName`), now hoisted into the shared constant + `managedCloneNamePrefix`. Base simulators (`EXAMPLE_BAZEL_BASE_...`) are + excluded — they're intentionally long-lived templates and are never + reference-counted, so they must never be touched by the reaper. Anything + without this prefix (a developer's own simulator, Xcode's own devices) is + untouched. +- **`SimulatorManager.reapOrphanedSimulators()`** (new, private) computes + `known = Set(referenceCount.keys)` and treats any listed clone UDID not in + that set as an orphan candidate. `referenceCount` is the right thing to + check: a device gets an entry the instant it's claimed, before any + `await`, and the entry is removed only in `delete()` — including while the + device sits in its idle-timer grace period, where the count is `0` but the + key stays. So "no entry" reliably means "the manager has no idea this + exists." +- **Two-sweep confirmation.** A clone that's mid-creation exists on disk (via + `simctl clone`) for a moment before `createCloneTask` resumes and records + it in `referenceCount`. To avoid mistaking that window for an orphan, a + device must show up as unknown on two consecutive sweeps + (`previousOrphanCandidates` intersected with the current sweep's + candidates) before it's deleted. +- **Deletes bypass `delete()`.** An orphan has no slot pointing at it in + `simulatorSlots`, so there's nothing for `delete()`'s slot bookkeeping to + update. The reaper calls `simulatorControl.delete()` directly instead — the + same lower-level operation `delete()` itself wraps. +- **`SimulatorManager.startReaper(interval:)`** (new, public) runs the sweep + on a loop and is started from `Main.swift` right after `restoreLeases()`. + The interval is configurable via the new `--reap-interval-seconds` flag + (default 300; `0` disables the reaper), and the task is cancelled in + `deinit` alongside the other background tasks. + +## Follow-up: the report didn't go away + +After the fix above, the same customer report came back. Investigating that +turned up two separate things: + +**The fix likely never shipped.** The commit was still local to this branch +(`git status` showed it 1 commit ahead of `origin/yannic-simulator-manager`, +never pushed), this package has no `BUILD` file or `Package.swift` — nothing +wires it into a build or deploy pipeline — and this branch's history includes +an earlier commit titled `DO NOT MERGE: Simulator manager`. Nothing here +indicates a build customers actually run has changed at all. + +**Independent of that, a real gap in the reaper itself.** Both the original +`delete()` and the new reaper's delete path called `simctl delete` without +ever calling `shutdown` first, and both discarded failures with `try?` and no +escalation. That's survivable for a healthy device, but a device that's +genuinely wedged — exactly the profile of a `launchd_sim` that's been running +for days — will fail `shutdown` and `delete` for the same underlying reason +every time. The reaper would correctly identify such a device as orphaned +after two sweeps, then retry a delete that fails identically every 5 minutes, +forever, silently. Retrying a deterministically broken operation isn't a fix. +See the next section for what addresses this. + +## Additional fix: shut down before delete, escalate when delete keeps failing + +- **`shutdownSimulator`** (new private free function in `SimulatorControl.swift`) + factors out the shutdown-with-"already shut down"-handling logic that + previously only backed the public `shutdown()` method. `unlockedDelete` now + calls it before each of its (still up to 3) delete attempts, best-effort + (`try?` — a shutdown failure for any other reason shouldn't block trying + delete anyway, since delete is what actually matters). +- **`SimulatorControl.forceKillLaunchdSim(for:)`** (new protocol method) is the + last resort for a device that keeps failing to delete. It finds the + device's `launchd_sim` process with `pgrep -f ` (the process's own + command line embeds its data path, so the UDID is a safe, specific + pattern), double-checks the process name before signaling it (`ps -p + -o comm=`, must end in `launchd_sim`), and sends `SIGKILL` directly, + bypassing `simctl` entirely. +- **`SimulatorManager` tracks consecutive reap failures per UDID** + (`orphanDeleteFailureCounts`). `reapOrphan` (new, factored out of + `reapOrphanedSimulators`) deletes normally on each sweep; after 3 + consecutive failures for the same UDID, it calls `forceKillLaunchdSim` and + makes one more delete attempt. If that still fails, it logs loudly ("needs + manual cleanup") instead of retrying the same failing call forever. The + failure count resets whenever a delete succeeds or the UDID stops being an + orphan at all. + +## Files changed + +| File | Change | +|:-----|:-------| +| `SimulatorControl.swift` | Retry + shutdown-first logic in `unlockedDelete`; new `listManagedClones()` and `forceKillLaunchdSim(for:)`; hoisted `managedCloneNamePrefix` constant; factored-out `shutdownSimulator` | +| `SimulatorManager.swift` | New `startReaper(interval:)`, `reapOrphanedSimulators()`, and `reapOrphan(_:)`; new `reaperTask`, `previousOrphanCandidates`, and `orphanDeleteFailureCounts` state | +| `Main.swift` | New `--reap-interval-seconds` flag; starts the reaper after `restoreLeases()` | +| `LEASE_LIFECYCLE.md` | Corrected Phase 8's claim that nothing hunts down orphans; added Phase 9 describing the reaper and its escalation path | + +## What this does not fix + +- The slot state machine itself (`SimulatorSlot` cases, `getSimulator`, the + "mutate before first `await`" rule) is unchanged. The reaper works + alongside it by going straight to `simulatorControl`, so the delicate + actor-isolation invariant didn't need to be touched. +- The Go port at `experiments/yannic/macsimulatormanager/go` was out of + scope for this fix — the bug report and investigation were both scoped to + the Swift implementation. + +## Verification status + +`bazel build //experiments/yannic/macsimulatormanager/swift:macsimulatormanager` +now succeeds for real (see [`README.md`](README.md#building-it)), and +`bazel-bin/experiments/yannic/macsimulatormanager/swift/macsimulatormanager +--help` runs and prints the expected flags, including +`--reap-interval-seconds`. That confirms the code compiles and links against +`ShellOut`/`ArgumentParser`/SwiftNIO correctly, but not the runtime behavior +described above — there's no CoreSimulator runtime installed on the machine +this was built on (`xcrun simctl list runtimes` is empty there), so the +lease/reaper/escalation logic itself is still verified only by code review +against the invariants in `LEASE_LIFECYCLE.md`, not by exercising it. +Getting the toolchain wired up also required one unplanned fix: this repo's +`common:clang` config enforces `-Werror=sign-compare` globally (including for +host tools), which `rules_swift`'s own bundled `tools/common/process.cc` +doesn't build clean under — exempted via a `--per_file_copt` in `.bazelrc` +rather than patching upstream, mirroring the existing `.pb.cc` exemption +right above it. + +Before relying on this in production: + +1. Confirm this branch is actually the source for whatever customers run — + given the `DO NOT MERGE` history noted above, that's not yet established. + If it is, push this branch and get it merged; if there's a separate + deploy path, find it. +2. Manually exercise the restart-orphan path: lease a config, `kill -9` the + daemon, start a new instance with a short `--reap-interval-seconds`, and + confirm the orphaned clone disappears from `xcrun simctl list devices` + within two sweep intervals without touching unrelated simulators. +3. Manually exercise the wedged-device escalation path: lease a config, then + independently hang or otherwise make its clone's `simctl shutdown`/`delete` + fail out-of-band, and confirm the reaper force-kills its `launchd_sim` + after 3 failed sweeps rather than retrying forever. diff --git a/tools/simulator_manager/Sources/Package.resolved b/tools/simulator_manager/Sources/Package.resolved new file mode 100644 index 0000000..0691b04 --- /dev/null +++ b/tools/simulator_manager/Sources/Package.resolved @@ -0,0 +1,176 @@ +{ + "pins" : [ + { + "identity" : "shellout", + "kind" : "remoteSourceControl", + "location" : "https://github.com/JohnSundell/ShellOut", + "state" : { + "revision" : "e1577acf2b6e90086d01a6d5e2b8efdaae033568", + "version" : "2.3.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", + "version" : "1.8.2" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", + "version" : "1.1.5" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-certificates", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-certificates.git", + "state" : { + "revision" : "449dbbecd0f31e82b510ada227ca152caa8b5e98", + "version" : "1.19.4" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", + "version" : "4.5.1" + } + }, + { + "identity" : "swift-http-structured-headers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-structured-headers.git", + "state" : { + "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", + "version" : "1.7.0" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types.git", + "state" : { + "revision" : "db774a277f60063a32d854f2980299caf06da041", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", + "version" : "2.101.3" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras", + "state" : { + "revision" : "88a51340f59cf181ebde888bd1b749296b3ec029", + "version" : "1.34.3" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "45bdf670248be5f16ec0340e125dca285536f0fb", + "version" : "1.45.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "d930168b86f46ca51a4bc09c5ca45c1833db8067", + "version" : "2.37.2" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", + "version" : "2.12.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", + "version" : "1.8.1" + } + } + ], + "version" : 2 +} diff --git a/tools/simulator_manager/Sources/Package.swift b/tools/simulator_manager/Sources/Package.swift new file mode 100644 index 0000000..e2830e1 --- /dev/null +++ b/tools/simulator_manager/Sources/Package.swift @@ -0,0 +1,16 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +// This manifest exists only so `rules_swift_package_manager` can resolve and vendor +// this daemon's third-party dependencies for Bazel (see the BUILD file in this +// directory). It declares no targets/products of its own -- the daemon is built as +// a `swift_binary` directly from these sources, not as a Swift package. +let package = Package( + name: "macsimulatormanager", + dependencies: [ + .package(url: "https://github.com/JohnSundell/ShellOut", from: "2.3.0"), + .package(url: "https://github.com/apple/swift-argument-parser", from: "1.8.2"), + .package(url: "https://github.com/apple/swift-nio-extras", from: "1.34.3"), + ] +) diff --git a/tools/simulator_manager/Sources/README.md b/tools/simulator_manager/Sources/README.md new file mode 100644 index 0000000..1a334af --- /dev/null +++ b/tools/simulator_manager/Sources/README.md @@ -0,0 +1,155 @@ +# Mac simulator manager + +A long-running daemon that manages the lifecycle of iOS Simulator devices and +hands them out, over HTTP, as leased resources for tests running on a Mac +worker. It exists so that concurrent test runs share a small pool of +simulators instead of each provisioning and tearing down its own, which is +slow and resource-hungry. + +The daemon speaks HTTP over a Unix domain socket rather than a TCP port, +since it is only ever talked to by processes on the same machine. + +## Building it + +```bash +bazel build //experiments/yannic/macsimulatormanager/swift:macsimulatormanager +``` + +Third-party dependencies (`ShellOut`, `swift-argument-parser`, SwiftNIO) are +resolved via [`rules_swift_package_manager`](https://github.com/cgrindel/rules_swift_package_manager) +from the `Package.swift` / `Package.resolved` pair in this directory, not +hand-vendored. If you change `Package.swift`, regenerate `Package.resolved` +with `swift package resolve` and run `bazel mod tidy` at the repo root to +pick up any new or changed external repos in `MODULE.bazel`. + +## Running it + +`Main.swift` is a `swift-argument-parser` command; `bazel run` works the same +way, with arguments after `--`. Notable options: + +- `--pid-path` / `--unix-socket-path` — where the daemon writes its PID file + and creates the listening socket. +- `--delete-idle-after` / `--delete-recently-used-idle-after` — how long an + unleased simulator is kept around before deletion, depending on whether its + config was recently leased (see [Idle deletion](#idle-deletion)). +- `--recently-used-capacity` — how many distinct simulator configs count as + "recently used" at once. +- `--startup-process` (repeatable) — extra processes to launch alongside the + daemon; see [Child processes](#child-processes). +- `--post-boot` — a script run against every freshly booted clone (see + [`SimulatorControl.swift`](SimulatorControl.swift)). +- `--lease-path` — where leases are mirrored to disk so a replacement daemon + can adopt them; see [Restarts and lease persistence](#restarts-and-lease-persistence). + +The `--version` flag is not read from source; it is passed in externally +(e.g. by a wrapper script) so the manager doesn't need to be recompiled just +to change how it reports its own version. + +## HTTP API + +Requests are parsed by `SimulatorManagerHTTPHandler.swift` and routed by +`HTTPServer.swift` on the first path component: + +| Method | Path | Query params | Description | +|--------|---------------------|--------------------------------------------|--------------| +| `POST` | `/simulator/` | `exclusive`, `deviceType`, `os`, `version` | Lease a simulator matching the given config to the process ``. Returns the simulator's UDID. | +| `DELETE` | `/simulator/` | | Release the simulator leased to ``. | +| `GET` | `/version` | | The daemon's version string. | +| `GET` | `/leases` | | The number of leases whose leasing process is still alive; for diagnostics. | +| `POST` | `/shutdown` | | Begin a graceful shutdown. | + +`exclusive=1` requests a simulator that no other lease may share; otherwise +the manager may hand out a simulator that is already leased non-exclusively +for the same config. Request parsing and response mapping live in +[`SimulatorRequestHandler.swift`](SimulatorRequestHandler.swift). + +## Architecture + +``` +HTTPServer (SwiftNIO, Unix domain socket) + └─ AccumulatedHTTPHandler buffers HTTP head/body/end into one FullHTTPRequest + └─ SimulatorManagerHTTPHandler parses method/path/query into a SimulatorManagerRequest + └─ SimulatorRequestHandler maps HTTP requests to SimulatorManager calls + └─ SimulatorManager (actor) lease/slot/reference-count bookkeeping + └─ SimulatorControl wraps `xcrun simctl` (create/clone/boot/delete) +``` + +### `SimulatorManager` + +[`SimulatorManager.swift`](SimulatorManager.swift) is the core state machine, +implemented as an actor so its bookkeeping is safe under concurrent leases. +For each `SimulatorConfig` (device type, OS, version) it keeps an array of +slots, each of which is `empty`, `pendingCreation`, `active`, `pendingDeletion`, +or `deleting`. Leasing a config walks the slots in a fixed preference order +(reuse an active non-exclusive simulator, then a pending deletion, then an +empty slot, then a pending creation) and either reuses what's there or clones +a fresh device from a per-config base simulator (created lazily and cached in +`getBaseSimulatorTasks`). + +Every active simulator has a reference count. Leasing increments it; releasing +decrements it, and it is only queued for deletion once it drops to zero, so a +non-exclusive simulator with multiple leasers survives until all of them +release it. + +### Idle deletion + +Once a simulator's reference count hits zero it isn't deleted immediately — +`pendingDeletion` schedules a delayed delete so a device can be reused by the +next lease for the same config. The wait is either `deleteIdleAfter` or the +longer `deleteRecentlyUsedIdleAfter`, chosen by whether that config appears in +`recentlyLeased`, an `LRUSet` (see [`LRUSet.swift`](LRUSet.swift)) capped at +`recentlyUsedCapacity` distinct configs. This keeps simulators for +in-demand configs around longer while letting one-off configs get cleaned up +quickly. + +### Restarts and lease persistence + +A new daemon version replaces the running one (the caller kills the old +process and starts the new one), which would otherwise lose track of leases +held by tests that are still running. `LeaseStore.swift` mirrors every lease +change to a JSON file; on startup, `restoreLeases()` reads it back and rebuilds +enough state (the lease, the slot holding the device, the reference count, and +an exit listener) for `release` to work normally. A lease is only adopted if +its leasing process is still running under the *same* start time (`kinfo_proc` +via `processStartTime`, not just the same PID — PIDs get recycled) and doesn't +conflict with another lease already restored; otherwise it's dropped, and its +device is picked up again automatically the next time something leases that +config. + +A leaser's exit is also watched for directly, via +`DispatchSource.makeProcessSource`, so a lease is released automatically if +its process dies without calling `DELETE /simulator/`. + +### `SimulatorControl` + +[`SimulatorControl.swift`](SimulatorControl.swift) wraps `xcrun simctl` calls +(`create`, `clone`, `bootstatus`, `shutdown`, `delete`, `list devices`) behind +a protocol, so `SimulatorManager` can be tested against a fake. Concurrent +calls for the same base/clone name are coalesced onto a single in-flight +`Task`, and deletion/existence checks for a given name are serialized through +`SimulatorDeleteOrExistenceMutex` so a `clone` can't observe a device that's +mid-deletion. + +### Child processes + +The daemon can launch extra long-lived processes alongside itself +(`--startup-process`). Each one's stdout/stderr is captured through a +`PTY` (see [`PTY.swift`](PTY.swift), needed because plain pipes make some +tools line-buffer differently) and logged line-by-line via `os.Logger`. +These processes are not restarted if they exit. + +## Files + +| File | Purpose | +|------|---------| +| `Main.swift` | CLI entry point; wires flags into a `SimulatorManager` and `HTTPServer`. | +| `HTTPServer.swift` | SwiftNIO server bound to a Unix domain socket; top-level request routing. | +| `AccumulatedHTTPHandler.swift` | Buffers streamed HTTP request parts into one in-memory request/response. | +| `SimulatorManagerHTTPHandler.swift` | Parses the HTTP request into method/path/query; serializes responses. | +| `SimulatorRequestHandler.swift` | Translates `/simulator` requests into `SimulatorManager` lease/release calls. | +| `SimulatorManager.swift` | Core actor: slots, reference counts, leases, idle deletion, child processes. | +| `SimulatorControl.swift` | `simctl`-backed implementation of creating, cloning, booting, and deleting simulators. | +| `LeaseStore.swift` | Persists leases to disk so a replacement daemon can adopt them. | +| `LRUSet.swift` | Fixed-capacity, least-recently-used set used to track recently leased configs. | +| `PTY.swift` | Minimal pseudo-terminal wrapper used to capture child process output. | +| `Logger.swift` | Shared `os.Logger` subsystem/category helper. | diff --git a/tools/simulator_manager/Sources/SimulatorControl.swift b/tools/simulator_manager/Sources/SimulatorControl.swift index bd8288a..b4b46c1 100644 --- a/tools/simulator_manager/Sources/SimulatorControl.swift +++ b/tools/simulator_manager/Sources/SimulatorControl.swift @@ -105,6 +105,20 @@ protocol SimulatorControl: Actor { runtimeIdentifier: String, context: @escaping @autoclosure () -> String? ) async throws -> String? + + // Every clone-named simulator that currently exists, across all runtimes. + // + // Used by the orphan reaper to reconcile the manager's bookkeeping against reality. + // Deliberately excludes base simulators, which are meant to be long-lived and are + // never reference-counted. + func listManagedClones() async throws -> [SimCtlDevice] + + // Kills the `launchd_sim` process for `simulator` directly, bypassing `simctl` + // entirely. Best-effort and never throws: this is the last resort for a device + // that keeps failing `delete()` through normal means (a wedged simulator can fail + // both `shutdown` and `delete` indefinitely), used by the orphan reaper's + // escalation path. + func forceKillLaunchdSim(for simulator: SimulatorUDID) async } actor RealSimulatorControl: SimulatorControl { @@ -303,20 +317,7 @@ actor RealSimulatorControl: SimulatorControl { } func shutdown(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { - do { - _ = try await simctl(["shutdown", simulator], context: context()) - } catch let error as ProcessError { - // Exit code 149 is related to the simulator already being shut down - guard error.exitCode == 149 else { - throw error - } - - Logger.simulatorControl.warning( - """ - ⚠️ Shutdown failed, but probably \"already shut down\": \(error, privacy: .public) - """ - ) - } + try await shutdownSimulator(simulator, context: context()) } func cleanTempFiles(in simulator: SimulatorUDID) { @@ -363,6 +364,72 @@ actor RealSimulatorControl: SimulatorControl { } } + func listManagedClones() async throws -> [SimCtlDevice] { + let output = try await simctl(["list", "devices", "-j"], context: "listManagedClones") + + guard let jsonData = output.data(using: .utf8) else { + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to convert output to data"] + ) + } + + let devicesByRuntime: [String: [SimCtlDevice]] + do { + devicesByRuntime = try JSONDecoder().decode(SimCtlDevices.self, from: jsonData).devices + } catch { + let json = String(data: jsonData, encoding: .utf8) ?? "" + Logger.simulatorControl.error( + """ + ❌ Failed to decode 'simctl list devices -j': \(error, privacy: .public). + Output: \(json, privacy: .public) + """ + ) + throw NSError( + domain: "SimulatorControl", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to decode output: \(error) - \(json)"] + ) + } + + return devicesByRuntime.values.flatMap { $0 }.filter { $0.name.hasPrefix(managedCloneNamePrefix) } + } + + func forceKillLaunchdSim(for simulator: SimulatorUDID) async { + // `launchd_sim`'s command line embeds the device's own data path (see the ps + // output that motivated this: ".../Devices//data/var/run/..."), so the + // UDID is a safe, specific `pgrep -f` pattern -- it can only match that one + // device's process tree. + let pids: [Int32] + do { + let output = try await subprocess("/usr/bin/pgrep", ["-f", simulator]) + pids = output.split(separator: "\n").compactMap { Int32($0.trimmingCharacters(in: .whitespaces)) } + } catch { + // `pgrep` exits non-zero (surfaced here as a thrown `ProcessError`) when nothing + // matches -- there's nothing left to kill. + return + } + + for pid in pids { + // Double check this is actually `launchd_sim` before signaling it. `pgrep -f` + // matches the UDID anywhere in the command line; being wrong here would kill an + // unrelated process that merely mentioned this device (e.g. in a log path). + guard let comm = try? await subprocess("/bin/ps", ["-p", "\(pid)", "-o", "comm="]), + comm.trimmingCharacters(in: .whitespacesAndNewlines).hasSuffix("launchd_sim") else { + continue + } + + Logger.simulatorControl.warning( + """ + 🔨 Force-killing launchd_sim (pid \(pid, privacy: .public)) for stuck simulator \ + \(simulator, privacy: .public) + """ + ) + kill(pid, SIGKILL) + } + } + func ensureBooted(_ simulator: SimulatorUDID, context: @escaping @autoclosure () -> String?) async throws { for retriesLeft in (0...1).reversed() { do { @@ -544,20 +611,66 @@ actor SimulatorDeleteOrExistenceMutex { ) async throws { Logger.simulatorControl.info("🗑️ Deleting simulator \(simulator, privacy: .public)") - do { - _ = try await simctl(["delete", simulator], context: context()) - } catch { - Logger.simulatorControl.error( - """ - ❌ Failed to delete simulator \(simulator, privacy: .public): \ - \(error, privacy: .public) - """ - ) + // `simctl delete` can fail transiently (CoreSimulator daemon busy, a lingering + // child process, disk I/O) or because the device is still booted -- `delete` + // does not reliably shut a booted device down on its own. Callers treat a thrown + // error here as "give up until the next event," so shut down and retry a few + // times before surfacing failure -- it's much cheaper than leaving the simulator + // running until the orphan reaper's next sweep catches it. + let maxAttempts = 3 + for attempt in 1...maxAttempts { + // Best-effort: proceed to the delete attempt regardless of whether this + // succeeds. A shutdown failure for a reason other than "already shut down" + // (already handled inside `shutdownSimulator`) shouldn't block trying delete + // anyway, since delete is the operation that actually matters here. + try? await shutdownSimulator(simulator, context: context()) + + do { + _ = try await simctl(["delete", simulator], context: context()) + Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") + return + } catch { + guard attempt < maxAttempts else { + Logger.simulatorControl.error( + """ + ❌ Failed to delete simulator \(simulator, privacy: .public) after \ + \(maxAttempts, privacy: .public) attempts: \(error, privacy: .public) + """ + ) + throw error + } + + Logger.simulatorControl.warning( + """ + ⚠️ Delete attempt \(attempt, privacy: .public)/\(maxAttempts, privacy: .public) for \ + simulator \(simulator, privacy: .public) failed, retrying: \(error, privacy: .public) + """ + ) + try? await Task.sleep(for: .seconds(2)) + } + } + } +} +/// Shared by `RealSimulatorControl.shutdown` and `unlockedDelete`, since delete needs +/// to shut the device down first and shouldn't reimplement this. +private func shutdownSimulator( + _ simulator: SimulatorUDID, + context: @escaping @autoclosure () -> String? +) async throws { + do { + _ = try await simctl(["shutdown", simulator], context: context()) + } catch let error as ProcessError { + // Exit code 149 is related to the simulator already being shut down + guard error.exitCode == 149 else { throw error } - Logger.simulatorControl.info("🗑️ Deleted simulator \(simulator, privacy: .public)") + Logger.simulatorControl.warning( + """ + ⚠️ Shutdown failed, but probably \"already shut down\": \(error, privacy: .public) + """ + ) } } @@ -612,13 +725,18 @@ private func syncSubprocess( } } +/// Prefix shared by every simulator this manager creates for cloning (see +/// `SimulatorConfig.cloneDeviceName`). Used to scope the orphan reaper to devices +/// it manages, so it never touches a developer's own simulators or base templates. +let managedCloneNamePrefix = "EXAMPLE_BAZEL_CLONE_" + extension SimulatorConfig { func baseDeviceName() -> String { return "EXAMPLE_BAZEL_BASE_\(deviceType)_\(version)" } func cloneDeviceName(index: Int) -> String { - return "EXAMPLE_BAZEL_CLONE_\(deviceType)_\(version)_\(index)" + return "\(managedCloneNamePrefix)\(deviceType)_\(version)_\(index)" } func runtimeIdentifier() -> String { diff --git a/tools/simulator_manager/Sources/SimulatorManager.swift b/tools/simulator_manager/Sources/SimulatorManager.swift index cff32d1..a488b5e 100644 --- a/tools/simulator_manager/Sources/SimulatorManager.swift +++ b/tools/simulator_manager/Sources/SimulatorManager.swift @@ -51,19 +51,21 @@ private enum SimulatorSlot { extension SimulatorSlot { var sortOrder: Int { switch self { - // Try to use active or pending creation simulators first (should only be - // one of either for non-exclusive) + // Try to use an active simulator first (should only be one for non-exclusive) case .active: return 0 - case .pendingCreation: - return 1 - // Use a pending deletion before any empty slots + // A pending deletion is already created and booted, so reuse it before waiting + // on a pending creation or starting a fresh one. case .pendingDeletion: - return 2 + return 1 - // Finally use empty slots + // Use an empty slot before waiting on a pending creation, so a lease doesn't + // block on someone else's in-flight clone when a fresh slot is free to start. case .empty: + return 2 + + case .pendingCreation: return 3 // Deleting simulator can't be used, so put it at the end @@ -104,6 +106,22 @@ actor SimulatorManager { private var childProcessTasks: [Task] = [] private var childProcesses: [String: (Process, DispatchSourceRead, DispatchSourceRead)] = [:] + private var reaperTask: Task? + /// Clone UDIDs that looked orphaned on the *previous* sweep. A device must appear + /// unknown on two consecutive sweeps before the reaper deletes it, so a clone that's + /// mid-creation (it exists on disk before `createCloneTask` resumes and records it in + /// `referenceCount`) never gets caught by a single unlucky sweep. + private var previousOrphanCandidates: Set = [] + /// How many sweeps in a row the reaper has failed to delete a confirmed orphan. + /// Reset once a delete succeeds, or once the UDID stops being an orphan at all + /// (claimed by a new lease, or already cleaned up some other way). + private var orphanDeleteFailureCounts: [SimulatorUDID: Int] = [:] + /// After this many consecutive failed deletes, a device is treated as wedged + /// rather than merely unlucky: normal `simctl shutdown`/`delete` retries have + /// already been exhausted inside `SimulatorControl.delete()` every sweep, so more + /// of the same is unlikely to help. Escalate to killing its `launchd_sim` directly. + private let forceKillAfterFailedReapAttempts = 3 + init( simulatorControl: SimulatorControl, deleteRecentlyUsedIdleAfter: UInt16, @@ -129,6 +147,8 @@ actor SimulatorManager { } deinit { + reaperTask?.cancel() + for task in childProcessTasks { task.cancel() } @@ -146,6 +166,139 @@ actor SimulatorManager { } } + /// Starts the periodic sweep that deletes clone simulators the manager has lost + /// track of. + /// + /// Every other cleanup path is event-driven: an explicit release, a PID-exit + /// watcher, an idle timer, or "rediscovered by name on the next lease of the same + /// config." Each of those can miss a device -- a daemon restart drops a lease + /// whose device is never leased again (see `restoreLeases`), or a `simctl delete` + /// call fails and is swallowed (see `delete`) -- and nothing else ever looks for + /// it again. This sweep is the backstop: it reconciles against what CoreSimulator + /// actually has running, independent of how a device became untracked. + /// + /// `interval <= .zero` disables it, matching how `deleteIdleAfter` of 0 means + /// "immediately" elsewhere in this file rather than "never." + func startReaper(interval: Duration) { + guard interval > .zero else { return } + + reaperTask = Task { + while !Task.isCancelled { + try? await Task.sleep(for: interval) + guard !Task.isCancelled else { break } + await reapOrphanedSimulators() + } + } + } + + /// One sweep of the orphan reaper: list every clone simulator that exists, and + /// delete the ones with no entry in `referenceCount`. + /// + /// `referenceCount` is the right ground truth to check against, not `leases` or + /// `simulatorSlots`: a device gets an entry in it the instant it's claimed (before + /// any `await`, per the invariant on `getSimulator`), and the entry is removed + /// only in `delete()` -- including while the device sits in the idle-timer grace + /// period, where the count is `0` but the key stays. So "no entry" reliably means + /// "the manager has no idea this exists," not merely "nothing is leasing it right + /// now." + /// + /// Bypasses `delete()` deliberately: that function updates a slot in + /// `simulatorSlots`, but an orphan by definition has no slot pointing at it, so + /// there is nothing there to update. This calls `simulatorControl` directly, the + /// same lower-level operation `delete()` itself wraps. + private func reapOrphanedSimulators() async { + let known = Set(referenceCount.keys) + + let managedClones: [SimCtlDevice] + do { + managedClones = try await simulatorControl.listManagedClones() + } catch { + Logger.simulatorManager.error( + "❌ Orphan reaper failed to list simulators, skipping this sweep: \(error, privacy: .public)" + ) + return + } + + let currentOrphanCandidates = Set(managedClones.map(\.udid)).subtracting(known) + let confirmedOrphans = currentOrphanCandidates.intersection(previousOrphanCandidates) + previousOrphanCandidates = currentOrphanCandidates + + // Forget the failure count for anything that isn't a confirmed orphan any more + // (claimed by a new lease, or already cleaned up), so a UDID that's reused later + // starts with a clean slate rather than inheriting an old device's history. + orphanDeleteFailureCounts = orphanDeleteFailureCounts.filter { confirmedOrphans.contains($0.key) } + + guard !confirmedOrphans.isEmpty else { return } + + for device in managedClones where confirmedOrphans.contains(device.udid) { + await reapOrphan(device) + } + } + + /// Deletes one confirmed-orphaned device, escalating to a direct kill of its + /// `launchd_sim` if it has already failed to delete + /// `forceKillAfterFailedReapAttempts` sweeps in a row. + /// + /// `SimulatorControl.delete()` already shuts the device down and retries a few + /// times internally before throwing, so a failure reaching here means those + /// retries were exhausted -- consistent with a genuinely wedged device (the same + /// profile as the multi-day-old `launchd_sim` processes that motivated this), not + /// a one-off transient error. Retrying the same call every 5-minute sweep forever + /// would just fail the same way forever, silently; escalating is what makes this a + /// backstop instead of another silent no-op. + private func reapOrphan(_ device: SimCtlDevice) async { + Logger.simulatorManager.warning( + """ + 🧹 Reaping orphaned simulator \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)); the manager has no lease or reference to it + """ + ) + + do { + try await simulatorControl.delete(device.udid, name: device.name, context: "orphan reaper") + orphanDeleteFailureCounts.removeValue(forKey: device.udid) + return + } catch { + let failures = (orphanDeleteFailureCounts[device.udid] ?? 0) + 1 + orphanDeleteFailureCounts[device.udid] = failures + + Logger.simulatorManager.error( + """ + ❌ Orphan reaper failed to delete \(device.udid, privacy: .public) \ + (\(device.name, privacy: .public)), attempt \(failures, privacy: .public): \ + \(error, privacy: .public) + """ + ) + + guard failures >= forceKillAfterFailedReapAttempts else { return } + + Logger.simulatorManager.error( + """ + 🔨 \(device.udid, privacy: .public) (\(device.name, privacy: .public)) has failed to \ + delete \(failures, privacy: .public) sweeps in a row; force-killing its launchd_sim + """ + ) + + await simulatorControl.forceKillLaunchdSim(for: device.udid) + + do { + try await simulatorControl.delete( + device.udid, + name: device.name, + context: "orphan reaper, post force-kill" + ) + orphanDeleteFailureCounts.removeValue(forKey: device.udid) + } catch { + Logger.simulatorManager.error( + """ + ❌ \(device.udid, privacy: .public) (\(device.name, privacy: .public)) still failed to \ + delete after force-killing launchd_sim: \(error, privacy: .public). Needs manual cleanup. + """ + ) + } + } + } + /// Adopts the leases a previous daemon left behind. /// /// Leases used to live only in this actor's memory, so replacing the daemon -- @@ -646,12 +799,18 @@ actor SimulatorManager { let processSource = DispatchSource.makeProcessSource(identifier: leaser, eventMask: .exit, queue: .main) + // Avoid double handling of exit in case the process exits between + // `processSource.resume()` and the check with `kill` below. The event handler runs + // on `.main`, while the liveness check below runs on whatever thread calls this + // method, so the flag guarding against a double call must itself be synchronized. + let handledExitLock = NSLock() var handledExit = false let onExitHandler: () -> Void = { [weak self] in - // Avoid double handling of exit in case the process exits between - // `processSource.resume()` and the check with `kill` - guard !handledExit else { return } + handledExitLock.lock() + let alreadyHandled = handledExit handledExit = true + handledExitLock.unlock() + guard !alreadyHandled else { return } Task { guard let self else { return } @@ -665,8 +824,8 @@ actor SimulatorManager { processSource.setEventHandler { onExitHandler() } processSource.resume() - // Check to see if the process is already dead and cancel the source if it is, which will - // trigger `setCancelHandler`, which releases the simulator + // Check to see if the process is already dead. There is no `setCancelHandler`, so + // handle the exit directly here rather than relying on cancellation to do it. guard processIsRunning(leaser) else { processSource.cancel() onExitHandler() diff --git a/tools/simulator_manager/go/BUILD.bazel b/tools/simulator_manager/go/BUILD.bazel new file mode 100644 index 0000000..66174e4 --- /dev/null +++ b/tools/simulator_manager/go/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "go_lib", + srcs = [ + "http_server.go", + "lease_store.go", + "logger.go", + "lru_set.go", + "main.go", + "pty.go", + "simulator_control.go", + "simulator_manager.go", + "simulator_request_handler.go", + ], + importpath = "github.com/EngFlow/example/tools/simulator_manager/go", + visibility = ["//visibility:private"], + deps = ["@org_golang_x_sys//unix"], +) + +go_binary( + name = "go", + embed = [":go_lib"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_test", + srcs = [ + "fake_simulator_control_test.go", + "http_server_test.go", + "lease_store_test.go", + "lru_set_test.go", + "main_test.go", + "simulator_control_test.go", + "simulator_manager_test.go", + "simulator_request_handler_test.go", + ], + embed = [":go_lib"], + deps = [ + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/tools/simulator_manager/go/README.md b/tools/simulator_manager/go/README.md new file mode 100644 index 0000000..e8d926b --- /dev/null +++ b/tools/simulator_manager/go/README.md @@ -0,0 +1,162 @@ +# Mac Simulator Manager (Go) + +This is a Go translation of the Swift Mac Simulator Manager located in `../swift/`. + +## Architecture + +The simulator manager is an HTTP server that manages iOS/macOS simulator instances for testing. It provides lease-based access to simulators with the following features: + +- **Lease Management**: PIDs can lease simulators (exclusively or shared) +- **Automatic Cleanup**: Simulators are automatically deleted when idle +- **Persistence**: Leases are persisted to disk so daemon restarts don't lose state +- **Base + Clone Pattern**: Creates base simulators and clones them for efficiency +- **Child Process Management**: Can launch and monitor startup processes + +## Key Components + +### Main Application (`main.go`) +Entry point that parses command-line arguments and starts the server. + +### HTTP Server (`http_server.go`) +Serves requests on a Unix domain socket with endpoints: +- `POST /simulator/` - Lease a simulator +- `DELETE /simulator/` - Release a simulator +- `GET /version` - Get manager version +- `GET /leases` - Get count of live leases +- `POST /shutdown` - Shutdown the server + +### Simulator Manager (`simulator_manager.go`) +Core orchestration logic: +- Manages simulator slots per configuration +- Handles lease/release operations +- Tracks reference counts +- Implements automatic deletion with LRU eviction + +### Simulator Control (`simulator_control.go`) +Low-level interface to `simctl`: +- Creates base simulators +- Clones simulators +- Boots and shuts down simulators +- Deletes simulators + +### Lease Store (`lease_store.go`) +Persists leases to disk in JSON format with atomic writes. Includes process start time to detect PID reuse. + +### Supporting Components +- `lru_set.go` - LRU cache for tracking recently used configurations +- `pty.go` - PTY creation for child process I/O +- `logger.go` - Structured logging setup + +## Translation Notes + +### Differences from Swift + +These are concrete, verified behavioral or structural differences between the +two implementations -- not just "different language, same thing" restatements. + +1. **Concurrency model**: + - Swift's `SimulatorManager` and `SimulatorControl` are actors, so the + compiler serializes access and only allows suspension at explicit + `await` points. Go has no equivalent, so the same invariants are + enforced by hand with `sync.Mutex`, manually unlocking around any + blocking call (channel receive, `simctl` invocation) and re-locking + afterward -- easy to get subtly wrong in a future change, unlike the + actor version. + - Coalescing concurrent callers onto one in-flight operation (`getBase`, + `createBase`, `clone`) uses Swift's `Task`, which is directly + awaitable and memoizes its result. Go instead uses a buffered channel + (`chan taskResult`) that every caller receives from. + - `SimulatorDeleteOrExistenceMutex`, which serializes delete/existence + checks for a given device name, is a hand-rolled actor with an explicit + waiter queue in Swift; Go uses a plain `sync.Mutex`. + +2. **Leaser-exit detection is slower in Go**: + - Swift watches for a leaser's exit with `DispatchSource.makeProcessSource` + (kqueue `EVFILT_PROC`), so `release` runs as soon as the kernel reports + the exit. + - Go's `registerReleaseOnExit` instead polls once a second with + `time.Ticker` + `kill(pid, 0)`, so reclaiming a dead leaser's simulator + can lag up to ~1s behind Swift. (The separate idle-deletion timer in + `pendingDeletion` already polls once a second in *both* implementations, + so that part carried over unchanged.) + +3. **`POST /shutdown` is abrupt in Go, graceful in Swift**: + - Swift's only shutdown path is `POST /shutdown`, which triggers + SwiftNIO's `ServerQuiescingHelper`: stop accepting connections, let + in-flight ones finish, then fall through to the cleanup code that + removes the socket and PID files. + - Go additionally handles SIGINT/SIGTERM by calling `http.Server.Shutdown` + (a comparable graceful drain), but its `POST /shutdown` handler just + calls `os.Exit(0)` from a goroutine after writing the response. That + skips draining any other in-flight requests and skips the deferred + socket/PID-file cleanup in `HTTPServer.Run`, which only runs after + `server.Serve()` returns -- which `os.Exit` prevents. An HTTP-triggered + shutdown in the Go version can leave stale socket/PID files behind. + +4. **Post-boot script failures lose detail in Go**: + - Swift runs the post-boot script through the same `subprocess`/`shellOut` + helper used for every `simctl` call, so a failure comes back as a + `ProcessError` with captured stdout/stderr. + - Go's `Clone` invokes the post-boot script directly with `exec.Command` + instead of the shared `subprocess`/`simctl` helper: stdout/stderr are + discarded rather than captured, and a failure surfaces as a wrapped + `*exec.ExitError` rather than a `*ProcessError`. + +5. **`--post-boot` required vs. optional**: + - Swift declares `postBoot` as a non-optional `String` with no default, so + `swift-argument-parser` requires the flag and fails startup without it. + - Go's flag defaults to `""` and is treated as "skip the post-boot step", + so it's optional. + +6. **Debug logging is unreachable in Go**: + - Go's `slog` handler is hardcoded to `slog.LevelInfo`, so every + `Debug`-level call (reference-count changes, existing-simulator lookups) + is compiled but never emitted, and there's no flag to raise the level. + - Swift's `os.Logger` calls at `.debug` are still captured live by the + unified logging system (visible via `log stream`/Console), just not + persisted long-term. + +7. **HTTP framework**: + - Swift hand-rolls the server on SwiftNIO: a channel pipeline of + `AccumulatedHTTPHandler` (buffers head/body/end into one in-memory + request) followed by `SimulatorManagerHTTPHandler` (parses + method/path/query), with one task per connection. + - Go uses the standard library's `net/http` + `http.ServeMux`, which + already delivers a fully parsed, buffered `*http.Request` per call, so + there's no need for an equivalent two-stage accumulate/parse pipeline. + +8. **Child-process output capture**: + - Swift watches each PTY with `DispatchSource.makeReadSource` + (event-driven, woken by the runloop on readability). + - Go uses pipes instead of PTYs (to avoid CGO) and parks a dedicated + goroutine in a blocking `Read` loop per pipe. This works identically + for line-buffered output capture but wouldn't preserve interactive + terminal behavior if child processes expected a real TTY. + +9. **JSON serialization**: both use built-in JSON support with struct tags; + no behavioral difference here. + +### Build + +```bash +# With Bazel +bazel build //experiments/yannic/macsimulatormanager/go:macsimulatormanager + +# With Go +cd experiments/yannic/macsimulatormanager/go +go build -o macsimulatormanager +``` + +### Usage + +```bash +./macsimulatormanager \ + --version="1.0.0" \ + --pid-path="/tmp/simulator-manager.pid" \ + --unix-socket-path="/tmp/simulator-manager.sock" \ + --delete-recently-used-idle-after=300 \ + --delete-idle-after=60 \ + --recently-used-capacity=1 \ + --post-boot="/path/to/post-boot-script.sh" \ + --lease-path="/tmp/leases.json" +``` diff --git a/tools/simulator_manager/go/fake_simulator_control_test.go b/tools/simulator_manager/go/fake_simulator_control_test.go new file mode 100644 index 0000000..ae1c148 --- /dev/null +++ b/tools/simulator_manager/go/fake_simulator_control_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "fmt" + "sync" +) + +// fakeSimulatorControl is an in-memory SimulatorControl used to test +// SimulatorManager's lease/slot/reference-count logic without shelling out to +// xcrun simctl. +type fakeSimulatorControl struct { + mu sync.Mutex + + udidCounter int + + createBaseCalls int + + cloneCalls []string // clone device names, in call order + // cloneGate, if non-nil, blocks the first Clone call until the test sends + // (or closes) it -- used to deterministically exercise in-flight clone + // sharing between concurrent Lease calls. + cloneGate chan struct{} + cloneGateOnce sync.Once + cloneGateFired bool + + ensureBootedCalls []SimulatorUDID + // ensureBootedErrs are one-shot: each error is returned exactly once for + // the matching UDID, then cleared, so a retry can succeed. + ensureBootedErrs map[SimulatorUDID]error + + deleteCalls []SimulatorUDID + deleteErrs map[SimulatorUDID]error + + cleanTempFilesCalls []SimulatorUDID + + // runningOverride, keyed by clone name, is returned by RunningSimulators + // for that name. The fake doesn't model real booted state on its own + // (there's nothing to derive it from), so this defaults to empty -- + // tests that want to exercise the "duplicate running simulator" assertion + // set an override explicitly. + runningOverride map[string][]SimCtlDevice +} + +func newFakeSimulatorControl() *fakeSimulatorControl { + return &fakeSimulatorControl{ + ensureBootedErrs: make(map[SimulatorUDID]error), + deleteErrs: make(map[SimulatorUDID]error), + runningOverride: make(map[string][]SimCtlDevice), + } +} + +func (f *fakeSimulatorControl) nextUDID() SimulatorUDID { + f.udidCounter++ + return fmt.Sprintf("udid-%d", f.udidCounter) +} + +func (f *fakeSimulatorControl) CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { + f.mu.Lock() + f.createBaseCalls++ + udid := f.nextUDID() + f.mu.Unlock() + return udid, nil +} + +func (f *fakeSimulatorControl) Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { + f.mu.Lock() + f.cloneCalls = append(f.cloneCalls, name) + gate := f.cloneGate + alreadyFired := f.cloneGateFired + f.mu.Unlock() + + if gate != nil && !alreadyFired { + f.cloneGateOnce.Do(func() { + <-gate + f.mu.Lock() + f.cloneGateFired = true + f.mu.Unlock() + }) + } + + f.mu.Lock() + udid := f.nextUDID() + f.mu.Unlock() + return udid, nil +} + +func (f *fakeSimulatorControl) EnsureBooted(simulator SimulatorUDID, context string) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.ensureBootedCalls = append(f.ensureBootedCalls, simulator) + if err, ok := f.ensureBootedErrs[simulator]; ok { + delete(f.ensureBootedErrs, simulator) + return err + } + return nil +} + +func (f *fakeSimulatorControl) CleanTempFiles(simulator SimulatorUDID) { + f.mu.Lock() + defer f.mu.Unlock() + f.cleanTempFilesCalls = append(f.cleanTempFilesCalls, simulator) +} + +func (f *fakeSimulatorControl) Delete(simulator SimulatorUDID, name string, context string) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.deleteCalls = append(f.deleteCalls, simulator) + return f.deleteErrs[simulator] +} + +func (f *fakeSimulatorControl) GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { + // SimulatorManager never calls this directly -- it's only used internally + // by RealSimulatorControl's own createBase/clone rediscovery logic, which + // this fake doesn't need to replicate. + return "", nil +} + +func (f *fakeSimulatorControl) RunningSimulators(name string) ([]SimCtlDevice, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.runningOverride[name], nil +} + +// erroringRunningSimulatorsControl wraps a fakeSimulatorControl to make +// RunningSimulators always fail, for testing that the lease-time assertion +// treats a failure to perform the check as non-fatal to the lease itself. +type erroringRunningSimulatorsControl struct { + *fakeSimulatorControl +} + +func (c *erroringRunningSimulatorsControl) RunningSimulators(name string) ([]SimCtlDevice, error) { + return nil, fmt.Errorf("simctl unavailable") +} + +// fakeLeaseStore is an in-memory LeaseStore for testing RestoreLeases and +// persistence side effects without touching disk. +type fakeLeaseStore struct { + mu sync.Mutex + loaded []PersistedLease + saved [][]PersistedLease +} + +func (f *fakeLeaseStore) Load() []PersistedLease { + f.mu.Lock() + defer f.mu.Unlock() + return f.loaded +} + +func (f *fakeLeaseStore) Save(leases []PersistedLease) { + f.mu.Lock() + defer f.mu.Unlock() + f.saved = append(f.saved, leases) +} + +func (f *fakeLeaseStore) lastSaved() []PersistedLease { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.saved) == 0 { + return nil + } + return f.saved[len(f.saved)-1] +} diff --git a/tools/simulator_manager/go/http_server.go b/tools/simulator_manager/go/http_server.go new file mode 100644 index 0000000..a641935 --- /dev/null +++ b/tools/simulator_manager/go/http_server.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" +) + +type HTTPServer struct { + simulatorRequestHandler *SimulatorRequestHandler + version string +} + +func NewHTTPServer(simulatorRequestHandler *SimulatorRequestHandler, version string) *HTTPServer { + return &HTTPServer{ + simulatorRequestHandler: simulatorRequestHandler, + version: version, + } +} + +func (s *HTTPServer) Run(pidPath string, unixSocketPath string) error { + if err := os.Remove(unixSocketPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove existing socket: %w", err) + } + + if err := os.Remove(pidPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove existing PID file: %w", err) + } + + pid := os.Getpid() + if err := os.WriteFile(pidPath, []byte(strconv.Itoa(pid)), 0644); err != nil { + return fmt.Errorf("failed to write PID file: %w", err) + } + + listener, err := net.Listen("unix", unixSocketPath) + if err != nil { + return fmt.Errorf("failed to listen on unix socket: %w", err) + } + + httpServerLogger.Info("Server running on UDS", "path", unixSocketPath) + + mux := http.NewServeMux() + mux.HandleFunc("/simulator", s.handleSimulatorRequest) + mux.HandleFunc("/simulator/", s.handleSimulatorRequest) + mux.HandleFunc("/version", s.handleVersionRequest) + mux.HandleFunc("/leases", s.handleLeasesRequest) + mux.HandleFunc("/shutdown", s.handleShutdownRequest) + + server := &http.Server{Handler: mux} + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + go func() { + <-sigChan + httpServerLogger.Info("Shutting down server") + _ = server.Shutdown(context.Background()) + }() + + err = server.Serve(listener) + if err != nil && err != http.ErrServerClosed { + return err + } + + httpServerLogger.Info("Server shut down") + + _ = os.Remove(unixSocketPath) + _ = os.Remove(pidPath) + + return nil +} + +func (s *HTTPServer) handleSimulatorRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + + response := s.simulatorRequestHandler.HandleRequest(r.Method, r.URL.Path, r.URL.Query()) + + accumulatedHTTPLogger.Info("Sending response", "status", response.Status) + + w.WriteHeader(response.Status) + fmt.Fprintf(w, "%s\n", response.Message) +} + +func (s *HTTPServer) handleVersionRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + accumulatedHTTPLogger.Info("Sending response", "status", http.StatusOK) + + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "%s\n", s.version) +} + +func (s *HTTPServer) handleLeasesRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + + count := s.simulatorRequestHandler.LiveLeaseCount() + + accumulatedHTTPLogger.Info("Sending response", "status", http.StatusOK) + + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "%d\n", count) +} + +func (s *HTTPServer) handleShutdownRequest(w http.ResponseWriter, r *http.Request) { + accumulatedHTTPLogger.Info("Received request", "method", r.Method, "path", r.URL.Path) + + httpServerLogger.Info("Shutdown request received") + + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "Server shutting down\n") + + go func() { + os.Exit(0) + }() +} diff --git a/tools/simulator_manager/go/http_server_test.go b/tools/simulator_manager/go/http_server_test.go new file mode 100644 index 0000000..a0ea240 --- /dev/null +++ b/tools/simulator_manager/go/http_server_test.go @@ -0,0 +1,75 @@ +package main + +// These tests exercise HTTPServer's HTTP handler methods directly via +// net/http/httptest, bypassing Run(). Run() itself (unix socket setup, PID +// file management, OS signal handling) and handleShutdownRequest (which +// calls os.Exit) are integration-level, OS-facing plumbing that would +// require killing or replacing the test process itself to exercise safely, +// so they're intentionally left uncovered here. + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHandleVersionRequest(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1.2.3") + + req := httptest.NewRequest(http.MethodGet, "/version", nil) + rec := httptest.NewRecorder() + + server.handleVersionRequest(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "v1.2.3\n", rec.Body.String()) +} + +func TestHandleLeasesRequest(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1") + + req := httptest.NewRequest(http.MethodGet, "/leases", nil) + rec := httptest.NewRecorder() + + server.handleLeasesRequest(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "0\n", rec.Body.String()) +} + +func TestHandleSimulatorRequest_RoutesToHandler(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1") + + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + req := httptest.NewRequest(http.MethodPost, "/simulator/1?"+params.Encode(), nil) + rec := httptest.NewRecorder() + + server.handleSimulatorRequest(rec, req) + + require.Equal(t, http.StatusCreated, rec.Code) + assert.NotEmpty(t, rec.Body.String()) +} + +func TestHandleSimulatorRequest_BadRequestSurfacesMessage(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + server := NewHTTPServer(NewSimulatorRequestHandler(sm), "v1") + + req := httptest.NewRequest(http.MethodDelete, "/simulator/1", nil) + rec := httptest.NewRecorder() + + server.handleSimulatorRequest(rec, req) + + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Contains(t, rec.Body.String(), "doesn't have a simulator leased") +} diff --git a/tools/simulator_manager/go/lease_store.go b/tools/simulator_manager/go/lease_store.go new file mode 100644 index 0000000..e0a3ef1 --- /dev/null +++ b/tools/simulator_manager/go/lease_store.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "os" + + "golang.org/x/sys/unix" +) + +type SimulatorUDID = string + +// PersistedLease is a lease, in the form it takes on disk. +// +// Holds everything needed to rebuild the daemon's in-memory bookkeeping for one +// lease: which device, under which configuration, in which slot, and whether it +// was owned exclusively. Nothing here has to be re-derived by asking simctl. +type PersistedLease struct { + PID int32 `json:"pid"` + LeaserStartTime *uint64 `json:"leaserStartTime,omitempty"` + UDID SimulatorUDID `json:"udid"` + Config SimulatorConfig `json:"config"` + Exclusive bool `json:"exclusive"` + SlotIndex int `json:"slotIndex"` +} + +// LeaseStore is where the daemon keeps its leases so a successor can pick them up. +type LeaseStore interface { + // Save replaces the stored set with leases. Failures are logged, not thrown: + // losing persistence degrades a restart, but failing the lease that triggered + // the write would break a test that is otherwise fine. + Save(leases []PersistedLease) + + // Load returns the stored set, or empty if there is nothing readable to restore. + Load() []PersistedLease +} + +// FileLeaseStore is a LeaseStore backed by a JSON file. +// +// Written atomically, so a reader never sees a half-written set and a crash +// mid-write leaves the previous contents intact. Combined with writing on every +// change, this means the file is always current -- there is no flush-on-shutdown +// step for a kill -9 to skip. +type FileLeaseStore struct { + path string +} + +func NewFileLeaseStore(path string) *FileLeaseStore { + return &FileLeaseStore{path: path} +} + +func (f *FileLeaseStore) Save(leases []PersistedLease) { + data, err := json.MarshalIndent(leases, "", " ") + if err != nil { + leaseStoreLogger.Error("Failed to marshal leases", "error", err, "count", len(leases), "path", f.path) + return + } + + // Write atomically using a temp file and rename + tmpPath := f.path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0644); err != nil { + leaseStoreLogger.Error("Failed to write leases", "error", err, "count", len(leases), "path", f.path) + return + } + + if err := os.Rename(tmpPath, f.path); err != nil { + leaseStoreLogger.Error("Failed to rename leases file", "error", err, "count", len(leases), "path", f.path) + return + } +} + +func (f *FileLeaseStore) Load() []PersistedLease { + if _, err := os.Stat(f.path); os.IsNotExist(err) { + return nil + } + + data, err := os.ReadFile(f.path) + if err != nil { + leaseStoreLogger.Error("Failed to read leases; continuing with none", "error", err, "path", f.path) + return nil + } + + var leases []PersistedLease + if err := json.Unmarshal(data, &leases); err != nil { + leaseStoreLogger.Error("Failed to decode leases; continuing with none", "error", err, "path", f.path) + return nil + } + + return leases +} + +// processStartTime returns when pid started, in microseconds since the epoch, or nil if it cannot be +// determined (typically because the process is gone). +// +// A PID alone does not identify a process across a daemon restart: PIDs are +// recycled, so a lease for a dead PID could otherwise be restored onto whatever +// unrelated process now holds that number, tying up a device until it exits. +// Start time distinguishes them -- the kernel assigns it at fork, so a recycled +// PID has a different one. +func processStartTime(pid int32) *uint64 { + info, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid)) + if err != nil { + return nil + } + + startTime := uint64(info.Proc.P_starttime.Sec)*1_000_000 + uint64(info.Proc.P_starttime.Usec) + return &startTime +} diff --git a/tools/simulator_manager/go/lease_store_test.go b/tools/simulator_manager/go/lease_store_test.go new file mode 100644 index 0000000..65b80f3 --- /dev/null +++ b/tools/simulator_manager/go/lease_store_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFileLeaseStore_LoadNonexistentFile(t *testing.T) { + store := NewFileLeaseStore(filepath.Join(t.TempDir(), "does-not-exist.json")) + assert.Nil(t, store.Load()) +} + +func TestFileLeaseStore_SaveAndLoadRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "leases.json") + store := NewFileLeaseStore(path) + + startTime := uint64(123) + leases := []PersistedLease{ + { + PID: 42, + LeaserStartTime: &startTime, + UDID: "udid-1", + Config: SimulatorConfig{DeviceType: "iPhone", OS: "iOS", Version: "17.0"}, + Exclusive: true, + SlotIndex: 0, + }, + { + PID: 43, + UDID: "udid-2", + Config: SimulatorConfig{DeviceType: "iPad", OS: "iOS", Version: "17.0"}, + Exclusive: false, + SlotIndex: 1, + }, + } + + store.Save(leases) + + loaded := store.Load() + require.Equal(t, leases, loaded) +} + +func TestFileLeaseStore_SaveIsAtomic(t *testing.T) { + path := filepath.Join(t.TempDir(), "leases.json") + store := NewFileLeaseStore(path) + + store.Save([]PersistedLease{{PID: 1, UDID: "udid-1"}}) + store.Save([]PersistedLease{{PID: 2, UDID: "udid-2"}}) + + // The temp file used for the atomic rename should never be left behind. + _, err := os.Stat(path + ".tmp") + assert.True(t, os.IsNotExist(err)) + + loaded := store.Load() + require.Len(t, loaded, 1) + assert.Equal(t, int32(2), loaded[0].PID) +} + +func TestFileLeaseStore_LoadCorruptFileReturnsNilInsteadOfCrashing(t *testing.T) { + path := filepath.Join(t.TempDir(), "leases.json") + require.NoError(t, os.WriteFile(path, []byte("not valid json"), 0644)) + + store := NewFileLeaseStore(path) + assert.Nil(t, store.Load()) +} + +func TestProcessStartTime_CurrentProcessIsStable(t *testing.T) { + pid := int32(os.Getpid()) + + first := processStartTime(pid) + require.NotNil(t, first, "should be able to read the current process's own start time") + + second := processStartTime(pid) + require.NotNil(t, second) + + assert.Equal(t, *first, *second, "the start time of a still-running process must not change between calls") +} + +func TestProcessStartTime_ExitedProcessReturnsNil(t *testing.T) { + assert.Nil(t, processStartTime(spawnDeadPID(t))) +} diff --git a/tools/simulator_manager/go/logger.go b/tools/simulator_manager/go/logger.go new file mode 100644 index 0000000..e788273 --- /dev/null +++ b/tools/simulator_manager/go/logger.go @@ -0,0 +1,21 @@ +package main + +import ( + "log/slog" + "os" +) + +var ( + logger = newLogger("manager") + childProcessLogger = newLogger("manager.child-process") + httpServerLogger = newLogger("server") + accumulatedHTTPLogger = newLogger("accumulated_http") + simulatorControlLogger = newLogger("control") + leaseStoreLogger = newLogger("manager.lease-store") +) + +func newLogger(category string) *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })).With("subsystem", "com.example.tools.simulator_manager", "category", category) +} diff --git a/tools/simulator_manager/go/lru_set.go b/tools/simulator_manager/go/lru_set.go new file mode 100644 index 0000000..4f7c9a4 --- /dev/null +++ b/tools/simulator_manager/go/lru_set.go @@ -0,0 +1,58 @@ +package main + +// LRUSet is a Set that has a maximum capacity and evicts the least recently used item when full. +type LRUSet[T comparable] struct { + capacity int + // An array to keep track of the order in which elements were inserted. + // The first element in the array is the least recently used. + order []T + // A map to enable fast O(1) membership tests. + storage map[T]struct{} +} + +func NewLRUSet[T comparable](capacity int) *LRUSet[T] { + if capacity <= 0 { + panic("Capacity must be greater than zero.") + } + return &LRUSet[T]{ + capacity: capacity, + order: make([]T, 0, capacity), + storage: make(map[T]struct{}), + } +} + +// Insert returns an element that was evicted from the set. +func (s *LRUSet[T]) Insert(element T) *T { + var evicted *T + if _, exists := s.storage[element]; exists { + // Remove from current position in order. Nothing is evicted here -- + // the element is already in the set, just moving to the + // most-recently-used end. + for i, e := range s.order { + if e == element { + s.order = append(s.order[:i], s.order[i+1:]...) + break + } + } + s.order = append(s.order, element) + } else { + if len(s.order) >= s.capacity && len(s.order) > 0 { + oldest := s.order[0] + s.order = s.order[1:] + delete(s.storage, oldest) + evicted = &oldest + } + s.order = append(s.order, element) + s.storage[element] = struct{}{} + } + return evicted +} + +func (s *LRUSet[T]) Contains(element T) bool { + _, exists := s.storage[element] + return exists +} + +func (s *LRUSet[T]) Elements() []T { + return s.order +} diff --git a/tools/simulator_manager/go/lru_set_test.go b/tools/simulator_manager/go/lru_set_test.go new file mode 100644 index 0000000..dabdc24 --- /dev/null +++ b/tools/simulator_manager/go/lru_set_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLRUSet_InsertAndContains(t *testing.T) { + s := NewLRUSet[string](2) + + assert.Nil(t, s.Insert("a")) + assert.True(t, s.Contains("a")) + assert.False(t, s.Contains("b")) +} + +func TestLRUSet_EvictsLeastRecentlyUsed(t *testing.T) { + s := NewLRUSet[string](2) + + s.Insert("a") + s.Insert("b") + + evicted := s.Insert("c") + assert.NotNil(t, evicted) + assert.Equal(t, "a", *evicted) + assert.False(t, s.Contains("a")) + assert.True(t, s.Contains("b")) + assert.True(t, s.Contains("c")) +} + +func TestLRUSet_ReinsertingExistingElementRefreshesItsPosition(t *testing.T) { + s := NewLRUSet[string](2) + + s.Insert("a") + s.Insert("b") + + // Touching "a" again should make "b" the least-recently-used one instead. + evicted := s.Insert("a") + assert.Nil(t, evicted, "re-inserting an existing element should not evict anything") + + evicted = s.Insert("c") + assert.NotNil(t, evicted) + assert.Equal(t, "b", *evicted, "\"b\" should have become least-recently-used after \"a\" was touched") + assert.True(t, s.Contains("a")) + assert.True(t, s.Contains("c")) +} + +func TestLRUSet_Elements_ReflectsInsertionOrder(t *testing.T) { + s := NewLRUSet[string](3) + s.Insert("a") + s.Insert("b") + s.Insert("c") + + assert.Equal(t, []string{"a", "b", "c"}, s.Elements()) +} + +func TestLRUSet_CapacityMustBePositive(t *testing.T) { + assert.Panics(t, func() { NewLRUSet[string](0) }) + assert.Panics(t, func() { NewLRUSet[string](-1) }) +} diff --git a/tools/simulator_manager/go/main.go b/tools/simulator_manager/go/main.go new file mode 100644 index 0000000..11199ef --- /dev/null +++ b/tools/simulator_manager/go/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" +) + +type arrayFlags []string + +func (a *arrayFlags) String() string { + return strings.Join(*a, ", ") +} + +func (a *arrayFlags) Set(value string) error { + *a = append(*a, value) + return nil +} + +func main() { + var version string + var pidPath string + var unixSocketPath string + var deleteRecentlyUsedIdleAfter uint + var deleteIdleAfter uint + var recentlyUsedCapacity int + var startupProcesses arrayFlags + var postBoot string + var leasePath string + + flag.StringVar(&version, "version", "", "Version of the simulator manager") + flag.StringVar(&pidPath, "pid-path", "", "Path to where the pid should be written") + flag.StringVar(&unixSocketPath, "unix-socket-path", "", "Path to where the unix domain socket should be created") + flag.UintVar(&deleteRecentlyUsedIdleAfter, "delete-recently-used-idle-after", 0, "Number of seconds to wait before deleting a recently used idle simulator") + flag.UintVar(&deleteIdleAfter, "delete-idle-after", 0, "Number of seconds to wait before deleting a non-recently used idle simulator") + flag.IntVar(&recentlyUsedCapacity, "recently-used-capacity", 1, "The number of simulators to keep in the recently used list") + flag.Var(&startupProcesses, "startup-process", "The path to a startup process that will be run when the simulator manager is started") + flag.StringVar(&postBoot, "post-boot", "", "Path to an executable that will run after a simulator clone is booted") + flag.StringVar(&leasePath, "lease-path", "", "Path to a file where leases are mirrored") + + flag.Parse() + + if version == "" { + fmt.Fprintln(os.Stderr, "Error: --version is required") + flag.Usage() + os.Exit(1) + } + + if pidPath == "" { + fmt.Fprintln(os.Stderr, "Error: --pid-path is required") + flag.Usage() + os.Exit(1) + } + + if unixSocketPath == "" { + fmt.Fprintln(os.Stderr, "Error: --unix-socket-path is required") + flag.Usage() + os.Exit(1) + } + + if recentlyUsedCapacity <= 0 { + fmt.Fprintln(os.Stderr, "Error: --recently-used-capacity must be greater than 0") + os.Exit(1) + } + + seen := make(map[string]bool) + for _, proc := range startupProcesses { + if seen[proc] { + fmt.Fprintf(os.Stderr, "Error: --startup-process must be unique, found duplicate: %s\n", proc) + os.Exit(1) + } + seen[proc] = true + } + + var leaseStore LeaseStore + if leasePath != "" { + leaseStore = NewFileLeaseStore(leasePath) + } + + var postBootPtr *string + if postBoot != "" { + postBootPtr = &postBoot + } + + simulatorManager := NewSimulatorManager( + NewRealSimulatorControl(), + uint16(deleteRecentlyUsedIdleAfter), + uint16(deleteIdleAfter), + recentlyUsedCapacity, + true, // deleteOnPIDExit + startupProcesses, + postBootPtr, + leaseStore, + ) + + simulatorManager.RestoreLeases() + + if err := simulatorManager.StartChildProcesses(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to start child processes: %v\n", err) + os.Exit(1) + } + + httpServer := NewHTTPServer( + NewSimulatorRequestHandler(simulatorManager), + version, + ) + + if err := httpServer.Run(pidPath, unixSocketPath); err != nil { + fmt.Fprintf(os.Stderr, "Server error: %v\n", err) + os.Exit(1) + } +} diff --git a/tools/simulator_manager/go/main_test.go b/tools/simulator_manager/go/main_test.go new file mode 100644 index 0000000..8f23f96 --- /dev/null +++ b/tools/simulator_manager/go/main_test.go @@ -0,0 +1,26 @@ +package main + +// main() itself isn't covered here: it parses process-global flags and calls +// os.Exit on invalid input, neither of which can be exercised safely from +// within the test binary's own process. + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestArrayFlags_SetAppendsInOrder(t *testing.T) { + var flags arrayFlags + + assert.NoError(t, flags.Set("a")) + assert.NoError(t, flags.Set("b")) + + assert.Equal(t, arrayFlags{"a", "b"}, flags) + assert.Equal(t, "a, b", flags.String()) +} + +func TestArrayFlags_StringOnEmpty(t *testing.T) { + var flags arrayFlags + assert.Equal(t, "", flags.String()) +} diff --git a/tools/simulator_manager/go/pty.go b/tools/simulator_manager/go/pty.go new file mode 100644 index 0000000..ac0bb00 --- /dev/null +++ b/tools/simulator_manager/go/pty.go @@ -0,0 +1,24 @@ +package main + +import ( + "os" +) + +// PTY represents a pseudo-terminal pair (replaced with pipe for simplicity). +// For child process stdout/stderr capture, pipes work just as well as PTYs +// and don't require CGO. +type PTY struct { + Parent int + Child int +} + +func NewPTY() (*PTY, error) { + r, w, err := os.Pipe() + if err != nil { + return nil, err + } + return &PTY{ + Parent: int(r.Fd()), + Child: int(w.Fd()), + }, nil +} diff --git a/tools/simulator_manager/go/simulator_control.go b/tools/simulator_manager/go/simulator_control.go new file mode 100644 index 0000000..38c53e1 --- /dev/null +++ b/tools/simulator_manager/go/simulator_control.go @@ -0,0 +1,511 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type SimulatorConfig struct { + DeviceType string `json:"deviceType"` + OS string `json:"os"` + Version string `json:"version"` +} + +func (c SimulatorConfig) String() string { + return fmt.Sprintf("%s (%s %s)", c.DeviceType, c.OS, c.Version) +} + +func (c SimulatorConfig) BaseDeviceName() string { + return fmt.Sprintf("EXAMPLE_BAZEL_BASE_%s_%s", c.DeviceType, c.Version) +} + +func (c SimulatorConfig) CloneDeviceName(index int) string { + return fmt.Sprintf("EXAMPLE_BAZEL_CLONE_%s_%s_%d", c.DeviceType, c.Version, index) +} + +func (c SimulatorConfig) RuntimeIdentifier() string { + runtimeVersion := strings.ReplaceAll(c.Version, ".", "-") + return fmt.Sprintf("com.apple.CoreSimulator.SimRuntime.%s-%s", c.OS, runtimeVersion) +} + +type SimCtlDevices struct { + Devices map[string][]SimCtlDevice `json:"devices"` +} + +type SimCtlDevice struct { + Name string `json:"name"` + UDID string `json:"udid"` + State string `json:"state"` +} + +type ProcessError struct { + Command string + Context string + ExitCode int + StdOut string + StdErr string +} + +func (e *ProcessError) Error() string { + contextStr := "" + if e.Context != "" { + contextStr = fmt.Sprintf(" (%s)", e.Context) + } + return fmt.Sprintf("\"%s\"%s failed with exit code %d:\n%s%s", + e.Command, contextStr, e.ExitCode, e.StdOut, e.StdErr) +} + +type SimulatorControl interface { + // CreateBase creates a base simulator with the given config. + // + // It also boots and shuts down the simulator, making it ready for cloning. + // + // If an existing simulator with the same name already exists, that is returned instead of + // creating a new one. This is to support the simulator manager being restarted and losing state. + CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) + + // Clone clones a base simulator. + // + // It also boots the cloned simulator, making it ready for use. + // + // If an existing simulator with the same name already exists, that is returned instead of + // creating a new one. This is to support the simulator manager being restarted and losing state. + Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) + + EnsureBooted(simulator SimulatorUDID, context string) error + + CleanTempFiles(simulator SimulatorUDID) + + Delete(simulator SimulatorUDID, name string, context string) error + + GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) + + // RunningSimulators returns every currently-booted simulator named `name`, + // across all runtimes. Used as a lease-time sanity check: a shared device + // should only ever have one real, booted simulator behind it, so finding + // more than one indicates the sharing/reuse logic let a duplicate device + // come into existence. + RunningSimulators(name string) ([]SimCtlDevice, error) +} + +type RealSimulatorControl struct { + createBaseTasks map[string]*resultBroadcaster + createBaseTasksLock sync.Mutex + + cloneTasks map[string]*resultBroadcaster + cloneTasksLock sync.Mutex + + deleteAndExistenceMutexes map[string]*deleteOrExistenceMutexEntry + deleteAndExistenceMutexesLock sync.Mutex +} + +type taskResult struct { + udid SimulatorUDID + err error +} + +// resultBroadcaster lets any number of callers await the same eventual +// taskResult without consuming it. A plain channel can't do this safely: if +// two goroutines both receive from the one channel used to coalesce a single +// in-flight CreateBase/Clone request, only the first gets the real value -- +// the second gets the channel's zero value (an empty UDID with a nil error, +// i.e. a phantom successful lease) once the channel is drained and closed. +// Waiting on a channel that's closed once, then reading a result set before +// that close, delivers the same value to every waiter instead. +type resultBroadcaster struct { + done chan struct{} + result taskResult +} + +func newResultBroadcaster() *resultBroadcaster { + return &resultBroadcaster{done: make(chan struct{})} +} + +func (b *resultBroadcaster) complete(result taskResult) { + b.result = result + close(b.done) +} + +func (b *resultBroadcaster) wait() taskResult { + <-b.done + return b.result +} + +type deleteOrExistenceMutexEntry struct { + mutex *SimulatorDeleteOrExistenceMutex + count int +} + +func NewRealSimulatorControl() *RealSimulatorControl { + return &RealSimulatorControl{ + createBaseTasks: make(map[string]*resultBroadcaster), + cloneTasks: make(map[string]*resultBroadcaster), + deleteAndExistenceMutexes: make(map[string]*deleteOrExistenceMutexEntry), + } +} + +func (r *RealSimulatorControl) CreateBase(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { + r.createBaseTasksLock.Lock() + if existing, ok := r.createBaseTasks[name]; ok { + r.createBaseTasksLock.Unlock() + result := existing.wait() + return result.udid, result.err + } + + broadcaster := newResultBroadcaster() + r.createBaseTasks[name] = broadcaster + r.createBaseTasksLock.Unlock() + + go func() { + defer func() { + r.createBaseTasksLock.Lock() + delete(r.createBaseTasks, name) + r.createBaseTasksLock.Unlock() + }() + + udid, err := r.createBaseImpl(name, config, runtimeIdentifier) + broadcaster.complete(taskResult{udid: udid, err: err}) + }() + + result := broadcaster.wait() + return result.udid, result.err +} + +func (r *RealSimulatorControl) createBaseImpl(name string, config SimulatorConfig, runtimeIdentifier string) (SimulatorUDID, error) { + if existingUDID, err := r.GetExisting(name, config.DeviceType, runtimeIdentifier, "createBase"); err == nil && existingUDID != "" { + simulatorControlLogger.Info("Base simulator already exists, skipping creation", "name", name, "udid", existingUDID) + + if err := r.shutdown(existingUDID, "createBase existing: "+name); err != nil { + simulatorControlLogger.Error("Failed to set up base simulator; deleting", "name", name, "udid", existingUDID) + _ = r.Delete(existingUDID, name, "createBase existing: "+name) + return "", err + } + + return existingUDID, nil + } + + simulatorControlLogger.Info("Creating base simulator", "config", config, "name", name) + + udid, err := simctl([]string{"create", name, config.DeviceType, runtimeIdentifier}, "") + if err != nil { + return "", err + } + udid = strings.TrimSpace(udid) + + if err := r.EnsureBooted(udid, "createBase new: "+name); err != nil { + simulatorControlLogger.Error("Failed to set up base simulator; deleting", "name", name, "udid", udid) + _ = r.Delete(udid, name, "createBase new: "+name) + return "", err + } + + // Give the simulator some time to do some post-boot processing + time.Sleep(5 * time.Second) + + if err := r.shutdown(udid, "createBase new: "+name); err != nil { + simulatorControlLogger.Error("Failed to set up base simulator; deleting", "name", name, "udid", udid) + _ = r.Delete(udid, name, "createBase new: "+name) + return "", err + } + + simulatorControlLogger.Info("Created base simulator", "config", config, "name", name, "udid", udid) + + return udid, nil +} + +func (r *RealSimulatorControl) Clone(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { + r.cloneTasksLock.Lock() + if existing, ok := r.cloneTasks[name]; ok { + r.cloneTasksLock.Unlock() + result := existing.wait() + return result.udid, result.err + } + + broadcaster := newResultBroadcaster() + r.cloneTasks[name] = broadcaster + r.cloneTasksLock.Unlock() + + go func() { + defer func() { + r.cloneTasksLock.Lock() + delete(r.cloneTasks, name) + r.cloneTasksLock.Unlock() + }() + + udid, err := r.cloneImpl(baseSimulator, name, deviceType, runtimeIdentifier, postBoot) + broadcaster.complete(taskResult{udid: udid, err: err}) + }() + + result := broadcaster.wait() + return result.udid, result.err +} + +func (r *RealSimulatorControl) cloneImpl(baseSimulator SimulatorUDID, name string, deviceType string, runtimeIdentifier string, postBoot *string) (SimulatorUDID, error) { + var udid string + var isExisting bool + + if existingUDID, err := r.GetExisting(name, deviceType, runtimeIdentifier, "clone"); err == nil && existingUDID != "" { + udid = existingUDID + isExisting = true + + simulatorControlLogger.Info("Cloned simulator already exists, skipping creation", "name", name, "udid", udid) + + if err := r.EnsureBooted(udid, "clone, existing: "+name); err != nil { + return "", err + } + } else { + isExisting = false + + simulatorControlLogger.Info("Cloning base simulator", "base", baseSimulator, "name", name) + + var err error + udid, err = simctl([]string{"clone", baseSimulator, name}, "") + if err != nil { + return "", err + } + udid = strings.TrimSpace(udid) + + simulatorControlLogger.Info("Cloned base simulator", "base", baseSimulator, "name", name, "udid", udid) + + if err := r.EnsureBooted(udid, "clone, new: "+name); err != nil { + return "", err + } + } + + if postBoot != nil && *postBoot != "" { + simulatorControlLogger.Info("Running post-boot script", "script", *postBoot, "udid", udid) + + cmd := exec.Command(*postBoot) + cmd.Env = append(os.Environ(), "SIMULATOR_UDID="+udid) + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("postBoot failed (isExisting: %v): %w", isExisting, err) + } + } + + return udid, nil +} + +func (r *RealSimulatorControl) shutdown(simulator SimulatorUDID, context string) error { + _, err := simctl([]string{"shutdown", simulator}, context) + if err != nil { + if pe, ok := err.(*ProcessError); ok && pe.ExitCode == 149 { + simulatorControlLogger.Warn("Shutdown failed, but probably already shut down", "error", err) + return nil + } + return err + } + return nil +} + +func (r *RealSimulatorControl) CleanTempFiles(simulator SimulatorUDID) { + homeDir, err := os.UserHomeDir() + if err != nil { + return + } + + deadCachesPath := filepath.Join(homeDir, "Library/Developer/CoreSimulator/Devices", simulator, + "data/Library/Caches/com.apple.containermanagerd/Dead") + + contents, err := os.ReadDir(deadCachesPath) + if err != nil { + return + } + + for _, item := range contents { + itemPath := filepath.Join(deadCachesPath, item.Name()) + _ = os.RemoveAll(itemPath) + } +} + +func (r *RealSimulatorControl) Delete(simulator SimulatorUDID, name string, context string) error { + return r.deleteAndExistenceMutex(name, func(mutex *SimulatorDeleteOrExistenceMutex) error { + return mutex.unlockedDelete(simulator, context) + }) +} + +func (r *RealSimulatorControl) RunningSimulators(name string) ([]SimCtlDevice, error) { + output, err := simctl([]string{"list", "devices", "-j"}, "assertAtMostOneRunning") + if err != nil { + return nil, err + } + + var devices SimCtlDevices + if err := json.Unmarshal([]byte(output), &devices); err != nil { + simulatorControlLogger.Error("Failed to decode 'simctl list devices -j'", "error", err, "output", output) + return nil, fmt.Errorf("failed to decode output: %w - %s", err, output) + } + + var running []SimCtlDevice + for _, deviceList := range devices.Devices { + for _, device := range deviceList { + if device.Name == name && device.State == "Booted" { + running = append(running, device) + } + } + } + return running, nil +} + +func (r *RealSimulatorControl) GetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { + var result string + err := r.deleteAndExistenceMutex(name, func(mutex *SimulatorDeleteOrExistenceMutex) error { + var err error + result, err = mutex.unlockedGetExisting(name, deviceType, runtimeIdentifier, context) + return err + }) + return result, err +} + +func (r *RealSimulatorControl) EnsureBooted(simulator SimulatorUDID, context string) error { + for retriesLeft := 1; retriesLeft >= 0; retriesLeft-- { + _, err := simctl([]string{"bootstatus", simulator, "-b"}, context) + if err == nil { + break + } + + if pe, ok := err.(*ProcessError); ok && pe.ExitCode == 149 && retriesLeft > 0 { + simulatorControlLogger.Warn("Boot failed, but probably already booted", "simulator", simulator, "error", err) + continue + } + + return err + } + return nil +} + +func (r *RealSimulatorControl) deleteAndExistenceMutex(name string, fn func(*SimulatorDeleteOrExistenceMutex) error) error { + r.deleteAndExistenceMutexesLock.Lock() + entry, ok := r.deleteAndExistenceMutexes[name] + if !ok { + entry = &deleteOrExistenceMutexEntry{ + mutex: NewSimulatorDeleteOrExistenceMutex(), + count: 0, + } + r.deleteAndExistenceMutexes[name] = entry + } + entry.count++ + r.deleteAndExistenceMutexesLock.Unlock() + + defer func() { + r.deleteAndExistenceMutexesLock.Lock() + entry.count-- + if entry.count == 0 { + delete(r.deleteAndExistenceMutexes, name) + } + r.deleteAndExistenceMutexesLock.Unlock() + }() + + return entry.mutex.WithLock(fn) +} + +type SimulatorDeleteOrExistenceMutex struct { + mu sync.Mutex +} + +func NewSimulatorDeleteOrExistenceMutex() *SimulatorDeleteOrExistenceMutex { + return &SimulatorDeleteOrExistenceMutex{} +} + +func (m *SimulatorDeleteOrExistenceMutex) WithLock(fn func(*SimulatorDeleteOrExistenceMutex) error) error { + m.mu.Lock() + defer m.mu.Unlock() + return fn(m) +} + +func (m *SimulatorDeleteOrExistenceMutex) unlockedGetExisting(name string, deviceType string, runtimeIdentifier string, context string) (string, error) { + simulatorControlLogger.Debug("Trying to find existing simulator", "name", name) + + output, err := simctl([]string{"list", "devices", "-j", deviceType}, context) + if err != nil { + return "", err + } + + var devices SimCtlDevices + if err := json.Unmarshal([]byte(output), &devices); err != nil { + simulatorControlLogger.Error("Failed to decode 'simctl list devices -j'", "error", err, "output", output) + return "", fmt.Errorf("failed to decode output: %w - %s", err, output) + } + + if deviceList, ok := devices.Devices[runtimeIdentifier]; ok { + for _, device := range deviceList { + if device.Name == name { + udid := device.UDID + simulatorControlLogger.Debug("Found existing simulator", "name", name, "udid", udid) + + homeDir, _ := os.UserHomeDir() + devicePath := filepath.Join(homeDir, "Library/Developer/CoreSimulator/Devices", udid) + if _, err := os.Stat(devicePath); os.IsNotExist(err) { + simulatorControlLogger.Debug("Simulator doesn't actually exist on disk; deleting", "udid", udid) + _ = m.unlockedDelete(udid, context) + return "", nil + } + + return udid, nil + } + } + } + + simulatorControlLogger.Debug("No existing simulator found", "name", name) + return "", nil +} + +func (m *SimulatorDeleteOrExistenceMutex) unlockedDelete(simulator SimulatorUDID, context string) error { + simulatorControlLogger.Info("Deleting simulator", "udid", simulator) + + if _, err := simctl([]string{"delete", simulator}, context); err != nil { + simulatorControlLogger.Error("Failed to delete simulator", "udid", simulator, "error", err) + return err + } + + simulatorControlLogger.Info("Deleted simulator", "udid", simulator) + return nil +} + +func simctl(args []string, context string) (string, error) { + return subprocess("/usr/bin/xcrun", append([]string{"simctl"}, args...), nil, context) +} + +func subprocess(executable string, args []string, env map[string]string, context string) (string, error) { + cmd := exec.Command(executable, args...) + if env != nil { + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + } + + quotedArgs := make([]string, len(args)) + for i, arg := range args { + quotedArgs[i] = fmt.Sprintf("'%s'", arg) + } + command := fmt.Sprintf("%s %s", executable, strings.Join(quotedArgs, " ")) + + simulatorControlLogger.Debug("Running command", "command", command) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + exitCode := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } + return "", &ProcessError{ + Command: command, + Context: context, + ExitCode: exitCode, + StdOut: stdout.String(), + StdErr: stderr.String(), + } + } + + return stdout.String(), nil +} diff --git a/tools/simulator_manager/go/simulator_control_test.go b/tools/simulator_manager/go/simulator_control_test.go new file mode 100644 index 0000000..3985914 --- /dev/null +++ b/tools/simulator_manager/go/simulator_control_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSimulatorConfig_Naming(t *testing.T) { + cfg := SimulatorConfig{DeviceType: "iPhone15,3", OS: "iOS", Version: "17.4"} + + assert.Equal(t, "iPhone15,3 (iOS 17.4)", cfg.String()) + assert.Equal(t, "EXAMPLE_BAZEL_BASE_iPhone15,3_17.4", cfg.BaseDeviceName()) + assert.Equal(t, "EXAMPLE_BAZEL_CLONE_iPhone15,3_17.4_0", cfg.CloneDeviceName(0)) + assert.Equal(t, "EXAMPLE_BAZEL_CLONE_iPhone15,3_17.4_3", cfg.CloneDeviceName(3)) + assert.Equal(t, "com.apple.CoreSimulator.SimRuntime.iOS-17-4", cfg.RuntimeIdentifier()) +} + +func TestProcessError_Error(t *testing.T) { + withoutContext := &ProcessError{Command: "xcrun simctl delete X", ExitCode: 1, StdOut: "out", StdErr: "err"} + assert.Equal(t, "\"xcrun simctl delete X\" failed with exit code 1:\nouterr", withoutContext.Error()) + + withContext := &ProcessError{Command: "xcrun simctl delete X", Context: "reaper", ExitCode: 2, StdOut: "", StdErr: "boom"} + assert.Equal(t, "\"xcrun simctl delete X\" (reaper) failed with exit code 2:\nboom", withContext.Error()) +} + +func TestSubprocess_CapturesStdout(t *testing.T) { + out, err := subprocess("/bin/echo", []string{"hello", "world"}, nil, "") + assert := assert.New(t) + assert.NoError(err) + assert.Equal("hello world\n", out) +} + +func TestSubprocess_FailureReturnsProcessErrorWithExitCode(t *testing.T) { + _, err := subprocess("/usr/bin/false", nil, nil, "test-context") + + pe, ok := err.(*ProcessError) + assert := assert.New(t) + assert.True(ok, "expected a *ProcessError, got %T", err) + assert.Equal(1, pe.ExitCode) + assert.Equal("test-context", pe.Context) +} + +func TestSubprocess_PassesEnvironment(t *testing.T) { + out, err := subprocess("/bin/sh", []string{"-c", "echo $SIMULATOR_UDID"}, map[string]string{"SIMULATOR_UDID": "udid-123"}, "") + assert := assert.New(t) + assert.NoError(err) + assert.Equal("udid-123\n", out) +} diff --git a/tools/simulator_manager/go/simulator_manager.go b/tools/simulator_manager/go/simulator_manager.go new file mode 100644 index 0000000..951742c --- /dev/null +++ b/tools/simulator_manager/go/simulator_manager.go @@ -0,0 +1,862 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "syscall" + "time" +) + +type SimulatorManagerError struct { + kind string + udid SimulatorUDID + message string +} + +func (e *SimulatorManagerError) Error() string { + if e.message != "" { + return e.message + } + return e.kind +} + +var ( + ErrAlreadyLeased = &SimulatorManagerError{kind: "alreadyLeased"} + ErrNoLease = &SimulatorManagerError{kind: "noLease"} + ErrLeaserExited = &SimulatorManagerError{kind: "leaserExited"} +) + +func NewAlreadyLeasedError(udid SimulatorUDID) error { + return &SimulatorManagerError{kind: "alreadyLeased", udid: udid} +} + +type simulatorLease struct { + udid SimulatorUDID + config SimulatorConfig + exclusive bool + slotIndex int + leaserStartTime *uint64 +} + +type simulatorSlot struct { + kind string + udid SimulatorUDID + exclusive bool + task *resultBroadcaster + cancel context.CancelFunc +} + +const ( + slotEmpty = "empty" + slotPendingCreation = "pendingCreation" + slotActive = "active" + slotPendingDeletion = "pendingDeletion" + slotDeleting = "deleting" +) + +func (s simulatorSlot) sortOrder() int { + switch s.kind { + case slotActive: + return 0 + case slotPendingCreation: + return 1 + case slotPendingDeletion: + return 2 + case slotEmpty: + return 3 + case slotDeleting: + return 4 + default: + return 5 + } +} + +type SimulatorManager struct { + simulatorControl SimulatorControl + + mu sync.Mutex + simulatorSlots map[SimulatorConfig][]simulatorSlot + referenceCount map[SimulatorUDID]int + leases map[int32]simulatorLease + leaserExitWatches map[int32]context.CancelFunc + + getBaseSimulatorTasks map[SimulatorConfig]*resultBroadcaster + + deleteIdleAfter uint16 + deleteRecentlyUsedIdleAfter uint16 + deleteOnPIDExit bool + + leaseStore LeaseStore + + recentlyLeased *LRUSet[SimulatorConfig] + + startupProcessPaths []string + postBoot *string + childProcessCancel []context.CancelFunc +} + +func NewSimulatorManager( + simulatorControl SimulatorControl, + deleteRecentlyUsedIdleAfter uint16, + deleteIdleAfter uint16, + recentlyUsedCapacity int, + deleteOnPIDExit bool, + startupProcesses []string, + postBoot *string, + leaseStore LeaseStore, +) *SimulatorManager { + if err := os.Chdir("/tmp"); err != nil { + logger.Warn("Failed to change directory to /tmp", "error", err) + } + + return &SimulatorManager{ + simulatorControl: simulatorControl, + simulatorSlots: make(map[SimulatorConfig][]simulatorSlot), + referenceCount: make(map[SimulatorUDID]int), + leases: make(map[int32]simulatorLease), + leaserExitWatches: make(map[int32]context.CancelFunc), + getBaseSimulatorTasks: make(map[SimulatorConfig]*resultBroadcaster), + deleteIdleAfter: deleteIdleAfter, + deleteRecentlyUsedIdleAfter: deleteRecentlyUsedIdleAfter, + deleteOnPIDExit: deleteOnPIDExit, + leaseStore: leaseStore, + recentlyLeased: NewLRUSet[SimulatorConfig](recentlyUsedCapacity), + startupProcessPaths: startupProcesses, + postBoot: postBoot, + } +} + +func (sm *SimulatorManager) StartChildProcesses() error { + for _, path := range sm.startupProcessPaths { + ctx, cancel := context.WithCancel(context.Background()) + sm.childProcessCancel = append(sm.childProcessCancel, cancel) + go sm.startChildProcess(ctx, path) + } + return nil +} + +func (sm *SimulatorManager) startChildProcess(ctx context.Context, path string) { + childProcessLogger.Info("Starting child process", "path", path) + + cmd := exec.Command(path) + + outPTY, err := NewPTY() + if err != nil { + childProcessLogger.Error("Failed to create output PTY", "path", path, "error", err) + return + } + + errPTY, err := NewPTY() + if err != nil { + childProcessLogger.Error("Failed to create error PTY", "path", path, "error", err) + return + } + + cmd.Stdout = os.NewFile(uintptr(outPTY.Child), "stdout") + cmd.Stderr = os.NewFile(uintptr(errPTY.Child), "stderr") + + go watchFD(outPTY.Parent, func(line string) { + childProcessLogger.Info(fmt.Sprintf("[%s] %s", path, line)) + }) + + go watchFD(errPTY.Parent, func(line string) { + childProcessLogger.Error(fmt.Sprintf("[%s] %s", path, line)) + }) + + if err := cmd.Start(); err != nil { + childProcessLogger.Error("Failed to start child process", "path", path, "error", err) + return + } + + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + select { + case <-ctx.Done(): + _ = cmd.Process.Kill() + return + case err := <-done: + if err != nil { + childProcessLogger.Warn("Child process exited with error", "path", path, "error", err) + } else { + childProcessLogger.Warn("Child process exited", "path", path) + } + } +} + +func watchFD(fd int, onLine func(string)) { + file := os.NewFile(uintptr(fd), "pipe") + defer file.Close() + + buf := make([]byte, 4096) + var lineBuffer []byte + + for { + n, err := file.Read(buf) + if err != nil { + if err != io.EOF { + logger.Error("Error reading from FD", "error", err) + } + break + } + + lineBuffer = append(lineBuffer, buf[:n]...) + + for { + newlineIdx := -1 + for i, b := range lineBuffer { + if b == '\n' { + newlineIdx = i + break + } + } + + if newlineIdx == -1 { + break + } + + line := string(lineBuffer[:newlineIdx]) + onLine(line) + lineBuffer = lineBuffer[newlineIdx+1:] + } + } +} + +func (sm *SimulatorManager) RestoreLeases() { + if sm.leaseStore == nil { + return + } + + persisted := sm.leaseStore.Load() + if len(persisted) == 0 { + return + } + + sm.mu.Lock() + defer sm.mu.Unlock() + + adopted := 0 + dropped := 0 + + for _, lease := range persisted { + if !sm.leaserSurvived(lease) { + dropped++ + continue + } + + if !sm.canAdopt(lease) { + logger.Error("Not restoring conflicting lease", "pid", lease.PID, "udid", lease.UDID) + dropped++ + continue + } + + sm.leases[lease.PID] = simulatorLease{ + udid: lease.UDID, + config: lease.Config, + exclusive: lease.Exclusive, + slotIndex: lease.SlotIndex, + leaserStartTime: lease.LeaserStartTime, + } + + slots := sm.simulatorSlots[lease.Config] + for len(slots) <= lease.SlotIndex { + slots = append(slots, simulatorSlot{kind: slotEmpty}) + } + slots[lease.SlotIndex] = simulatorSlot{ + kind: slotActive, + udid: lease.UDID, + exclusive: lease.Exclusive, + } + sm.simulatorSlots[lease.Config] = slots + + sm.incrementReferenceCount(lease.UDID) + + if sm.deleteOnPIDExit { + sm.registerReleaseOnExit(lease.PID) + } + + adopted++ + } + + logger.Info("Restored leases from previous simulator manager", "adopted", adopted, "dropped", dropped) + + sm.persistLeases() +} + +func (sm *SimulatorManager) leaserSurvived(lease PersistedLease) bool { + if !processIsRunning(lease.PID) { + return false + } + + if lease.LeaserStartTime == nil { + return true + } + + currentStart := processStartTime(lease.PID) + if currentStart == nil { + return false + } + + if *currentStart != *lease.LeaserStartTime { + logger.Info("PID is running but started at a different time; treating as exited", "pid", lease.PID) + return false + } + + return true +} + +func (sm *SimulatorManager) canAdopt(lease PersistedLease) bool { + for _, existing := range sm.leases { + if existing.udid == lease.UDID && (existing.exclusive || lease.Exclusive) { + return false + } + + if existing.config == lease.Config && + existing.slotIndex == lease.SlotIndex && + existing.udid != lease.UDID { + return false + } + } + return true +} + +func (sm *SimulatorManager) persistLeases() { + if sm.leaseStore == nil { + return + } + + leases := make([]PersistedLease, 0, len(sm.leases)) + for pid, lease := range sm.leases { + leases = append(leases, PersistedLease{ + PID: pid, + LeaserStartTime: lease.leaserStartTime, + UDID: lease.udid, + Config: lease.config, + Exclusive: lease.exclusive, + SlotIndex: lease.slotIndex, + }) + } + + sm.leaseStore.Save(leases) +} + +func (sm *SimulatorManager) Lease(leaser int32, exclusive bool, config SimulatorConfig) (SimulatorUDID, error) { + sm.mu.Lock() + if existingLease, ok := sm.leases[leaser]; ok { + sm.mu.Unlock() + return "", NewAlreadyLeasedError(existingLease.udid) + } + sm.mu.Unlock() + + logger.Info("Leasing simulator", "exclusive", exclusive, "config", config, "pid", leaser) + + if sm.deleteOnPIDExit && !processIsRunning(leaser) { + logger.Info("PID exited before its lease could be provisioned", "pid", leaser) + return "", ErrLeaserExited + } + + simulator, slotIndex, err := sm.getSimulator(config, exclusive) + if err != nil { + return "", err + } + + if err := sm.assertAtMostOneRunning(config, slotIndex); err != nil { + return "", err + } + + sm.mu.Lock() + sm.recentlyLeased.Insert(config) + + sm.leases[leaser] = simulatorLease{ + udid: simulator, + config: config, + exclusive: exclusive, + slotIndex: slotIndex, + leaserStartTime: processStartTime(leaser), + } + sm.persistLeases() + sm.mu.Unlock() + + if sm.deleteOnPIDExit && !processIsRunning(leaser) { + logger.Info("PID exited while simulator was being provisioned; returning to pool", "pid", leaser, "udid", simulator) + _ = sm.Release(leaser) + return "", ErrLeaserExited + } + + logger.Info("Leased simulator", "udid", simulator, "pid", leaser) + + if sm.deleteOnPIDExit { + sm.mu.Lock() + sm.registerReleaseOnExit(leaser) + sm.mu.Unlock() + } + + return simulator, nil +} + +// assertAtMostOneRunning is a lease-time sanity check, not a recoverable +// error path: a device backing one slot should only ever have one real, +// booted simulator behind it. If simctl reports more than one booted +// simulator sharing this slot's name, the reuse/sharing logic has let a +// duplicate device come into existence -- exactly the failure mode that +// causes leaked, unreferenced launchd_sim processes. Failing loudly here +// catches that at the moment it happens, rather than relying solely on +// after-the-fact reaping. +// +// A failure to even perform the check (simctl itself erroring) is logged and +// ignored rather than failing the lease -- the check is a diagnostic, and +// its own failure shouldn't block a lease that's otherwise fine. +func (sm *SimulatorManager) assertAtMostOneRunning(config SimulatorConfig, slotIndex int) error { + name := config.CloneDeviceName(slotIndex) + + running, err := sm.simulatorControl.RunningSimulators(name) + if err != nil { + logger.Warn("Failed to list running simulators for lease assertion", "name", name, "error", err) + return nil + } + + if len(running) <= 1 { + return nil + } + + udids := make([]string, len(running)) + for i, device := range running { + udids[i] = device.UDID + } + + return fmt.Errorf( + "assertion failed: %d simulators named %q are running (expected at most 1): %s", + len(running), name, strings.Join(udids, ", "), + ) +} + +func (sm *SimulatorManager) LiveLeaseCount() int { + sm.mu.Lock() + defer sm.mu.Unlock() + + count := 0 + for pid := range sm.leases { + if processIsRunning(pid) { + count++ + } + } + return count +} + +func (sm *SimulatorManager) Release(leaser int32) error { + sm.mu.Lock() + lease, ok := sm.leases[leaser] + if !ok { + sm.mu.Unlock() + return ErrNoLease + } + delete(sm.leases, leaser) + + logger.Info("Releasing simulator", "udid", lease.udid, "pid", leaser) + + sm.persistLeases() + sm.removeReleaseOnExit(leaser) + sm.mu.Unlock() + + sm.simulatorControl.CleanTempFiles(lease.udid) + + return sm.decrementReferenceCount(lease.udid, lease.config, lease.slotIndex) +} + +func (sm *SimulatorManager) getBase(config SimulatorConfig) (SimulatorUDID, error) { + sm.mu.Lock() + if existing, ok := sm.getBaseSimulatorTasks[config]; ok { + sm.mu.Unlock() + result := existing.wait() + return result.udid, result.err + } + + broadcaster := newResultBroadcaster() + sm.getBaseSimulatorTasks[config] = broadcaster + sm.mu.Unlock() + + go func() { + defer func() { + sm.mu.Lock() + delete(sm.getBaseSimulatorTasks, config) + sm.mu.Unlock() + }() + + logger.Info("Creating base simulator", "config", config) + + baseSimulator, err := sm.simulatorControl.CreateBase( + config.BaseDeviceName(), + config, + config.RuntimeIdentifier(), + ) + + if err == nil { + logger.Info("Created base simulator", "config", config, "udid", baseSimulator) + } + + broadcaster.complete(taskResult{udid: baseSimulator, err: err}) + }() + + result := broadcaster.wait() + return result.udid, result.err +} + +func (sm *SimulatorManager) incrementReferenceCount(simulator SimulatorUDID) { + count := sm.referenceCount[simulator] + count++ + sm.referenceCount[simulator] = count + + logger.Debug("Reference count increased", "udid", simulator, "count", count) +} + +func (sm *SimulatorManager) decrementReferenceCount(simulator SimulatorUDID, config SimulatorConfig, slotIndex int) error { + sm.mu.Lock() + count, ok := sm.referenceCount[simulator] + if !ok { + sm.mu.Unlock() + return nil + } + + count-- + sm.referenceCount[simulator] = count + + logger.Debug("Reference count decreased", "udid", simulator, "count", count) + + if count > 0 { + sm.mu.Unlock() + return nil + } + sm.mu.Unlock() + + sm.pendingDeletion(simulator, config, slotIndex) + return nil +} + +func (sm *SimulatorManager) getSimulator(config SimulatorConfig, exclusive bool) (SimulatorUDID, int, error) { + sm.mu.Lock() + defer sm.mu.Unlock() + + if _, ok := sm.simulatorSlots[config]; !ok { + sm.simulatorSlots[config] = []simulatorSlot{} + } + + slots := sm.simulatorSlots[config] + type indexedSlot struct { + index int + slot simulatorSlot + } + + sortedSlots := make([]indexedSlot, len(slots)) + for i, slot := range slots { + sortedSlots[i] = indexedSlot{index: i, slot: slot} + } + + // Sort by sort order, then by index + for i := 0; i < len(sortedSlots); i++ { + for j := i + 1; j < len(sortedSlots); j++ { + iOrder := sortedSlots[i].slot.sortOrder() + jOrder := sortedSlots[j].slot.sortOrder() + if iOrder > jOrder || (iOrder == jOrder && sortedSlots[i].index > sortedSlots[j].index) { + sortedSlots[i], sortedSlots[j] = sortedSlots[j], sortedSlots[i] + } + } + } + + for _, is := range sortedSlots { + slot := is.slot + index := is.index + + switch slot.kind { + case slotActive: + if !slot.exclusive && !exclusive { + sm.mu.Unlock() + sim, err := sm.reuseSimulator(slot.udid, config, exclusive, index) + sm.mu.Lock() + return sim, index, err + } + + case slotPendingDeletion: + logger.Info("Turning pending deletion into active simulator", "udid", slot.udid, "exclusive", exclusive) + + sm.simulatorSlots[config][index] = simulatorSlot{ + kind: slotActive, + udid: slot.udid, + exclusive: exclusive, + } + + if slot.cancel != nil { + slot.cancel() + } + + sm.mu.Unlock() + sim, err := sm.reuseSimulator(slot.udid, config, exclusive, index) + sm.mu.Lock() + return sim, index, err + + case slotEmpty: + task, cancel := sm.createCloneTask(config, exclusive, index) + sm.simulatorSlots[config][index] = simulatorSlot{ + kind: slotPendingCreation, + task: task, + exclusive: exclusive, + cancel: cancel, + } + sm.mu.Unlock() + result := task.wait() + sm.mu.Lock() + return result.udid, index, result.err + + case slotPendingCreation: + if !slot.exclusive && !exclusive { + sm.mu.Unlock() + result := slot.task.wait() + if result.err == nil { + sm.mu.Lock() + sm.incrementReferenceCount(result.udid) + sm.mu.Unlock() + } + sm.mu.Lock() + return result.udid, index, result.err + } + } + } + + index := len(sm.simulatorSlots[config]) + task, cancel := sm.createCloneTask(config, exclusive, index) + sm.simulatorSlots[config] = append(sm.simulatorSlots[config], simulatorSlot{ + kind: slotPendingCreation, + task: task, + exclusive: exclusive, + cancel: cancel, + }) + sm.mu.Unlock() + result := task.wait() + sm.mu.Lock() + return result.udid, index, result.err +} + +func (sm *SimulatorManager) reuseSimulator(simulator SimulatorUDID, config SimulatorConfig, exclusive bool, slotIndex int) (SimulatorUDID, error) { + sm.mu.Lock() + sm.incrementReferenceCount(simulator) + sm.mu.Unlock() + + err := sm.simulatorControl.EnsureBooted(simulator, fmt.Sprintf("getSimulator, reused: %s", config.CloneDeviceName(slotIndex))) + if err != nil { + if pe, ok := err.(*ProcessError); ok && pe.ExitCode == 148 { + logger.Warn("Boot of existing simulator failed; deleting and returning new simulator", "udid", simulator, "error", err) + + sm.mu.Lock() + _ = sm.delete(simulator, config, slotIndex, false, "getSimulator, reused: "+config.CloneDeviceName(slotIndex)) + + task, cancel := sm.createCloneTask(config, exclusive, slotIndex) + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotPendingCreation, + task: task, + exclusive: exclusive, + cancel: cancel, + } + sm.mu.Unlock() + + result := task.wait() + return result.udid, result.err + } + return "", err + } + + return simulator, nil +} + +func (sm *SimulatorManager) createCloneTask(config SimulatorConfig, exclusive bool, slotIndex int) (*resultBroadcaster, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + broadcaster := newResultBroadcaster() + + go func() { + baseUDID, err := sm.getBase(config) + if err != nil { + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + sm.mu.Unlock() + broadcaster.complete(taskResult{err: err}) + return + } + + if ctx.Err() != nil { + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + sm.mu.Unlock() + broadcaster.complete(taskResult{err: ctx.Err()}) + return + } + + simulator, err := sm.simulatorControl.Clone( + baseUDID, + config.CloneDeviceName(slotIndex), + config.DeviceType, + config.RuntimeIdentifier(), + sm.postBoot, + ) + + if err != nil { + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + sm.mu.Unlock() + broadcaster.complete(taskResult{err: err}) + return + } + + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotActive, + udid: simulator, + exclusive: exclusive, + } + sm.incrementReferenceCount(simulator) + sm.mu.Unlock() + + broadcaster.complete(taskResult{udid: simulator, err: nil}) + }() + + return broadcaster, cancel +} + +func (sm *SimulatorManager) registerReleaseOnExit(leaser int32) { + ctx, cancel := context.WithCancel(context.Background()) + sm.leaserExitWatches[leaser] = cancel + + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !processIsRunning(leaser) { + logger.Debug("PID exited", "pid", leaser) + _ = sm.Release(leaser) + return + } + } + } + }() +} + +func (sm *SimulatorManager) removeReleaseOnExit(leaser int32) { + if cancel, ok := sm.leaserExitWatches[leaser]; ok { + delete(sm.leaserExitWatches, leaser) + cancel() + } +} + +func (sm *SimulatorManager) pendingDeletion(simulator SimulatorUDID, config SimulatorConfig, slotIndex int) { + if sm.deleteIdleAfter == 0 && sm.deleteRecentlyUsedIdleAfter == 0 { + sm.mu.Lock() + _ = sm.delete(simulator, config, slotIndex, true, "pendingDeletion immediate") + sm.mu.Unlock() + return + } + + ctx, cancel := context.WithCancel(context.Background()) + + sm.mu.Lock() + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotPendingDeletion, + udid: simulator, + cancel: cancel, + } + sm.mu.Unlock() + + go func() { + logger.Info("Scheduling delete of simulator", "udid", simulator, "idleAfter", sm.deleteIdleAfter, "recentlyUsedIdleAfter", sm.deleteRecentlyUsedIdleAfter) + + now := time.Now() + shortDeadline := now.Add(time.Duration(sm.deleteIdleAfter) * time.Second) + recentlyUsedDeadline := now.Add(time.Duration(sm.deleteRecentlyUsedIdleAfter) * time.Second) + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + var remainingTime time.Duration + sm.mu.Lock() + if sm.recentlyLeased.Contains(config) { + remainingTime = time.Until(recentlyUsedDeadline) + } else { + remainingTime = time.Until(shortDeadline) + } + sm.mu.Unlock() + + if remainingTime <= 0 { + sm.mu.Lock() + slots := sm.simulatorSlots[config] + if slotIndex < len(slots) { + slot := slots[slotIndex] + if slot.kind == slotPendingDeletion && slot.udid == simulator { + _ = sm.delete(simulator, config, slotIndex, true, "pendingDeletion delayed") + } + } + sm.mu.Unlock() + return + } + } + } + }() +} + +func (sm *SimulatorManager) delete(simulator SimulatorUDID, config SimulatorConfig, slotIndex int, cleanUpSlots bool, context string) error { + name := config.CloneDeviceName(slotIndex) + + logger.Info("Deleting simulator", "udid", simulator, "name", name) + + sm.simulatorSlots[config][slotIndex] = simulatorSlot{ + kind: slotDeleting, + udid: simulator, + } + + delete(sm.referenceCount, simulator) + + defer func() { + sm.simulatorSlots[config][slotIndex] = simulatorSlot{kind: slotEmpty} + + if cleanUpSlots { + slots := sm.simulatorSlots[config] + for len(slots) > 0 && slots[len(slots)-1].kind == slotEmpty { + slots = slots[:len(slots)-1] + } + sm.simulatorSlots[config] = slots + } + }() + + err := sm.simulatorControl.Delete(simulator, name, context) + if err == nil { + logger.Info("Deleted simulator", "udid", simulator, "name", name) + } + + return err +} + +func (sm *SimulatorManager) Close() { + for _, cancel := range sm.childProcessCancel { + cancel() + } +} + +func processIsRunning(pid int32) bool { + err := syscall.Kill(int(pid), 0) + if err == nil { + return true + } + return err != syscall.ESRCH +} diff --git a/tools/simulator_manager/go/simulator_manager_test.go b/tools/simulator_manager/go/simulator_manager_test.go new file mode 100644 index 0000000..c9ee2d4 --- /dev/null +++ b/tools/simulator_manager/go/simulator_manager_test.go @@ -0,0 +1,386 @@ +package main + +import ( + "os" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestManager(control SimulatorControl, deleteIdleAfter uint16, deleteRecentlyUsedIdleAfter uint16, deleteOnPIDExit bool, leaseStore LeaseStore) *SimulatorManager { + return NewSimulatorManager(control, deleteRecentlyUsedIdleAfter, deleteIdleAfter, 1, deleteOnPIDExit, nil, nil, leaseStore) +} + +func testConfig(deviceType string) SimulatorConfig { + return SimulatorConfig{DeviceType: deviceType, OS: "iOS", Version: "17.0"} +} + +// spawnDeadPID starts and waits for a trivial subprocess to exit, then +// returns its PID. Once a process is reaped, the kernel guarantees +// kill(pid, 0) reports ESRCH for that PID (barring an extremely unlikely +// immediate reuse), making this a reliable "definitely not running" PID. +func spawnDeadPID(t *testing.T) int32 { + t.Helper() + cmd := exec.Command("/usr/bin/true") + require.NoError(t, cmd.Run()) + return int32(cmd.Process.Pid) +} + +// spawnLiveProcess starts a long-lived subprocess, killing it automatically +// at test cleanup, and returns its PID. +func spawnLiveProcess(t *testing.T) (*exec.Cmd, int32) { + t.Helper() + cmd := exec.Command("/bin/sleep", "30") + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + return cmd, int32(cmd.Process.Pid) +} + +func TestLease_NonExclusive_SharesDevice(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + + udid2, err := sm.Lease(2, false, cfg) + require.NoError(t, err) + + assert.Equal(t, udid1, udid2, "two non-exclusive leases for the same config should share one device") + assert.Len(t, control.cloneCalls, 1, "only one clone should have been created") +} + +func TestLease_Exclusive_NeverShares(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, true, cfg) + require.NoError(t, err) + + udid2, err := sm.Lease(2, true, cfg) + require.NoError(t, err) + + assert.NotEqual(t, udid1, udid2, "exclusive leases must never share a device") + assert.Len(t, control.cloneCalls, 2) +} + +func TestLease_ExclusiveDoesNotShareWithNonExclusive(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + nonExclusive, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + + exclusive, err := sm.Lease(2, true, cfg) + require.NoError(t, err) + + assert.NotEqual(t, nonExclusive, exclusive) +} + +func TestLease_SamePID_AlreadyLeased(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + + _, err = sm.Lease(1, false, cfg) + require.Error(t, err) + + sme, ok := err.(*SimulatorManagerError) + require.True(t, ok, "expected a *SimulatorManagerError, got %T", err) + assert.Equal(t, "alreadyLeased", sme.kind) + assert.Equal(t, udid, sme.udid) +} + +func TestRelease_NoLease(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + + err := sm.Release(1) + assert.Equal(t, ErrNoLease, err) +} + +func TestRelease_DeletesImmediatelyWhenIdleAfterIsZero(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + cfg := testConfig("iPhone") + + udid, err := sm.Lease(1, true, cfg) + require.NoError(t, err) + + require.NoError(t, sm.Release(1)) + + assert.Equal(t, []SimulatorUDID{udid}, control.deleteCalls) + assert.Equal(t, []SimulatorUDID{udid}, control.cleanTempFilesCalls) +} + +func TestRelease_SharedDevice_OnlyDeletesAfterLastRelease(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + cfg := testConfig("iPhone") + + udid, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + _, err = sm.Lease(2, false, cfg) + require.NoError(t, err) + + require.NoError(t, sm.Release(1)) + assert.Empty(t, control.deleteCalls, "device is still referenced by PID 2") + + require.NoError(t, sm.Release(2)) + assert.Equal(t, []SimulatorUDID{udid}, control.deleteCalls, "device should be deleted once unreferenced") +} + +func TestPendingDeletion_Resurrection(t *testing.T) { + control := newFakeSimulatorControl() + // A long idle timeout means the background deletion task has no chance to + // fire before this test's synchronous re-lease below. + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + require.NoError(t, sm.Release(1)) + assert.Empty(t, control.deleteCalls, "device should be pending deletion, not deleted yet") + + udid2, err := sm.Lease(2, false, cfg) + require.NoError(t, err) + + assert.Equal(t, udid1, udid2, "a new lease should resurrect the pending-deletion device") + assert.Len(t, control.cloneCalls, 1, "resurrection must not create a second device") +} + +func TestReuseSimulator_InvalidDeviceIsRecreated(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + udid1, err := sm.Lease(1, false, cfg) + require.NoError(t, err) + require.NoError(t, sm.Release(1)) + + // Simulate the device having gone corrupt/invalid in the interim. + control.ensureBootedErrs[udid1] = &ProcessError{ExitCode: 148} + + udid2, err := sm.Lease(2, false, cfg) + require.NoError(t, err) + + assert.NotEqual(t, udid1, udid2, "an invalid device must be replaced, not reused") + assert.Contains(t, control.deleteCalls, udid1, "the corrupt device should have been deleted") + assert.Len(t, control.cloneCalls, 2, "one original clone plus one replacement") +} + +func TestLease_ConcurrentNonExclusiveRequests_ShareOneInFlightClone(t *testing.T) { + control := newFakeSimulatorControl() + control.cloneGate = make(chan struct{}) + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + type leaseResult struct { + udid SimulatorUDID + err error + } + results := make(chan leaseResult, 2) + + go func() { + udid, err := sm.Lease(1, false, cfg) + results <- leaseResult{udid, err} + }() + + // Wait for the first request to actually start cloning (and block on the + // gate) before issuing the second, so this deterministically exercises + // the "join an in-flight clone" path rather than racing two fresh clones. + require.Eventually(t, func() bool { + control.mu.Lock() + defer control.mu.Unlock() + return len(control.cloneCalls) == 1 + }, 2*time.Second, 5*time.Millisecond) + + go func() { + udid, err := sm.Lease(2, false, cfg) + results <- leaseResult{udid, err} + }() + + // Give the second request a moment to reach the shared pendingCreation + // slot before unblocking the clone, then let it complete. + time.Sleep(50 * time.Millisecond) + close(control.cloneGate) + + first := <-results + second := <-results + require.NoError(t, first.err) + require.NoError(t, second.err) + + assert.Equal(t, first.udid, second.udid, "both leasers should get the same, singly-cloned device") + assert.Len(t, control.cloneCalls, 1, "only one clone should ever have been started") +} + +func TestLease_LeaserAlreadyExited(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, true, nil) + deadPID := spawnDeadPID(t) + + _, err := sm.Lease(deadPID, false, testConfig("iPhone")) + + assert.Equal(t, ErrLeaserExited, err) + assert.Empty(t, control.cloneCalls, "must not provision a device for a leaser that's already gone") +} + +func TestDeleteOnPIDExit_AutoReleasesWhenProcessDies(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, true, nil) + cmd, pid := spawnLiveProcess(t) + + udid, err := sm.Lease(pid, true, testConfig("iPhone")) + require.NoError(t, err) + + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + + require.Eventually(t, func() bool { + return sm.Release(pid) == ErrNoLease + }, 3*time.Second, 50*time.Millisecond, "manager should auto-release once the leasing process exits") + + assert.Contains(t, control.deleteCalls, udid) +} + +func TestLease_FailsAssertionWhenMoreThanOneRunningSimulatorShareTheSlot(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + // Simulate the exact failure mode the assertion exists to catch: two real + // booted devices ended up sharing what should be one slot's name. + name := cfg.CloneDeviceName(0) + control.runningOverride[name] = []SimCtlDevice{ + {Name: name, UDID: "udid-a", State: "Booted"}, + {Name: name, UDID: "udid-b", State: "Booted"}, + } + + _, err := sm.Lease(1, false, cfg) + + require.Error(t, err) + assert.Contains(t, err.Error(), "assertion failed") + assert.Contains(t, err.Error(), "udid-a") + assert.Contains(t, err.Error(), "udid-b") + assert.Equal(t, ErrNoLease, sm.Release(1), "a lease that failed the assertion must not have been recorded") + assert.Equal(t, 0, sm.LiveLeaseCount()) +} + +func TestLease_SucceedsWhenAtMostOneRunningSimulatorForTheSlot(t *testing.T) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 60, 60, false, nil) + cfg := testConfig("iPhone") + + name := cfg.CloneDeviceName(0) + control.runningOverride[name] = []SimCtlDevice{ + {Name: name, UDID: "udid-a", State: "Booted"}, + } + + udid, err := sm.Lease(1, false, cfg) + + require.NoError(t, err) + assert.NotEmpty(t, udid) +} + +func TestLease_IgnoresRunningSimulatorsListErrorRatherThanFailingTheLease(t *testing.T) { + control := &erroringRunningSimulatorsControl{fakeSimulatorControl: newFakeSimulatorControl()} + sm := newTestManager(control, 60, 60, false, nil) + + udid, err := sm.Lease(1, false, testConfig("iPhone")) + + require.NoError(t, err, "a failure to run the diagnostic check itself must not fail the lease") + assert.NotEmpty(t, udid) +} + +func TestLiveLeaseCount(t *testing.T) { + control := newFakeSimulatorControl() + // deleteOnPIDExit disabled so a lease for an already-dead PID isn't + // rejected or auto-released -- this test wants both a live and a dead + // lease to coexist in order to prove LiveLeaseCount filters correctly. + sm := newTestManager(control, 60, 60, false, nil) + + _, livePID := spawnLiveProcess(t) + deadPID := spawnDeadPID(t) + + _, err := sm.Lease(livePID, true, testConfig("iPhone-live")) + require.NoError(t, err) + _, err = sm.Lease(deadPID, true, testConfig("iPhone-dead")) + require.NoError(t, err) + + assert.Equal(t, 1, sm.LiveLeaseCount()) +} + +func TestRestoreLeases_AdoptsRunningLease_DropsExitedLease(t *testing.T) { + control := newFakeSimulatorControl() + deadPID := spawnDeadPID(t) + // The test process itself is guaranteed to still be running. + livePID := int32(os.Getpid()) + + store := &fakeLeaseStore{ + loaded: []PersistedLease{ + { + PID: livePID, + UDID: "udid-running", + Config: testConfig("running"), + Exclusive: true, + SlotIndex: 0, + }, + { + PID: deadPID, + UDID: "udid-exited", + Config: testConfig("exited"), + Exclusive: true, + SlotIndex: 0, + }, + }, + } + + sm := newTestManager(control, 60, 60, false, store) + sm.RestoreLeases() + + assert.NoError(t, sm.Release(livePID), "the lease for a still-running process should have been adopted") + assert.Equal(t, ErrNoLease, sm.Release(deadPID), "the lease for an exited process should have been dropped") +} + +func TestRestoreLeases_ConflictingExclusiveLeases_KeepsFirstOnly(t *testing.T) { + control := newFakeSimulatorControl() + _, pid1 := spawnLiveProcess(t) + _, pid2 := spawnLiveProcess(t) + + cfg := testConfig("iPhone") + store := &fakeLeaseStore{ + loaded: []PersistedLease{ + {PID: pid1, UDID: "udid-shared", Config: cfg, Exclusive: true, SlotIndex: 0}, + {PID: pid2, UDID: "udid-shared", Config: cfg, Exclusive: true, SlotIndex: 0}, + }, + } + + sm := newTestManager(control, 60, 60, false, store) + sm.RestoreLeases() + + firstErr := sm.Release(pid1) + secondErr := sm.Release(pid2) + + // Restoration walks the persisted list in order, so the first entry wins + // and the second is rejected as conflicting -- exactly one adopted. + adopted := 0 + if firstErr == nil { + adopted++ + } + if secondErr == nil { + adopted++ + } + assert.Equal(t, 1, adopted, "exactly one of the two conflicting leases should have been adopted") +} diff --git a/tools/simulator_manager/go/simulator_request_handler.go b/tools/simulator_manager/go/simulator_request_handler.go new file mode 100644 index 0000000..fd8ca3d --- /dev/null +++ b/tools/simulator_manager/go/simulator_request_handler.go @@ -0,0 +1,166 @@ +package main + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +type SimulatorManagerResponse struct { + Status int + Message string +} + +type SimulatorRequestHandler struct { + simulatorManager *SimulatorManager +} + +func NewSimulatorRequestHandler(simulatorManager *SimulatorManager) *SimulatorRequestHandler { + return &SimulatorRequestHandler{ + simulatorManager: simulatorManager, + } +} + +func (h *SimulatorRequestHandler) LiveLeaseCount() int { + return h.simulatorManager.LiveLeaseCount() +} + +func (h *SimulatorRequestHandler) HandleRequest(method string, path string, queryParams url.Values) SimulatorManagerResponse { + pathComponents := strings.Split(strings.Trim(path, "/"), "/") + if len(pathComponents) > 0 && pathComponents[0] == "simulator" { + pathComponents = pathComponents[1:] + } + + switch method { + case http.MethodPost: + return h.handlePost(pathComponents, queryParams) + case http.MethodDelete: + return h.handleDelete(pathComponents, queryParams) + default: + return SimulatorManagerResponse{ + Status: http.StatusMethodNotAllowed, + Message: fmt.Sprintf("Unsupported HTTP method: %s", method), + } + } +} + +func (h *SimulatorRequestHandler) handlePost(pathComponents []string, queryParams url.Values) SimulatorManagerResponse { + if len(pathComponents) < 1 || pathComponents[0] == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify ", + } + } + + leaser, err := strconv.ParseInt(pathComponents[0], 10, 32) + if err != nil { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Leaser PID must be an integer", + } + } + + exclusiveStr := queryParams.Get("exclusive") + if exclusiveStr == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'exclusive' query parameter", + } + } + exclusive := exclusiveStr == "1" + + deviceType := queryParams.Get("deviceType") + if deviceType == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'deviceType' query parameter", + } + } + + osParam := queryParams.Get("os") + if osParam == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'os' query parameter", + } + } + + version := queryParams.Get("version") + if version == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify 'version' query parameter", + } + } + + config := SimulatorConfig{ + DeviceType: deviceType, + OS: osParam, + Version: version, + } + + udid, err := h.simulatorManager.Lease(int32(leaser), exclusive, config) + if err != nil { + if err == ErrLeaserExited { + return SimulatorManagerResponse{ + Status: http.StatusGone, + Message: fmt.Sprintf("PID %d exited before its simulator was provisioned", leaser), + } + } + + if sme, ok := err.(*SimulatorManagerError); ok && sme.kind == "alreadyLeased" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: fmt.Sprintf("PID %d has already leased another simulator: %s", leaser, sme.udid), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusInternalServerError, + Message: fmt.Sprintf("Internal server error: %v", err), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusCreated, + Message: udid, + } +} + +func (h *SimulatorRequestHandler) handleDelete(pathComponents []string, queryParams url.Values) SimulatorManagerResponse { + if len(pathComponents) < 1 || pathComponents[0] == "" { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Must specify ", + } + } + + leaser, err := strconv.ParseInt(pathComponents[0], 10, 32) + if err != nil { + return SimulatorManagerResponse{ + Status: http.StatusBadRequest, + Message: "Leaser PID must be an integer", + } + } + + if err := h.simulatorManager.Release(int32(leaser)); err != nil { + if err == ErrNoLease { + return SimulatorManagerResponse{ + Status: http.StatusNotFound, + Message: fmt.Sprintf("PID %d doesn't have a simulator leased", leaser), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusInternalServerError, + Message: fmt.Sprintf("Internal server error: %v", err), + } + } + + return SimulatorManagerResponse{ + Status: http.StatusOK, + Message: "Success", + } +} diff --git a/tools/simulator_manager/go/simulator_request_handler_test.go b/tools/simulator_manager/go/simulator_request_handler_test.go new file mode 100644 index 0000000..b5de461 --- /dev/null +++ b/tools/simulator_manager/go/simulator_request_handler_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "net/http" + "net/url" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestRequestHandler() (*SimulatorRequestHandler, *fakeSimulatorControl) { + control := newFakeSimulatorControl() + sm := newTestManager(control, 0, 0, false, nil) + return NewSimulatorRequestHandler(sm), control +} + +func TestHandleRequest_UnsupportedMethod(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodGet, "/simulator/1", url.Values{}) + + assert.Equal(t, http.StatusMethodNotAllowed, resp.Status) +} + +func TestHandlePost_MissingPID(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodPost, "/simulator/", url.Values{}) + + assert.Equal(t, http.StatusBadRequest, resp.Status) + assert.Contains(t, resp.Message, "leaser PID") +} + +func TestHandlePost_NonIntegerPID(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodPost, "/simulator/not-a-pid", url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}}) + + assert.Equal(t, http.StatusBadRequest, resp.Status) +} + +func TestHandlePost_MissingQueryParams(t *testing.T) { + h, _ := newTestRequestHandler() + + cases := []struct { + name string + params url.Values + }{ + {"missing exclusive", url.Values{"deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}}}, + {"missing deviceType", url.Values{"exclusive": {"1"}, "os": {"iOS"}, "version": {"17.0"}}}, + {"missing os", url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "version": {"17.0"}}}, + {"missing version", url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}}}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resp := h.HandleRequest(http.MethodPost, "/simulator/1", c.params) + assert.Equal(t, http.StatusBadRequest, resp.Status) + }) + } +} + +func TestHandlePost_Success(t *testing.T) { + h, control := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodPost, "/simulator/1", url.Values{ + "exclusive": {"1"}, + "deviceType": {"iPhone"}, + "os": {"iOS"}, + "version": {"17.0"}, + }) + + require.Equal(t, http.StatusCreated, resp.Status) + assert.Equal(t, control.cloneCalls[0], "EXAMPLE_BAZEL_CLONE_iPhone_17.0_0") + assert.NotEmpty(t, resp.Message, "response body should carry the leased UDID") +} + +func TestHandlePost_NonExclusiveWhenNotSetTo1(t *testing.T) { + h, control := newTestRequestHandler() + + // Anything other than exactly "1" for `exclusive` should be treated as + // non-exclusive, per the query-param contract. + _ = h.HandleRequest(http.MethodPost, "/simulator/1", url.Values{ + "exclusive": {"0"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}, + }) + first := h.HandleRequest(http.MethodPost, "/simulator/2", url.Values{ + "exclusive": {"0"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}, + }) + + require.Equal(t, http.StatusCreated, first.Status) + assert.Len(t, control.cloneCalls, 1, "both non-exclusive leases should share the same device") +} + +func TestHandlePost_AlreadyLeased(t *testing.T) { + h, _ := newTestRequestHandler() + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + + first := h.HandleRequest(http.MethodPost, "/simulator/1", params) + require.Equal(t, http.StatusCreated, first.Status) + + second := h.HandleRequest(http.MethodPost, "/simulator/1", params) + assert.Equal(t, http.StatusBadRequest, second.Status) + assert.Contains(t, second.Message, "already leased") +} + +func TestHandleDelete_MissingPID(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodDelete, "/simulator/", url.Values{}) + + assert.Equal(t, http.StatusBadRequest, resp.Status) +} + +func TestHandleDelete_NoLease(t *testing.T) { + h, _ := newTestRequestHandler() + + resp := h.HandleRequest(http.MethodDelete, "/simulator/1", url.Values{}) + + assert.Equal(t, http.StatusNotFound, resp.Status) + assert.Contains(t, resp.Message, "doesn't have a simulator leased") +} + +func TestHandleDelete_Success(t *testing.T) { + h, _ := newTestRequestHandler() + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + + require.Equal(t, http.StatusCreated, h.HandleRequest(http.MethodPost, "/simulator/1", params).Status) + + resp := h.HandleRequest(http.MethodDelete, "/simulator/1", url.Values{}) + assert.Equal(t, http.StatusOK, resp.Status) +} + +func TestLiveLeaseCount_Passthrough(t *testing.T) { + h, _ := newTestRequestHandler() + params := url.Values{"exclusive": {"1"}, "deviceType": {"iPhone"}, "os": {"iOS"}, "version": {"17.0"}} + + assert.Equal(t, 0, h.LiveLeaseCount()) + + // This test process's own PID is guaranteed to be running. + pid := strconv.Itoa(os.Getpid()) + require.Equal(t, http.StatusCreated, h.HandleRequest(http.MethodPost, "/simulator/"+pid, params).Status) + + assert.Equal(t, 1, h.LiveLeaseCount()) +} diff --git a/tools/simulator_manager/start.sh b/tools/simulator_manager/start.sh index be2d11d..59bda59 100755 --- a/tools/simulator_manager/start.sh +++ b/tools/simulator_manager/start.sh @@ -204,7 +204,7 @@ if [[ -z "$version" ]]; then # --- end runfiles.bash initialization v3 --- simulator_manager="$( - rlocation _main/tools/simulator_manager/simulator_manager + rlocation _main/tools/simulator_manager/go/go_/go )" bazel_prepare_simulator="$( rlocation _main/tools/simulator_manager/prepare_simulator