Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 23 additions & 11 deletions apps/cli/src/commands/archive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import {
import {
acquireCheckpointPin,
parseCheckpointSelector,
maintenanceOperation,
resolveCheckpoint,
withMaintenanceLock,
withRestoredCheckpoint,
Expand Down Expand Up @@ -313,7 +314,8 @@ export const archiveCreate = Command.make("create", {
`\n`,
),
)
const result = yield* Effect.tryPromise({
const result = yield* maintenanceOperation({
operation: "archive.create",
try: () =>
createArchiveGeneration(
dataDir,
Expand Down Expand Up @@ -476,7 +478,8 @@ export const archiveRebuild = Command.make("rebuild", {
const dataDir = resolve(Option.getOrUndefined(a.dataDir) ?? defaultDataDir())
const archiveDir = resolve(Option.getOrUndefined(a.archiveDir) ?? defaultArchiveDir())
const signalName: ArchiveSignalName = a.signal
const entries = yield* Effect.tryPromise({
const entries = yield* maintenanceOperation({
operation: "archive.rebuild_catalog",
try: () => rebuildCatalogWithMaintenanceLock(dataDir, archiveDir, signalName, randomUUID()),
catch: (error) =>
new ArchiveError({
Expand Down Expand Up @@ -510,7 +513,8 @@ export const archiveReconcile = Command.make("reconcile", {
// entry point (blocker 2): dry-run returns the plan without mutating;
// apply acquires the maintenance lock, migrates any v2 intent, then
// reconciles — never racing create/GC planning or pointer/catalog repair.
const decision = yield* Effect.tryPromise({
const decision = yield* maintenanceOperation({
operation: "archive.reconcile",
try: () => runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: a.dryRun }),
catch: (error) =>
new ArchiveError({ message: error instanceof Error ? error.message : String(error) }),
Expand Down Expand Up @@ -564,7 +568,8 @@ export const archiveGc = Command.make("gc", {
})
}
const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot)
const result = yield* Effect.tryPromise({
const result = yield* maintenanceOperation({
operation: "archive.gc",
try: () => runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: a.keep, dryRun: a.dryRun }),
catch: (error) =>
new ArchiveError({ message: error instanceof Error ? error.message : String(error) }),
Expand Down Expand Up @@ -615,7 +620,8 @@ export const archiveExpire = Command.make("expire", {
if (!a.apply)
return yield* new ArchiveError({ message: "refusing archive expiration without --apply" })
const roots = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot)
yield* Effect.tryPromise({
yield* maintenanceOperation({
operation: "archive.expire_day",
try: () =>
expireArchiveDay({
dataDir: roots.dataDir,
Expand Down Expand Up @@ -648,7 +654,8 @@ export const archiveRetireLive = Command.make("retire-live", {
if (!a.apply)
return yield* new ArchiveError({ message: "refusing live retirement without --apply" })
const roots = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot)
yield* Effect.tryPromise({
yield* maintenanceOperation({
operation: "archive.retire_day",
try: () =>
retireLiveDay({
dataDir: roots.dataDir,
Expand Down Expand Up @@ -1248,7 +1255,8 @@ const runCalibrationMatrix = (
// ONE atomic bridge over the still-raw checkpoint session. The callback body
// stays raw on purpose: Effect must never be run from inside a callback
// handed to a promise-based module.
const session = yield* Effect.tryPromise({
const session = yield* maintenanceOperation({
operation: "archive.calibrate_open",
try: () =>
withMaintenanceLock(dataDir, operationId, async () => {
await reconcileCalibration(archiveDir, roots)
Expand Down Expand Up @@ -1308,7 +1316,8 @@ const runCalibrationMatrix = (
// Kept in the typed error channel rather than `orDie`d: a failed reconcile
// is an expected archive failure with a useful message, not a defect.
const closed = yield* Effect.exit(
Effect.tryPromise({
maintenanceOperation({
operation: "archive.calibrate_close",
try: () =>
withMaintenanceLock(dataDir, operationId, () => reconcileCalibration(archiveDir, roots)),
catch: (error) => new ArchiveError({ message: errorMessage(error) }),
Expand Down Expand Up @@ -1614,7 +1623,8 @@ export const archiveCalibrateRun = Command.make("calibrate-run", {
})
const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot)
const checkpointSelector = Option.getOrUndefined(a.checkpointId) ?? "current"
yield* Effect.tryPromise({
yield* maintenanceOperation({
operation: "archive.calibrate_sample",
try: () =>
runCalibrateSample(a, dataDir, archiveDir, scratchRoot, checkpointSelector, rangeDate),
catch: (error) =>
Expand Down Expand Up @@ -1659,7 +1669,8 @@ export const archiveCalibrateSession = Command.make("calibrate-session", {
const checkpointSelector = Option.getOrUndefined(a.checkpointId) ?? "current"
const roots = { dataDir, archiveDir, scratchRoot }
if (action === "close") {
yield* Effect.tryPromise({
yield* maintenanceOperation({
operation: "archive.calibrate_session_close",
try: () =>
withMaintenanceLock(dataDir, randomUUID(), () =>
reconcileCalibration(archiveDir, roots),
Expand All @@ -1678,7 +1689,8 @@ export const archiveCalibrateSession = Command.make("calibrate-session", {
const pinPurpose = calibrationPinPurpose(operationId)
const scratchSubdir = derivedScratchSubdir(operationId)
const sampleDir = derivedSampleDir(archiveDir, operationId)
const result = yield* Effect.tryPromise({
const result = yield* maintenanceOperation({
operation: "archive.calibrate_session_open",
try: () =>
withMaintenanceLock(dataDir, operationId, async () => {
await reconcileCalibration(archiveDir, roots)
Expand Down
10 changes: 7 additions & 3 deletions apps/cli/src/commands/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
runLocalStoreMigration,
type MigrationPlan,
} from "../server/local-store-migrations"
import { maintenanceOperation } from "../server/checkpoints"
import { readMarker } from "../server/store-version"

class SchemaCommandError extends Schema.TaggedError<SchemaCommandError>()("@maple/cli/SchemaCommandError", {
Expand Down Expand Up @@ -132,7 +133,8 @@ export const schemaMigrate = Command.make("migrate", {
Command.withHandler(
Effect.fnUntraced(function* (args) {
const dataDir = resolvedDataDir(args.dataDir)
const preview = yield* Effect.tryPromise({
const preview = yield* maintenanceOperation({
operation: "schema.migrate_preview",
try: () => runLocalStoreMigration({ dataDir, dryRun: true }),
catch: (error) =>
new SchemaCommandError({
Expand All @@ -154,7 +156,8 @@ export const schemaMigrate = Command.make("migrate", {
return
}
}
const result = yield* Effect.tryPromise({
const result = yield* maintenanceOperation({
operation: "schema.migrate_apply",
try: () =>
runLocalStoreMigration({
dataDir,
Expand Down Expand Up @@ -198,7 +201,8 @@ export const schemaAbandon = Command.make("abandon", {
)
return
}
const quarantine = yield* Effect.tryPromise({
const quarantine = yield* maintenanceOperation({
operation: "schema.abandon",
try: () => abandonLocalStoreMigrationPreservingSource(dataDir),
catch: (error) =>
new SchemaCommandError({
Expand Down
42 changes: 42 additions & 0 deletions apps/cli/src/server/checkpoints.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1367,6 +1367,48 @@ export const withMaintenanceLock = async <A>(
}
}

/**
* The Effect boundary for promise-land work that takes the maintenance lock
* INTERNALLY — archive create/gc/reconcile/catalog, retention, and local-store
* migrations. Use it instead of a bare `Effect.tryPromise` at every such call.
*
* `Effect.tryPromise` is interruptible, and interruption ABANDONS the promise:
* the runtime marks the async resumed, aborts its signal and unwinds the fiber
* without waiting (see `callbackOptions` in effect's internal/effect.ts). Under
* `BunRuntime.runMain` a Ctrl-C therefore tore the process down while
* `withMaintenanceLock` was still mid-operation, so its `finally` never ran and
* `<dataDir>.maintenance.lock` survived with a plausible owner record. The next
* run only recovered because {@link acquireMaintenance} quarantines a provably
* dead PID — recovery by luck, one PID reuse away from a hard failure.
*
* `Effect.uninterruptible` fixes it without touching those modules: an interrupt
* arriving here is recorded on the fiber and NOT delivered, the fiber stays
* parked until the promise settles, the lock's `finally` releases, and the
* recorded interrupt fires as soon as interruptibility is restored. The work was
* never abortable — the promise ran to completion either way. All this changes
* is that Effect now waits for it instead of walking away mid-write.
*
* The cost is deliberate: Ctrl-C during a long operation is honoured when that
* operation finishes, not immediately. That is the correct trade for the only
* writer of a lock the next process must trust. A caller who truly cannot wait
* still has SIGKILL, which is the crash case the on-disk journals already
* reconcile. To make Ctrl-C prompt again, the promise bodies below would have to
* observe the `AbortSignal` that `try` already receives — until they do, an
* interruptible boundary would only delete the lock out from under work that
* keeps running.
*/
export const maintenanceOperation = <A, E>(options: {
readonly operation: string
readonly try: () => Promise<A>
readonly catch: (error: unknown) => E
}): Effect.Effect<A, E> =>
Effect.tryPromise({ try: options.try, catch: options.catch }).pipe(
Effect.uninterruptible,
Effect.withSpan("CheckpointService.maintenanceOperation", {
attributes: { "maple.checkpoint.maintenance_operation": options.operation },
}),
)

export const retireCheckpointIfEligible = async (
dataDir: string,
checkpointId: CheckpointId | null,
Expand Down
128 changes: 128 additions & 0 deletions apps/cli/test/maintenance-operation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// The maintenance lock's release lives in a `finally` inside promise land, so
// whether it runs is decided entirely by the Effect boundary above it. These
// tests pin that boundary from both sides: the bare `Effect.tryPromise` shape
// that used to wrap every archive/migration entry point leaks the lock on
// interruption, and `maintenanceOperation` does not.
import { describe, it } from "@effect/vitest"
import { Effect, Exit, Fiber } from "effect"
import { ok, strictEqual } from "node:assert"
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"
import { tmpdir } from "node:os"
import { join, resolve } from "node:path"
import { maintenanceOperation, withMaintenanceLock } from "../src/server/checkpoints"

// Deliberately recomputed from the on-disk contract rather than imported: this
// pins the sibling path a *different* process must find and reconcile.
const lockPathOf = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock`

const withTempDataDir = async (run: (dataDir: string) => Promise<void>): Promise<void> => {
const root = mkdtempSync(join(tmpdir(), "maple-maintenance-boundary-"))
const dataDir = join(root, "data")
mkdirSync(dataDir, { recursive: true, mode: 0o700 })
try {
await run(dataDir)
} finally {
rmSync(root, { recursive: true, force: true })
}
}

/** A locked operation that parks until released, so a test can interrupt it
* at a known point: strictly after the lock is taken, strictly before the
* `finally` that releases it. */
const parkedOperation = (dataDir: string) => {
let signalEntered!: () => void
let signalFinish!: () => void
const entered = new Promise<void>((r) => (signalEntered = r))
const finish = new Promise<void>((r) => (signalFinish = r))
const state = { completed: false }
const body = () =>
withMaintenanceLock(dataDir, crypto.randomUUID(), async () => {
signalEntered()
await finish
state.completed = true
})
return { body, entered, release: signalFinish, state }
}

describe("maintenance lock Effect boundary", () => {
it("bare Effect.tryPromise abandons the operation and leaks the lock", async () => {
await withTempDataDir(async (dataDir) => {
const op = parkedOperation(dataDir)
const fiber = Effect.runFork(Effect.tryPromise({ try: op.body, catch: (error) => error }))
await op.entered
ok(existsSync(lockPathOf(dataDir)), "precondition: the lock is held")

// Exactly what `BunRuntime.runMain`'s SIGINT handler does.
fiber.interruptUnsafe()
const exit = await Effect.runPromise(Fiber.await(fiber))

// The fiber is gone while the promise is still parked mid-operation.
ok(Exit.hasInterrupts(exit), "the fiber was interrupted")
strictEqual(op.state.completed, false, "the operation never finished")
ok(
existsSync(lockPathOf(dataDir)),
"THE BUG: the lock survives because the promise's `finally` never ran",
)

// Let the abandoned promise finish so the temp dir can be removed.
op.release()
await new Promise((r) => setTimeout(r, 50))
})
})

it("maintenanceOperation waits for the operation, so the lock is always released", async () => {
await withTempDataDir(async (dataDir) => {
const op = parkedOperation(dataDir)
const fiber = Effect.runFork(
maintenanceOperation({ operation: "test.parked", try: op.body, catch: (error) => error }),
)
await op.entered
ok(existsSync(lockPathOf(dataDir)), "precondition: the lock is held")

// Interrupt while the operation holds the lock, the same way
// `BunRuntime.runMain`'s SIGINT handler does. Nothing may unwind yet:
// the interrupt is recorded on the fiber, not delivered.
const interrupting = Effect.runPromise(Fiber.await(fiber))
fiber.interruptUnsafe()
await new Promise((r) => setTimeout(r, 50))
strictEqual(op.state.completed, false, "still parked — the interrupt did not abandon it")
ok(existsSync(lockPathOf(dataDir)), "still holding the lock it took")

op.release()
const exit = await interrupting

strictEqual(op.state.completed, true, "the operation ran to completion")
ok(Exit.hasInterrupts(exit), "the deferred interrupt is still delivered afterwards")
ok(!existsSync(lockPathOf(dataDir)), "THE FIX: the lock was released")
})
})

it("releases the lock on the ordinary success and failure paths too", async () => {
await withTempDataDir(async (dataDir) => {
const okExit = await Effect.runPromise(
maintenanceOperation({
operation: "test.success",
try: () => withMaintenanceLock(dataDir, crypto.randomUUID(), async () => 7),
catch: (error) => error,
}),
)
strictEqual(okExit, 7)
ok(!existsSync(lockPathOf(dataDir)), "released after success")

const failed = await Effect.runPromise(
Effect.exit(
maintenanceOperation({
operation: "test.failure",
try: () =>
withMaintenanceLock(dataDir, crypto.randomUUID(), async () => {
throw new Error("boom")
}),
catch: (error) => (error instanceof Error ? error.message : String(error)),
}),
),
)
strictEqual(Exit.isFailure(failed), true)
ok(!existsSync(lockPathOf(dataDir)), "released after failure")
})
})
})
Loading
Loading