From 0943c24a84f82471d30590b5c85ca75bad1af849 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Thu, 18 Jun 2026 23:26:33 +0000 Subject: [PATCH] feat(environments): archive grace period + Restore environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Archiving the last live thread in a managed environment no longer destroys its worktree immediately. The environment stays `retiring` for a configurable grace window (default 10s via MANAGED_ENVIRONMENT_RETIRE_GRACE_MS), during which: - un-archiving revives it in place (retire.cancelled) — a lossless undo that preserves the worktree and uncommitted work, and - an "Undo" toast on archive offers the same one click. After the window the worktree is reclaimed by the existing recovery sweep (the grace gate lives in advanceEnvironmentCleanup and is durable across restart via the row's updatedAt clock; no in-memory timer). The gate applies only when a revivable archived thread remains, so a deleted thread's orphaned environment is still cleaned up immediately. A new "Restore environment" action (POST /threads/:id/restore-environment) reprovisions a fresh environment for a thread whose environment is gone. The daemon's createWorktree now checks out an existing branch in place instead of `-B`-resetting it, so committed work (which survives `git worktree remove`) is recovered; the thread is re-seeded into idle without an automatic turn. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../banner/ThreadPromptContextBanner.test.tsx | 48 ++ .../banner/ThreadPromptContextBanner.tsx | 42 ++ .../hooks/mutations/thread-state-mutations.ts | 38 ++ apps/app/src/lib/api.ts | 6 + .../thread-detail/ThreadDetailPromptArea.tsx | 20 +- apps/server/src/constants.ts | 9 + apps/server/src/routes/threads/actions.ts | 27 +- .../environment-cleanup-internal.ts | 24 + .../src/services/system/periodic-sweeps.ts | 40 +- .../threads/thread-environment-restore.ts | 168 ++++++ .../threads/thread-provisioning-context.ts | 7 + .../thread-provisioning-environment.ts | 18 +- apps/server/src/start-server.ts | 2 + apps/server/src/types.ts | 7 + apps/server/test/helpers/test-app.ts | 2 + ...blic-thread-environment-decoupling.test.ts | 12 +- ...naged-environment-cleanup-recovery.test.ts | 107 +++- .../test/system/bb-app-managed-config.test.ts | 1 + packages/db/src/data/index.ts | 1 + packages/db/src/data/sweeps.ts | 7 + packages/db/src/data/threads.ts | 32 + packages/host-workspace/src/provisioning.ts | 35 +- packages/server-contract/src/public-api.ts | 6 + ...onment-restore-and-archive-grace-period.md | 561 ++++++++++++++++++ .../environment-isolation.test.ts | 79 +++ tests/integration/helpers/api.ts | 10 + tests/integration/helpers/harness.ts | 5 + 27 files changed, 1247 insertions(+), 67 deletions(-) create mode 100644 apps/server/src/services/threads/thread-environment-restore.ts create mode 100644 plans/environment-restore-and-archive-grace-period.md diff --git a/apps/app/src/components/promptbox/banner/ThreadPromptContextBanner.test.tsx b/apps/app/src/components/promptbox/banner/ThreadPromptContextBanner.test.tsx index d9452ce10..5264e3ddc 100644 --- a/apps/app/src/components/promptbox/banner/ThreadPromptContextBanner.test.tsx +++ b/apps/app/src/components/promptbox/banner/ThreadPromptContextBanner.test.tsx @@ -105,6 +105,54 @@ describe("ThreadPromptContextBanner", () => { expect(markup).not.toContain("Provision"); }); + it("renders an enabled Restore action once the environment is destroyed", () => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain("Restore environment"); + expect(markup).toContain(" { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain("Cleaning up..."); + expect(markup).toContain('disabled=""'); + expect(markup).not.toContain("Restore environment"); + }); + it("labels a standalone pull request without non-actionable attention text", () => { const markup = renderToStaticMarkup( ; + /** + * Restores the thread's environment by reprovisioning a fresh workspace on the + * thread's branch (recovering committed work). Enabled once the old workspace + * is fully gone (`destroyed`); while `destroying` the action shows a disabled + * "Cleaning up…" state. Omitted when restore isn't available (e.g. unmanaged + * environments). + */ + onRestore?: () => void; + restorePending?: boolean; } /** @@ -418,6 +427,33 @@ function PullRequestReadyTextAction({ ); } +function EnvironmentRestoreTextAction({ + status, + isPending, + onRestore, +}: { + status: Extract; + isPending?: boolean; + onRestore: () => void; +}) { + const cleaningUp = status === "destroying"; + const label = cleaningUp + ? "Cleaning up..." + : isPending + ? "Restoring..." + : "Restore environment"; + return ( + + ); +} + const PULL_REQUEST_MERGE_ACTIONS: readonly { method: PullRequestMergeMethod; label: string; @@ -692,6 +728,12 @@ export function ThreadPromptContextBanner({ isPending={archivedSection.unarchivePending} onUnarchive={archivedSection.onUnarchive} /> + ) : environmentGoneSection?.onRestore ? ( + ) : null } parentThreadSection={parentThreadSection} diff --git a/apps/app/src/hooks/mutations/thread-state-mutations.ts b/apps/app/src/hooks/mutations/thread-state-mutations.ts index 37c0990c3..0f54661f4 100644 --- a/apps/app/src/hooks/mutations/thread-state-mutations.ts +++ b/apps/app/src/hooks/mutations/thread-state-mutations.ts @@ -6,6 +6,7 @@ import type { UpdateThreadRequest, } from "@bb/server-contract"; import * as api from "@/lib/api"; +import { appToast } from "@/components/ui/app-toast"; import type { LifecycleErrorOperation } from "@/lib/lifecycle-errors"; import { applyReorderPinnedThreadResult, @@ -38,6 +39,14 @@ interface ThreadMutationRequest { id: string; } +/** + * How long the "Thread archived — Undo" toast stays up. Matches the server's + * archive grace window (`MANAGED_ENVIRONMENT_RETIRE_GRACE_MS`), within which an + * Undo revives the environment losslessly (its worktree has not been destroyed + * yet). After it elapses the durable Unarchive in the read-only banner remains. + */ +const ARCHIVE_UNDO_TOAST_DURATION_MS = 10_000; + type UpdateThreadMutationRequest = ThreadMutationRequest & UpdateThreadRequest; type ReorderPinnedThreadMutationRequest = ThreadMutationRequest & ReorderPinnedThreadRequest; @@ -216,6 +225,22 @@ export function useArchiveThread() { transaction: context, }); }, + onSuccess: (_data, { id }) => { + // Offer a quick, lossless Undo while the environment is still inside its + // grace window: un-archiving revives a retiring environment in place + // (retire.cancelled), so the worktree and uncommitted work are preserved. + appToast.message("Thread archived", { + action: { + label: "Undo", + onClick: () => { + void api.unarchiveThread(id).then(() => { + settleThreadListMembershipMutation({ queryClient, threadId: id }); + }); + }, + }, + duration: ARCHIVE_UNDO_TOAST_DURATION_MS, + }); + }, onSettled: (_data, _error, variables) => { settleThreadListMembershipMutation({ queryClient, @@ -282,6 +307,19 @@ export function useUnarchiveThread() { }); } +export function useRestoreThreadEnvironment() { + return useMutation({ + meta: { + errorMessage: "Failed to restore environment.", + }, + mutationFn: ({ id }: ThreadMutationRequest) => + api.restoreThreadEnvironment(id), + // The server reprovisions a fresh environment and re-seeds the thread; the + // resulting thread/environment changes arrive over the realtime channel, so + // no optimistic cache mutation is needed here. + }); +} + export function useDeleteThread() { const queryClient = useQueryClient(); diff --git a/apps/app/src/lib/api.ts b/apps/app/src/lib/api.ts index baf1a30a0..9c431418c 100644 --- a/apps/app/src/lib/api.ts +++ b/apps/app/src/lib/api.ts @@ -1264,6 +1264,12 @@ export async function unarchiveThread(id: string): Promise { ); } +export async function restoreThreadEnvironment(id: string): Promise { + await requestVoid( + apiClient.threads[":id"]["restore-environment"].$post({ param: { id } }), + ); +} + export async function deleteThread( id: string, opts: DeleteThreadRequest, diff --git a/apps/app/src/views/thread-detail/ThreadDetailPromptArea.tsx b/apps/app/src/views/thread-detail/ThreadDetailPromptArea.tsx index 614b670c4..70e1a461c 100644 --- a/apps/app/src/views/thread-detail/ThreadDetailPromptArea.tsx +++ b/apps/app/src/views/thread-detail/ThreadDetailPromptArea.tsx @@ -54,7 +54,10 @@ import { useSendThreadQueuedMessage, useStopThread, } from "@/hooks/mutations/thread-runtime-mutations"; -import { useUnarchiveThread } from "@/hooks/mutations/thread-state-mutations"; +import { + useRestoreThreadEnvironment, + useUnarchiveThread, +} from "@/hooks/mutations/thread-state-mutations"; import { getLatestPendingInteraction, useThreadQueuedMessages, @@ -277,6 +280,7 @@ export function ThreadDetailPromptArea({ const reorderQueuedMessage = useReorderThreadQueuedMessage(); const stopThread = useStopThread(); const unarchiveThread = useUnarchiveThread(); + const restoreEnvironment = useRestoreThreadEnvironment(); const uploadPromptAttachment = useUploadPromptAttachment(); // The personal project isn't a meaningful label in the footer, so skip it. const projectName = useProjectDisplayName( @@ -798,6 +802,12 @@ export function ThreadDetailPromptArea({ const handleUnarchiveCurrentThread = useCallback(() => { unarchiveThread.mutate({ id: thread.id }); }, [thread.id, unarchiveThread]); + const isRestoreEnvironmentPending = + restoreEnvironment.isPending && + restoreEnvironment.variables?.id === thread.id; + const handleRestoreEnvironment = useCallback(() => { + restoreEnvironment.mutate({ id: thread.id }); + }, [thread.id, restoreEnvironment]); const attachmentsConfig = useMemo( () => ({ @@ -1017,7 +1027,11 @@ export function ThreadDetailPromptArea({ environmentGoneSection={ environmentGoneStatus === null ? null - : { status: environmentGoneStatus } + : { + status: environmentGoneStatus, + onRestore: handleRestoreEnvironment, + restorePending: isRestoreEnvironmentPending, + } } parentThreadSection={parentThreadSection} childThreadsSection={childThreadsSection} @@ -1070,9 +1084,11 @@ export function ThreadDetailPromptArea({ handleSendQueuedImmediately, handleToggleBannerSection, handleUnarchiveCurrentThread, + handleRestoreEnvironment, environmentGoneStatus, isFollowUpSubmitting, isUnarchiveCurrentThreadPending, + isRestoreEnvironmentPending, isQueueMutationPending, goal, isGoalExpanded, diff --git a/apps/server/src/constants.ts b/apps/server/src/constants.ts index 24858ef65..8a7a1c2c9 100644 --- a/apps/server/src/constants.ts +++ b/apps/server/src/constants.ts @@ -3,6 +3,15 @@ export const HEARTBEAT_INTERVAL_MS = 5_000; export const LEASE_TIMEOUT_MS = 30_000; export const DAEMON_DISCONNECT_GRACE_MS = 5_000; export const DAEMON_ACTIVE_WORK_DISCONNECT_GRACE_MS = LEASE_TIMEOUT_MS; +/** + * Grace window after the last live thread in a managed environment is archived + * before its worktree is destroyed. The environment stays `retiring` (revivable + * via unarchive → `retire.cancelled`, worktree intact) for this long so an + * accidental archive can be undone losslessly. Surfaced as the archive toast's + * "Undo" duration. The destroy is gated on the environment's `updatedAt` (the + * retire-requested time), so the window is durable across restart. + */ +export const MANAGED_ENVIRONMENT_RETIRE_GRACE_MS = 10_000; export const WORKSPACE_DIFF_MAX_DIFF_BYTES = 2 * 1024 * 1024; export const WORKSPACE_DIFF_MAX_FILE_LIST_BYTES = 256 * 1024; diff --git a/apps/server/src/routes/threads/actions.ts b/apps/server/src/routes/threads/actions.ts index 8d903250c..3125af119 100644 --- a/apps/server/src/routes/threads/actions.ts +++ b/apps/server/src/routes/threads/actions.ts @@ -31,6 +31,7 @@ import { requestEnvironmentCleanupAdvance, wouldCleanupEnvironment, } from "../../services/environments/environment-cleanup-internal.js"; +import { applyLoggedEnvironmentLifecycleEvent } from "../../services/environments/lifecycle-outcome.js"; import { requirePublicThread } from "../../services/lib/entity-lookup.js"; import { validatePromptAttachmentReferences } from "../../services/projects/attachments.js"; import { @@ -57,6 +58,7 @@ import { archiveThreadAndChildren, archiveThreadWithLifecycleEffects, } from "../../services/threads/thread-archive.js"; +import { restoreThreadEnvironment } from "../../services/threads/thread-environment-restore.js"; import { requireThreadCommandEnvironment, requireThreadHostCommandEnvironment, @@ -376,10 +378,14 @@ export function registerThreadActionRoutes(app: Hono, deps: AppDeps): void { }); }); - // Un-archive is a pure record op: it clears archivedAt and nothing else. It - // deliberately does not touch the environment lifecycle; cleanup is monotonic - // and never cancelled, and a thread whose environment is gone surfaces a - // read-only "environment is gone" banner instead of resurrecting it. + // Un-archive clears archivedAt. When the thread's managed environment is still + // inside its archive grace window (`retiring`), un-archiving revives it via the + // existing `retire.cancelled` event so the intact worktree is restored — the + // lossless undo of an accidental archive. If the grace window already elapsed + // and the environment was destroyed, `retire.cancelled` is a no-op (illegal + // from destroying/destroyed) and the thread surfaces the read-only "environment + // is gone" banner — use Restore to reprovision — instead of resurrecting the + // terminal row. post(routes.unarchive, (context) => { const thread = requirePublicThread(deps.db, context.req.param("id")); const providerThreadId = getLastProviderThreadId(deps, thread.id); @@ -387,6 +393,12 @@ export function registerThreadActionRoutes(app: Hono, deps: AppDeps): void { const environment = thread.environmentId ? getEnvironment(deps.db, thread.environmentId) : null; + if (environment?.status === "retiring") { + applyLoggedEnvironmentLifecycleEvent(deps, { + environmentId: environment.id, + event: { type: "retire.cancelled" }, + }); + } if (providerThreadId && environment) { dispatchThreadUnarchiveCommand(deps, { environment, @@ -397,6 +409,13 @@ export function registerThreadActionRoutes(app: Hono, deps: AppDeps): void { return context.json({ ok: true }); }); + post(routes.restoreEnvironment, async (context) => { + await restoreThreadEnvironment(deps, { + threadId: context.req.param("id"), + }); + return context.json({ ok: true }); + }); + post(routes.read, (context) => { requirePublicThread(deps.db, context.req.param("id")); const thread = updateThread(deps.db, deps.hub, context.req.param("id"), { diff --git a/apps/server/src/services/environments/environment-cleanup-internal.ts b/apps/server/src/services/environments/environment-cleanup-internal.ts index 6f8b41633..47166077f 100644 --- a/apps/server/src/services/environments/environment-cleanup-internal.ts +++ b/apps/server/src/services/environments/environment-cleanup-internal.ts @@ -5,6 +5,7 @@ import { getEnvironment, getActiveSession, hasPendingThreadShutdownInEnvironment, + hasRevivableArchivedThreadInEnvironment, listLiveThreadsInEnvironment, type DbNotifier, type DbQueryConnection, @@ -398,6 +399,29 @@ async function advanceEnvironmentCleanup( return; } + // Archive grace window: a freshly retired managed worktree stays revivable + // (worktree intact, undoable via unarchive → retire.cancelled) for the + // configured grace window so an accidental archive can be undone losslessly. + // `updatedAt` is the retire-requested time — a retiring row is only + // ever mutated by the events that exit retiring, so it is a faithful clock that + // survives restart. Scope: only the path-bearing `retiring` case waits; a + // pathless env (handled above) has no worktree to lose, and `error` is failed + // cleanup rather than an accidental-archive brick. The window applies only when + // the environment still has a revivable archived thread — an env left retiring + // by a deleted/tombstoned thread has nothing to unarchive, so it is cleaned up + // immediately rather than lingering. + if ( + refreshedEnvironment.status === "retiring" && + refreshedEnvironment.path !== null && + Date.now() - refreshedEnvironment.updatedAt < + deps.config.managedEnvironmentRetireGraceMs && + hasRevivableArchivedThreadInEnvironment(deps.db, { + environmentId: refreshedEnvironment.id, + }) + ) { + return; + } + if ( countLiveThreadsInEnvironment(deps.db, { environmentId: refreshedEnvironment.id, diff --git a/apps/server/src/services/system/periodic-sweeps.ts b/apps/server/src/services/system/periodic-sweeps.ts index dff3f895d..1eed0f2d5 100644 --- a/apps/server/src/services/system/periodic-sweeps.ts +++ b/apps/server/src/services/system/periodic-sweeps.ts @@ -170,14 +170,11 @@ export async function runPeriodicSweepJobs( } } -async function evaluateManagedEnvironmentArchiveCleanupCandidates( +async function advanceRetiringManagedEnvironments( deps: LoggedPendingInteractionWorkSessionDeps, - orphanedDestroyUpdatedBefore: number, ): Promise { - recoverOrphanedEnvironmentDestroyRequests(deps, { - updatedBefore: orphanedDestroyUpdatedBefore, - }); - + // The advance enforces the archive grace window per environment, so this sweeps + // every retiring candidate each tick and lets in-grace ones short-circuit. const environmentsToClean = sweepManagedEnvironments(deps.db); if (environmentsToClean.length === 0) { return { @@ -374,23 +371,24 @@ export async function runManagedEnvironmentArchiveCleanupRecoverySweep( deps: LoggedPendingInteractionWorkSessionDeps, now: number, ): Promise { + // Orphaned-destroy recovery only touches environments stuck `destroying` for + // longer than the daemon command timeout, so it stays throttled — it is a rare + // backstop, not the steady-state driver. if ( - now - lastManagedEnvironmentArchiveCleanupRecoveryAt < + now - lastManagedEnvironmentArchiveCleanupRecoveryAt >= MANAGED_ENVIRONMENT_ARCHIVE_CLEANUP_RECOVERY_INTERVAL_MS ) { - return; - } - - const result = await evaluateManagedEnvironmentArchiveCleanupCandidates( - deps, - now - ORPHANED_ENVIRONMENT_DESTROY_RECOVERY_DELAY_MS, - ); - if ( - result.candidates > 0 && - result.hostUnavailableDeferrals < result.candidates - ) { + recoverOrphanedEnvironmentDestroyRequests(deps, { + updatedBefore: now - ORPHANED_ENVIRONMENT_DESTROY_RECOVERY_DELAY_MS, + }); lastManagedEnvironmentArchiveCleanupRecoveryAt = now; } + + // Grace-gated destroy runs every tick: a retiring managed worktree is reclaimed + // ~one sweep tick after its archive grace window elapses. The advance enforces + // the window against the durable `updatedAt` clock (no in-memory timer), so it + // survives restart. + await advanceRetiringManagedEnvironments(deps); } export async function runProjectDeletionSweep( @@ -607,7 +605,11 @@ export async function runStartupRecoverySweep( ): Promise { await runEnvironmentProvisioningSweep(deps); await runThreadLifecycleSweep(deps); - await evaluateManagedEnvironmentArchiveCleanupCandidates(deps, Date.now()); + // On restart any lingering `destroying` row is presumed orphaned, so recover + // them regardless of age; advance retiring environments (the advance enforces + // the grace window, destroying only those whose window already elapsed). + recoverOrphanedEnvironmentDestroyRequests(deps, { updatedBefore: Date.now() }); + await advanceRetiringManagedEnvironments(deps); } export async function runPeriodicSweeps( diff --git a/apps/server/src/services/threads/thread-environment-restore.ts b/apps/server/src/services/threads/thread-environment-restore.ts new file mode 100644 index 000000000..c11ca29e6 --- /dev/null +++ b/apps/server/src/services/threads/thread-environment-restore.ts @@ -0,0 +1,168 @@ +import { getEnvironment, getThread, unarchiveThread } from "@bb/db"; +import type { Environment, Thread } from "@bb/domain"; +import type { AppDeps } from "../../types.js"; +import { ApiError } from "../../errors.js"; +import { applyLoggedEnvironmentLifecycleEvent } from "../environments/lifecycle-outcome.js"; +import { applyLoggedThreadLifecycleEvent } from "./lifecycle-outcome.js"; +import { buildExecutionOptions } from "./thread-commands.js"; +import { + requireSourceForHost, + storedBaseBranchNameToSpec, +} from "./thread-create-helpers.js"; +import { createClientTurnRequestId } from "./thread-events.js"; +import { + createMetadataPendingContext, + type ThreadProvisionEnvironmentIntent, +} from "./thread-provisioning-context.js"; +import { advanceThreadProvisioning } from "./thread-provisioning.js"; +import { saveThreadProvisionContext } from "./thread-provisioning-environment.js"; + +interface RestoreThreadEnvironmentArgs { + threadId: string; +} + +/** + * Builds the provisioning intent for a fresh environment that replaces a gone + * one. Managed worktrees re-checkout the destroyed environment's exact branch so + * its committed work is recovered (the daemon checks the branch out in place); + * personal workspaces are recreated empty. Unmanaged workspaces are user-owned + * and cannot be recreated. + */ +function restoreEnvironmentIntent( + deps: AppDeps, + thread: Thread, + destroyed: Environment, +): ThreadProvisionEnvironmentIntent { + if (!destroyed.managed || destroyed.workspaceProvisionType === "unmanaged") { + throw new ApiError( + 409, + "invalid_request", + "This environment is unmanaged and can't be restored automatically.", + ); + } + if (destroyed.workspaceProvisionType === "personal") { + return { + type: "direct-personal", + hostId: destroyed.hostId, + workspaceProvisionType: "personal", + }; + } + const source = requireSourceForHost(deps, thread.projectId, destroyed.hostId); + return { + type: "direct-managed", + hostId: destroyed.hostId, + sourcePath: source.path, + baseBranch: storedBaseBranchNameToSpec(destroyed.baseBranch), + workspaceProvisionType: "managed-worktree", + ...(destroyed.branchName ? { branchName: destroyed.branchName } : {}), + }; +} + +/** + * Restores a working environment for a thread whose environment is gone. + * + * - `retiring` (still inside the archive grace window): revives the environment + * in place via `retire.cancelled`; the worktree is intact, so nothing is lost. + * - `destroying`: the tear-down RPC is in flight; reject with 409 so the client + * retries once the row reaches `destroyed`. + * - `destroyed` (or the row was already pruned to null): mints a fresh + * environment and reprovisions it. For a managed worktree the destroyed + * branch is re-checked-out (recovering committed work); uncommitted work was + * lost when the worktree was torn down. The thread is re-seeded into `idle` + * (no automatic turn) so the user continues from their next message. + * - anything else (`ready`/`provisioning`/`error`): the environment is not gone, + * so this is a no-op. + * + * An archived thread is un-archived first so Restore is one click from the + * read-only banner. + */ +export async function restoreThreadEnvironment( + deps: AppDeps, + args: RestoreThreadEnvironmentArgs, +): Promise { + const thread = getThread(deps.db, args.threadId); + if (!thread || thread.deletedAt !== null) { + throw new ApiError(404, "thread_not_found", "Thread not found"); + } + + if (thread.archivedAt !== null) { + unarchiveThread(deps.db, deps.hub, thread.id); + } + const currentThread = getThread(deps.db, thread.id) ?? thread; + + const environment = currentThread.environmentId + ? getEnvironment(deps.db, currentThread.environmentId) + : null; + + if (environment?.status === "retiring") { + // Lossless undo: still inside the grace window, the worktree is intact. + applyLoggedEnvironmentLifecycleEvent(deps, { + environmentId: environment.id, + event: { type: "retire.cancelled" }, + }); + return; + } + if (environment?.status === "destroying") { + throw new ApiError( + 409, + "invalid_request", + "Environment is still being torn down; try again shortly.", + ); + } + if (environment && environment.status !== "destroyed") { + // ready / provisioning / error — the environment is not gone; nothing to do. + return; + } + + if (!environment) { + // The destroyed row was pruned (no branch/host metadata survives), so there + // is nothing to reprovision against. Surface it instead of guessing. + throw new ApiError( + 409, + "environment_not_ready", + "This environment was cleaned up and can no longer be restored. Start a new thread.", + ); + } + + const intent = restoreEnvironmentIntent(deps, currentThread, environment); + + // Move the thread back to `starting` so the provisioning advance runs. Legal + // from idle and error (the statuses a thread lands in when its environment is + // destroyed); a no-op otherwise, which then short-circuits the advance. + const preparing = applyLoggedThreadLifecycleEvent(deps, { + threadId: currentThread.id, + event: { type: "run.preparing" }, + }); + if (!preparing.applied) { + throw new ApiError( + 409, + "thread_not_writable", + "This thread can't be restored from its current state.", + ); + } + deps.hub.notifyThread(currentThread.id, ["status-changed"]); + + const execution = await buildExecutionOptions( + deps, + {}, + { threadId: currentThread.id }, + "client/turn/requested", + ); + // seedWithoutRun re-seeds the thread into `idle` once the workspace is ready + // without dispatching a turn — Restore brings the environment back, it does + // not run the agent. + const context = createMetadataPendingContext({ + clientRequestId: createClientTurnRequestId(), + environmentIntent: intent, + execution, + fork: null, + input: [], + titleProvided: true, + seedWithoutRun: true, + }); + saveThreadProvisionContext({ threadId: currentThread.id, context }); + await advanceThreadProvisioning(deps, { + context, + threadId: currentThread.id, + }); +} diff --git a/apps/server/src/services/threads/thread-provisioning-context.ts b/apps/server/src/services/threads/thread-provisioning-context.ts index 8000d97dc..52ff8f8e1 100644 --- a/apps/server/src/services/threads/thread-provisioning-context.ts +++ b/apps/server/src/services/threads/thread-provisioning-context.ts @@ -35,6 +35,13 @@ const directManagedIntentSchema = z.object({ sourcePath: z.string().min(1), baseBranch: baseBranchSpecSchema, workspaceProvisionType: z.literal("managed-worktree"), + /** + * Explicit branch to provision. Set when restoring an environment so the fresh + * worktree re-checks-out the destroyed environment's exact branch (recovering + * its committed work) instead of deriving a new `bb/-` name. + * Omitted for a brand-new thread, which derives the branch from its title slug. + */ + branchName: z.string().min(1).optional(), }); const directPersonalIntentSchema = z.object({ diff --git a/apps/server/src/services/threads/thread-provisioning-environment.ts b/apps/server/src/services/threads/thread-provisioning-environment.ts index 66b1382ef..384545cdf 100644 --- a/apps/server/src/services/threads/thread-provisioning-environment.ts +++ b/apps/server/src/services/threads/thread-provisioning-environment.ts @@ -226,6 +226,11 @@ interface ManagedEnvironmentPlanArgs { hostId: string; sourcePath: string; baseBranch: BaseBranchSpec; + /** + * Explicit branch to provision (environment restore). When omitted, the branch + * is derived from the thread's title slug for a brand-new thread. + */ + branchName?: string; thread: Thread; workspaceProvisionType: "managed-worktree"; } @@ -824,10 +829,12 @@ function buildManagedEnvironmentPlan( }, buildRequest: ({ context, environment }) => { const command = buildEnvironmentProvisionCommand({ - branchName: buildManagedBranchName({ - branchSlug: context.request.branchSlug, - threadId: args.thread.id, - }), + branchName: + args.branchName ?? + buildManagedBranchName({ + branchSlug: context.request.branchSlug, + threadId: args.thread.id, + }), baseBranch: args.baseBranch, environmentId: environment.id, hostId: args.hostId, @@ -903,6 +910,9 @@ async function resolveEnvironmentCreationPlan( hostId: args.intent.hostId, sourcePath: args.intent.sourcePath, baseBranch: args.intent.baseBranch, + ...(args.intent.branchName + ? { branchName: args.intent.branchName } + : {}), thread: args.thread, workspaceProvisionType: args.intent.workspaceProvisionType, }); diff --git a/apps/server/src/start-server.ts b/apps/server/src/start-server.ts index bcd09d30b..7c4b8b22f 100644 --- a/apps/server/src/start-server.ts +++ b/apps/server/src/start-server.ts @@ -21,6 +21,7 @@ import { createTelemetryService } from "./services/system/telemetry.js"; import { TerminalSessionLifecycle } from "./services/terminals/terminal-session-lifecycle.js"; import { resolveThreadStorageRootPath } from "./services/threads/thread-storage.js"; import { createLifecycleDedupers } from "./lifecycle-dedupers.js"; +import { MANAGED_ENVIRONMENT_RETIRE_GRACE_MS } from "./constants.js"; import type { ServerRuntimeConfig } from "./types.js"; import { NotificationHub } from "./ws/hub.js"; import { WatchInterestCoordinator } from "./ws/watch-interests.js"; @@ -59,6 +60,7 @@ export async function runServer(serverConfig: ServerConfig): Promise { hostDaemonPort: serverConfig.BB_HOST_DAEMON_PORT, inferenceModel: serverConfig.BB_INFERENCE, isDevelopment: !isProduction, + managedEnvironmentRetireGraceMs: MANAGED_ENVIRONMENT_RETIRE_GRACE_MS, openAiApiKey: serverConfig.OPENAI_API_KEY, serverPort: serverConfig.BB_SERVER_PORT, threadStorageRootPath, diff --git a/apps/server/src/types.ts b/apps/server/src/types.ts index b06db8503..a66d2cc05 100644 --- a/apps/server/src/types.ts +++ b/apps/server/src/types.ts @@ -30,6 +30,13 @@ export interface ServerRuntimeConfig { hostDaemonPort: number; inferenceModel: string; isDevelopment: boolean; + /** + * Grace window (ms) after the last live thread in a managed environment is + * archived before its worktree is destroyed, during which an accidental + * archive can be undone losslessly. Defaults to + * {@link MANAGED_ENVIRONMENT_RETIRE_GRACE_MS}; set to 0 to destroy immediately. + */ + managedEnvironmentRetireGraceMs: number; openAiApiKey: string; serverPort: number; threadStorageRootPath: string; diff --git a/apps/server/test/helpers/test-app.ts b/apps/server/test/helpers/test-app.ts index b1f8659cb..56ce02275 100644 --- a/apps/server/test/helpers/test-app.ts +++ b/apps/server/test/helpers/test-app.ts @@ -19,6 +19,7 @@ import { TerminalSessionLifecycle } from "../../src/services/terminals/terminal- import { resolveThreadStorageRootPath } from "../../src/services/threads/thread-storage.js"; import { createLifecycleDedupers } from "../../src/lifecycle-dedupers.js"; import type { ServerAppDeps, ServerRuntimeConfig } from "../../src/types.js"; +import { MANAGED_ENVIRONMENT_RETIRE_GRACE_MS } from "../../src/constants.js"; import type { NotificationHub } from "../../src/ws/hub.js"; import { NotificationHub as NotificationHubImpl } from "../../src/ws/hub.js"; import { WatchInterestCoordinator } from "../../src/ws/watch-interests.js"; @@ -132,6 +133,7 @@ export async function createTestAppHarness( hostDaemonPort: 3001, inferenceModel: "test/mock-model", isDevelopment: true, + managedEnvironmentRetireGraceMs: MANAGED_ENVIRONMENT_RETIRE_GRACE_MS, openAiApiKey: "test-openai-key", serverPort: 3334, threadStorageRootPath: resolveThreadStorageRootPath({ diff --git a/apps/server/test/public/public-thread-environment-decoupling.test.ts b/apps/server/test/public/public-thread-environment-decoupling.test.ts index cc88a8ae9..c79cc57d3 100644 --- a/apps/server/test/public/public-thread-environment-decoupling.test.ts +++ b/apps/server/test/public/public-thread-environment-decoupling.test.ts @@ -21,10 +21,10 @@ import { withTestHarness } from "../helpers/test-app.js"; * of reprovisioning. */ describe("thread environment decoupling (B*)", () => { - it("un-archives without touching a retiring environment", async () => { + it("revives a retiring environment on un-archive (lossless undo of an accidental archive)", async () => { await withTestHarness(async (harness) => { const { host } = seedHostSession(harness.deps, { - id: "host-unarchive-pure", + id: "host-unarchive-revive", }); const { project } = seedProjectWithSource(harness.deps, { hostId: host.id, @@ -32,6 +32,7 @@ describe("thread environment decoupling (B*)", () => { const environment = seedEnvironment(harness.deps, { hostId: host.id, managed: true, + path: "/tmp/unarchive-revive", projectId: project.id, status: "retiring", workspaceProvisionType: "managed-worktree", @@ -49,11 +50,12 @@ describe("thread environment decoupling (B*)", () => { ); expect(response.status).toBe(200); - // The thread is un-archived (pure record op)... expect(getThread(harness.db, thread.id)?.archivedAt).toBeNull(); - // ...and the retiring environment lifecycle is left untouched. + // The retiring environment is revived to ready via retire.cancelled: its + // worktree was never destroyed during the grace window, so the undo is + // lossless. expect(getEnvironment(harness.db, environment.id)).toMatchObject({ - status: "retiring", + status: "ready", }); }); }); diff --git a/apps/server/test/services/managed-environment-cleanup-recovery.test.ts b/apps/server/test/services/managed-environment-cleanup-recovery.test.ts index 8f29736f6..386975587 100644 --- a/apps/server/test/services/managed-environment-cleanup-recovery.test.ts +++ b/apps/server/test/services/managed-environment-cleanup-recovery.test.ts @@ -1,15 +1,22 @@ import { eq } from "drizzle-orm"; -import { createEnvironment, environments, getEnvironment } from "@bb/db"; +import { + archiveThread, + createEnvironment, + createThread, + environments, + getEnvironment, + markThreadDeleted, +} from "@bb/db"; import { describe, expect, it, vi } from "vitest"; import { runEnvironmentCleanupAdvance, settleEnvironmentDestroyCommandResult, } from "../../src/services/environments/environment-cleanup-internal.js"; import { - MANAGED_ENVIRONMENT_ARCHIVE_CLEANUP_RECOVERY_INTERVAL_MS, runManagedEnvironmentArchiveCleanupRecoverySweep, runStartupRecoverySweep, } from "../../src/services/system/periodic-sweeps.js"; +import { MANAGED_ENVIRONMENT_RETIRE_GRACE_MS } from "../../src/constants.js"; import { listQueuedEnvironmentCommands, } from "../helpers/commands.js"; @@ -411,61 +418,111 @@ describe("managed environment cleanup recovery sweep", () => { }); }); - it("throttles recovery without arming the throttle on empty sweeps", async () => { + it("defers a retiring environment's destroy until its grace window elapses while a revivable archived thread remains, then destroys it on the next sweep regardless of the recovery throttle", async () => { await withTestHarness(async (harness) => { const { host } = seedHostSession(harness.deps); const { project } = seedProjectWithSource(harness.deps, { hostId: host.id, }); + // First sweep arms the (15-minute) orphaned-destroy recovery throttle. await runManagedEnvironmentArchiveCleanupRecoverySweep( harness.deps, SWEEP_START_MS, ); - const firstEnvironment = createEnvironment(harness.db, harness.hub, { + const environment = createEnvironment(harness.db, harness.hub, { hostId: host.id, + isGitRepo: false, managed: true, + path: "/tmp/grace-window-environment", projectId: project.id, status: "retiring", workspaceProvisionType: "managed-worktree", }); + // An archived (not deleted) thread keeps the environment revivable via + // unarchive, so the grace window applies. + const thread = createThread(harness.db, harness.hub, { + projectId: project.id, + environmentId: environment.id, + providerId: "codex", + status: "idle", + }); + archiveThread(harness.db, harness.hub, thread.id); + // Freshly retired → still inside the grace window → not destroyed yet. await runManagedEnvironmentArchiveCleanupRecoverySweep( harness.deps, SWEEP_START_MS + 1, ); + expect(getEnvironment(harness.db, environment.id)?.status).toBe( + "retiring", + ); + expect( + listQueuedEnvironmentCommands( + harness, + "environment.destroy", + environment.id, + ), + ).toHaveLength(0); - expect(getEnvironment(harness.db, firstEnvironment.id)?.status).toBe( - "destroyed", + // Past the grace window → destroyed on the very next sweep, even though the + // recovery throttle window has not elapsed: the grace-gated retiring sweep + // is not throttled, only the orphaned-destroy recovery is. + harness.db + .update(environments) + .set({ updatedAt: Date.now() - MANAGED_ENVIRONMENT_RETIRE_GRACE_MS - 1 }) + .where(eq(environments.id, environment.id)) + .run(); + await runManagedEnvironmentArchiveCleanupRecoverySweep( + harness.deps, + SWEEP_START_MS + 2, + ); + expect(getEnvironment(harness.db, environment.id)?.status).toBe( + "destroying", ); + expect( + listQueuedEnvironmentCommands( + harness, + "environment.destroy", + environment.id, + ), + ).toHaveLength(1); + }); + }); + + it("destroys a retiring environment immediately when its only thread is deleted (nothing to unarchive)", async () => { + await withTestHarness(async (harness) => { + const { host } = seedHostSession(harness.deps); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); - const throttledEnvironment = createEnvironment(harness.db, harness.hub, { + const environment = createEnvironment(harness.db, harness.hub, { hostId: host.id, + isGitRepo: false, managed: true, + path: "/tmp/deleted-thread-environment", projectId: project.id, status: "retiring", workspaceProvisionType: "managed-worktree", }); + const thread = createThread(harness.db, harness.hub, { + projectId: project.id, + environmentId: environment.id, + providerId: "codex", + status: "idle", + }); + markThreadDeleted(harness.db, harness.hub, { threadId: thread.id }); - await runManagedEnvironmentArchiveCleanupRecoverySweep( - harness.deps, - SWEEP_START_MS + 10_000, - ); - - expect(getEnvironment(harness.db, throttledEnvironment.id)?.status).toBe( - "retiring", - ); - - await runManagedEnvironmentArchiveCleanupRecoverySweep( - harness.deps, - SWEEP_START_MS + - 1 + - MANAGED_ENVIRONMENT_ARCHIVE_CLEANUP_RECOVERY_INTERVAL_MS, - ); - - expect(getEnvironment(harness.db, throttledEnvironment.id)?.status).toBe( - "destroyed", + // Freshly retired, but the only thread is deleted (not archived): there is + // nothing to unarchive, so the grace window does not apply and cleanup + // destroys the orphaned workspace right away. + await runEnvironmentCleanupAdvance(harness.deps, { + environmentId: environment.id, + }); + expect(getEnvironment(harness.db, environment.id)?.status).toBe( + "destroying", ); }); }); diff --git a/apps/server/test/system/bb-app-managed-config.test.ts b/apps/server/test/system/bb-app-managed-config.test.ts index 43d00d192..951cd86d1 100644 --- a/apps/server/test/system/bb-app-managed-config.test.ts +++ b/apps/server/test/system/bb-app-managed-config.test.ts @@ -58,6 +58,7 @@ function createRuntimeConfig(): ServerRuntimeConfig { hostDaemonPort: 38887, inferenceModel: "openai/gpt-4o-mini", isDevelopment: false, + managedEnvironmentRetireGraceMs: 10_000, openAiApiKey: "ambient-openai-key", serverPort: 38886, threadStorageRootPath: "/tmp/bb-test/thread-storage", diff --git a/packages/db/src/data/index.ts b/packages/db/src/data/index.ts index 59c688566..e0dffa2a9 100644 --- a/packages/db/src/data/index.ts +++ b/packages/db/src/data/index.ts @@ -101,6 +101,7 @@ export { setThreadExecutionOverride, hasNonTerminalThreadInEnvironment, hasPendingThreadShutdownInEnvironment, + hasRevivableArchivedThreadInEnvironment, listHostThreadIds, listActiveVisiblePinnedThreadRoots, listActiveVisiblePinnedThreadRootsWithPendingInteractionState, diff --git a/packages/db/src/data/sweeps.ts b/packages/db/src/data/sweeps.ts index 76a247ed1..19ccb406e 100644 --- a/packages/db/src/data/sweeps.ts +++ b/packages/db/src/data/sweeps.ts @@ -408,6 +408,13 @@ export function sweepExpiredLeases( * Sweep retiring managed environments with zero non-archived threads. * Returns the list of environment records that are candidates for cleanup. * The caller decides what to do (e.g., queue destroy commands). + * + * The archive grace window (delay a retiring environment's destroy so an + * accidental archive can be undone) is enforced by the server in + * `advanceEnvironmentCleanup`, not here: this sweep returns a candidate as soon + * as it is retiring with no live threads, and the advance defers the actual + * destroy until the grace window elapses. Keeping the grace check in one place + * (the advance) avoids splitting the policy across the db query. */ export function sweepManagedEnvironments(db: DbConnection) { const rows = db diff --git a/packages/db/src/data/threads.ts b/packages/db/src/data/threads.ts index c8007a9bf..0e29f40ad 100644 --- a/packages/db/src/data/threads.ts +++ b/packages/db/src/data/threads.ts @@ -526,6 +526,10 @@ export interface ListLiveThreadsInEnvironmentArgs { environmentId: string; } +export interface HasRevivableArchivedThreadInEnvironmentArgs { + environmentId: string; +} + export interface CountNonDeletedAssignedChildThreadsArgs { parentThreadId: string; } @@ -1131,6 +1135,34 @@ export function countLiveThreadsInEnvironment( return liveThreadCount?.count ?? 0; } +/** + * Whether the environment has a thread that is archived but not deleted — i.e. a + * thread that could still be unarchived. The archive grace window (which delays + * destroying a retiring environment's worktree so an accidental archive can be + * undone) only applies when such a revivable thread exists; an environment left + * retiring solely by deleted/tombstoned threads has nothing to undo and is + * cleaned up immediately. + */ +export function hasRevivableArchivedThreadInEnvironment( + db: ThreadWriteConnection, + args: HasRevivableArchivedThreadInEnvironmentArgs, +): boolean { + const row = db + .select({ id: threads.id }) + .from(threads) + .where( + and( + eq(threads.environmentId, args.environmentId), + isNotNull(threads.archivedAt), + isNull(threads.deletedAt), + ), + ) + .limit(1) + .get(); + + return row !== undefined; +} + export function listLiveThreadsInEnvironment( db: ThreadWriteConnection, args: ListLiveThreadsInEnvironmentArgs, diff --git a/packages/host-workspace/src/provisioning.ts b/packages/host-workspace/src/provisioning.ts index ba4581962..27700a467 100644 --- a/packages/host-workspace/src/provisioning.ts +++ b/packages/host-workspace/src/provisioning.ts @@ -232,6 +232,18 @@ function throwIfProvisionAborted(signal: AbortSignal | undefined): void { } } +async function localBranchExists(args: { + sourcePath: string; + branchName: string; + signal: AbortSignal | undefined; +}): Promise { + const result = await runGit( + ["show-ref", "--verify", "--quiet", `refs/heads/${args.branchName}`], + { cwd: args.sourcePath, allowFailure: true, signal: args.signal }, + ); + return result.exitCode === 0; +} + export async function createWorktree( args: CreateWorkspaceArgs, ): Promise<{ path: string }> { @@ -252,14 +264,21 @@ export async function createWorktree( `Cannot resolve default branch for source: ${args.sourcePath}`, ); } - const gitArgs = [ - "worktree", - "add", - "-B", - args.branchName, - args.targetPath, - baseBranch, - ]; + // If the branch already exists in the source repo, check it out as-is so its + // commits are preserved. This matters for restoring an environment whose + // worktree was removed: `git worktree remove` deletes the worktree but never + // the branch, so committed work survives in the shared object store, and + // `-B ` would reset the branch back to the base and + // orphan it. A brand-new thread's branch never pre-exists, so it still takes + // the create-from-base path. + const branchAlreadyExists = await localBranchExists({ + sourcePath: args.sourcePath, + branchName: args.branchName, + signal: args.signal, + }); + const gitArgs = branchAlreadyExists + ? ["worktree", "add", args.targetPath, args.branchName] + : ["worktree", "add", "-B", args.branchName, args.targetPath, baseBranch]; const worktreeStartedAt = Date.now(); emitStep({ onProgress: args.onProgress, diff --git a/packages/server-contract/src/public-api.ts b/packages/server-contract/src/public-api.ts index 74129784d..dec733a0f 100644 --- a/packages/server-contract/src/public-api.ts +++ b/packages/server-contract/src/public-api.ts @@ -644,6 +644,12 @@ export const publicApiRoutes = { request: noRequest(), response: jsonResponse<{ ok: true }>(), }), + restoreEnvironment: defineRoute({ + path: "/threads/:id/restore-environment", + method: "post", + request: noRequest(), + response: jsonResponse<{ ok: true }>(), + }), read: defineRoute({ path: "/threads/:id/read", method: "post", diff --git a/plans/environment-restore-and-archive-grace-period.md b/plans/environment-restore-and-archive-grace-period.md new file mode 100644 index 000000000..f7f9c5652 --- /dev/null +++ b/plans/environment-restore-and-archive-grace-period.md @@ -0,0 +1,561 @@ +# Environment restore + archive grace period + +Status: Phases 1–4 implemented 2026-06-17. (T3 checkpointing remains out of scope — +see verdict below.) + +**Confirm dialog decision:** the planned Restore confirm modal was dropped. Restore +is non-destructive — by the time the environment is `destroyed`, uncommitted work is +already gone, so a "you'll lose work" modal would misattribute the loss. Restore is a +direct one-click action like Unarchive (the toast/banner copy carries expectations). + +**Decision (2026-06-17):** short **10-second** grace window, surfaced as an **"Undo" +in the archive toast** — the immediate-misclick catch. After it, the worktree is +destroyed and **Restore** (reprovision from the branch) is the recovery path. The +grace window is deliberately *not* a "come back later and unarchive to get +uncommitted work" window; its only job is the 10s lossless undo. + +## Problem + +Two related gaps around a thread losing its environment: + +1. **No "Restore environment" action.** When a thread's environment is + `destroying`/`destroyed`, the prompt banner shows a dead-end read-only row + ("Environment is no longer available") with no way forward. The thread is + bricked. +2. **Accidental archive instantly bricks a thread.** Archiving the *last* live + thread in a managed environment immediately tears down its git worktree, with + no undo and no grace period. A misclick destroys uncommitted work. + +This plan fixes the brick with a durable **grace period** (the cheap, high-value +win that makes accidental archive losslessly reversible), adds a **Restore +environment** action for the genuinely-gone case, and records a verdict on +whether T3's per-turn git-checkpoint scheme is relevant here (it is not, for this +change — see [T3 checkpointing](#t3-checkpointing-evaluation)). + +## Current behavior (verified in code) + +### Environment lifecycle is an event-sourced state machine + +`packages/domain/src/environment-lifecycle.ts` defines `ENVIRONMENT_LIFECYCLE` +(statuses × events → next status). Relevant transitions: + +- `ready` —`retire.requested`→ `retiring` +- `retiring` —`retire.cancelled`→ `ready` ← **an undo edge already exists** +- `retiring` —`destroy.started`→ `destroying` (stamps `destroyAttemptId`) +- `destroying` —`destroy.completed`→ `destroyed` (terminal) +- `destroyed`: `{}` — **terminal**. Doc comment: *"a thread that needs an + environment again gets a fresh record, it never resurrects the destroyed row + (future 'Provision environment')."* + +The durable state is the row's `(status, destroyAttemptId, updatedAt)` +(`packages/db/src/data/environments.ts`); there is no separate event log, but +`applyEnvironmentLifecycleEventRecord` (environments.ts:380) **stamps +`updatedAt` on every transition**, and the only legal exits from `retiring` are +`retire.cancelled` and `destroy.started`. So a `retiring` row's `updatedAt` is a +faithful "retire requested at" clock for the whole window — no new column is +strictly required for the grace period. (Caveat: `updateEnvironmentMetadata`, +environments.ts:235, is reachable via `PATCH /environments/:id` with no status +guard and bumps `updatedAt`; a metadata edit during the window *extends* it — +delay, never brick. See [open questions](#open-questions).) + +### The brick path + +`archiveThreadWithLifecycleEffects` → on the *last* live thread, +`archiveEnvironmentThreads` / `archiveThreadAndChildren` +(`apps/server/src/services/threads/thread-archive.ts:93-105,147-156`) call +`requestEnvironmentCleanup` (→ `retire.requested` → `retiring`) **and** +`requestEnvironmentCleanupAdvance`. The advance is deferred only by +`deferAfterResponse`/`setImmediate` (environment-cleanup-internal.ts:467-475), +then `advanceEnvironmentCleanup` dispatches `environment.destroy` in the same +turn if a daemon is connected. **There is no time delay.** + +Shared environments are already safe: cleanup only fires when +`wouldCleanupEnvironment` / `countLiveThreadsInEnvironment` report **zero** live +threads (excluding the one being archived). Archiving 1-of-N never retires. + +### The existing revive + sweep machinery we will reuse + +- `retire.cancelled` is already fired when work resumes on a retiring env + (`queued-messages.ts:89-94`, `thread-turn-dispatch.ts:103-108`). Unarchive does + **not** fire it — `routes/threads/actions.ts:380` even documents the opposite + ("does not touch the environment lifecycle"). That missing wire is the undo + bug. +- `sweepManagedEnvironments` (`packages/db/src/data/sweeps.ts:412`) already + returns all `retiring` managed envs with zero live threads. The periodic + recovery sweep (`periodic-sweeps.ts`) runs `runEnvironmentCleanupAdvance` over + them. Today it self-throttles to 15 min as a *fallback* (the archive path is + the primary, immediate driver). + +### Restore for a gone env does not exist yet + +`dispatchTurnDuringReprovision` (`thread-turn-dispatch.ts:96-202`) revives a +`retiring` env in place (`retire.cancelled`, worktree intact) and reprovisions an +`error` env on the **same** row — but for `destroying`/`destroyed` it +**throws** `throwThreadEnvironmentUnavailable` (lines 111-118). There is no +fresh-environment-for-an-existing-thread path. `createProvisioningEnvironment` +(`thread-provisioning-environment.ts`) is hard-gated on a brand-new +`status:'starting'` thread with an in-memory provision context, so it cannot be +called as-is for an idle/stopped thread. Restore-from-destroyed is genuinely +net-new code. + +### What is recoverable after a destroy + +Managed worktrees are `git worktree add` against the **shared** source-repo object +store. `environment.destroy`'s `destroyFn` is `removeWorktree` +(`packages/host-workspace/src/provisioning.ts:489`), which runs `git worktree +remove --force` + `fs.rm` on the worktree dir — it does **not** run `git branch +-D`. So the branch ref `bb/-` **and all its commits survive in the +source repo** after a destroy. **Committed** work is therefore recoverable; +**uncommitted / untracked** changes (only ever in the worktree dir) are gone. This +is *why the grace period is the primary fix* and Restore is best-effort secondary +recovery of committed work. + +**Caveat that shapes Phase 3:** the managed provision command resets the branch. +`createWorktree` (`provisioning.ts:255`) unconditionally runs +`git worktree add -B `, and `-B` **resets** +an existing branch to `` — which would orphan exactly the commits we +want back. So Restore cannot simply re-run the normal provision command; the +daemon must **check out the existing branch** when it already exists (see Phase 3). +A reassuring corollary: because *every* branch lives in the shared source repo and +survives the destroy, a restored worktree can `git checkout` any of them — so even +if we restore onto a slightly stale branch, the user's other branches are still +right there. + +## Design + +### Phase 1 — Durable grace period (fixes the brick) — ✅ IMPLEMENTED + +**As shipped** (refined from the original sketch below): +- Grace duration is a server config value `managedEnvironmentRetireGraceMs` + (`ServerRuntimeConfig`), defaulting to the `MANAGED_ENVIRONMENT_RETIRE_GRACE_MS` + constant (10s) in production and server-unit tests, and **0** in the integration + harness (which has no periodic sweep / time control, so it keeps immediate + destroy-on-archive). This also answers open question #1 (grace *is* configurable). +- The authoritative grace gate lives in `advanceEnvironmentCleanup` + (`environment-cleanup-internal.ts`), after the refreshed-status recheck and + before the `destroy.started` claim. It defers destroy while + `status === "retiring"`, `path !== null`, + `Date.now() - updatedAt < config.managedEnvironmentRetireGraceMs`, **and** the env + still has a revivable archived thread. +- **Revivability condition** (new, important): grace applies only when + `hasRevivableArchivedThreadInEnvironment` is true — the env has a thread that is + archived but not deleted, i.e. something to unarchive. An env left retiring by a + *deleted/tombstoned* thread has nothing to undo, so it is cleaned up immediately + (this is what keeps the deleted-thread reprovision path correct). +- No query-level grace predicate on `sweepManagedEnvironments` (the gate is the + single source of truth; at a 10s window an env is "in grace" for ~one tick, so the + per-tick no-op advance is negligible). No in-memory timer. +- The periodic recovery sweep was split: `recoverOrphanedEnvironmentDestroyRequests` + stays 15-min throttled; the retiring sweep+advance (`advanceRetiringManagedEnvironments`) + runs every ~10s tick, so a retired env is reclaimed roughly one tick after its + grace window. Durable across restart via `runStartupRecoverySweep`. +- Tests: `managed-environment-cleanup-recovery.test.ts` gained a grace-deferral test + (archived thread → retiring within window, destroyed after) and a deleted-thread + immediate-destroy test; existing destroy/throttle tests updated. Full `@bb/server` + (662) + `@bb/db` (305) + at-risk integration suites pass. + +Original sketch (kept for reference): + +Keep `retire.requested` on archive (env enters `retiring`, revivable), but **gate +the destroy** so it cannot fire for a short window. State lives in the DB row, so +it survives restart, daemon offline, and concurrency for free. The window is the +lossless-undo budget behind the archive toast — short by design. + +- Add `MANAGED_ENVIRONMENT_RETIRE_GRACE_MS = 10_000` (10s, matching the toast Undo + duration) near `MANAGED_ENVIRONMENT_ARCHIVE_CLEANUP_RECOVERY_INTERVAL_MS` + (`periodic-sweeps.ts:60`). +- **Grace gate in `advanceEnvironmentCleanup`** (environment-cleanup-internal.ts). + Place it **after the refreshed-status re-read/recheck block (lines 392–399) and + before the `destroy.started` claim (line 418)**, keyed on the *refreshed* row to + avoid a TOCTOU: + ```ts + if ( + refreshedEnvironment.status === "retiring" && + refreshedEnvironment.path !== null && + Date.now() - refreshedEnvironment.updatedAt < MANAGED_ENVIRONMENT_RETIRE_GRACE_MS + ) return; + ``` + Scope to `status === "retiring"` only: the pathless branch (no worktree to lose) + and the `error` → destroy path (already-failed cleanup, not an accidental-archive + brick) bypass the gate. +- The existing immediate `requestEnvironmentCleanupAdvance` archive calls now hit + the gate and **no-op** (they fire within `setImmediate`, well under 10s). Leave + them (lowest churn) — they're harmless — but they no longer drive the destroy. +- **Prompt cleanup at ~10s:** on archive, schedule a single deferred re-advance at + `+MANAGED_ENVIRONMENT_RETIRE_GRACE_MS` (e.g. via the existing deferral helper / + a 10s timer) so the destroy fires promptly once the window closes. This is an + *optimization for cleanup latency only*; correctness does not depend on it (the + gate is the floor, the sweep below is the durable backstop), so losing the timer + on restart just means the sweep reclaims a few seconds later. +- **Durable backstop = the recovery sweep.** Split the 15-min self-throttle (a real + refactor, not a one-liner — `evaluateManagedEnvironmentArchiveCleanupCandidates` + bundles three calls and `runStartupRecoverySweep` also invokes it): + - Keep the 15-min throttle around `recoverOrphanedEnvironmentDestroyRequests` + only (its original purpose). + - Let `sweepManagedEnvironments` + `runEnvironmentCleanupAdvance` run on the + outer sweep cadence so a retiring env left behind by a missed timer (e.g. + restart mid-window) is still reclaimed shortly after the window. + - Add an `AND updatedAt <= now - grace` predicate to `sweepManagedEnvironments` + (`sweeps.ts:412`) so in-grace rows are skipped at the query level (no per-tick + no-op advance for envs still inside their 10s window). + +Restart / offline / shared-env behavior (all free from the durable row): +- **Restart mid-grace:** `runStartupRecoverySweep` re-evaluates against the + persisted `updatedAt`; destroy fires after the window, or immediately if it + already elapsed during downtime (grace honored). +- **Daemon offline at expiry:** `workspaceCanBeDestroyedNow` already returns false + with no connected daemon (environment-cleanup-internal.ts:130); the sweep + pre-defers host-unavailable candidates. Offline strictly *extends* the safe + window; the deadline is "earliest destroy", never "guaranteed destroy at". +- **Shared env:** grace only starts when the last live thread is archived; the + `destroy.started` CAS re-asserts `NOT EXISTS live/stopping threads` + (environments.ts:401-414), so a thread created during the sweep load→write race + still blocks destroy. +- **`error` envs** are intentionally out of the grace window (gate keys on + `retiring`; `sweepManagedEnvironments` returns only `retiring` rows). Confirm + `error` → destroy is unaffected. + +### Phase 2 — Undo via unarchive (loss-free revive), server — ✅ IMPLEMENTED + +Shipped: the unarchive route (`routes/threads/actions.ts`) now fires +`retire.cancelled` when the thread's environment is `retiring`, reviving it to +`ready` with the worktree intact. If the env is already `destroying`/`destroyed` +the event is a no-op (the thread shows the env-gone banner → Restore). Regression +test updated in `public-thread-environment-decoupling.test.ts`. + +This is the server side of the toast "Undo". Wire unarchive to the existing +`retire.cancelled` edge so undoing within the 10s window restores the **intact** +worktree (uncommitted work preserved), no reprovision. The same route also handles +the post-window case: if the env is already gone, unarchive can't revive it and the +client is routed to Restore (Phase 3). + +- In the unarchive route (`routes/threads/actions.ts:383-398`), after + `unarchiveThread`: if the thread's environment is `retiring`, fire + `retire.cancelled` via `applyLoggedEnvironmentLifecycleEvent` (reuse the + `queued-messages.ts:89-94` pattern). Fire it for the env regardless of which + thread is unarchived, so an archived sibling can rescue a shared env. +- **Inspect the outcome** (don't ignore it, per critique): if `retire.cancelled` + was *not applied* because the sweep already won the CAS (env now + `destroying`/`destroyed`), do not silently succeed — surface env-gone / route the + client to Restore. +- Update the stale `actions.ts:380` comment that codifies the monotonic-cleanup + invariant this phase intentionally breaks. + +### Phase 3 — Restore environment route (post-destroy recovery), server + daemon — ✅ IMPLEMENTED + +Shipped (refined from the sketch below): +- **Daemon branch-preservation** (`packages/host-workspace/src/provisioning.ts`): + `createWorktree` now checks whether the branch already exists in the source repo + (`git show-ref`), and if so checks it out in place (`git worktree add + `) instead of `-B`-resetting it to base. This recovers committed work; a + brand-new thread's branch never pre-exists, so it's unaffected. +- **`POST /threads/:id/restore-environment`** route + `restoreThreadEnvironment` + service (`thread-environment-restore.ts`). Routing: `retiring` → in-place + `retire.cancelled`; `destroying` → 409; `ready/provisioning/error` → no-op; + pruned env (null) → 409; `destroyed` → mint a fresh env and reprovision. +- The fresh-env reprovision reuses the create-path choreography: `run.preparing` + moves the thread `→ starting`, then a `seedWithoutRun` provisioning context with + a `direct-managed` intent carrying the **stored branch name** (new optional + `branchName` on the intent) creates the fresh env and re-seeds the thread into + `idle` (no automatic turn) once the workspace is ready. +- Guards: `unmanaged` → 409 (user-owned, can't recreate); `personal` → recreates + the empty scratch dir. +- Validated end-to-end by an integration test that commits work, archives (destroy), + restores, and asserts the committed file reappears in the fresh worktree on the + same branch. + +Original sketch (kept for reference): + +For a thread whose env is genuinely gone. **`destroyed` is terminal — do not add a +`destroyed → provisioning` edge.** Mint a fresh environment row and repoint the +thread. This is net-new (the existing reprovision paths operate on the same row and +reject gone envs at `thread-turn-dispatch.ts:111-118`). + +The earlier "blocker" (the create path is gated on a `starting` thread with an +in-memory provision context) is **surmountable**: `run.preparing` is a legal +`idle→starting` and pre-start-`error`→`starting` transition +(`packages/domain/src/thread-lifecycle.ts`; already used at +`thread-turn-dispatch.ts:141-149`). So Restore re-enters the *existing* thread +provisioning choreography rather than hand-rolling it — with two corrections the +naive "just reuse the create path" misses (branch name + the `-B` reset). + +`POST /threads/{id}/restoreEnvironment` (thread-scoped; next to unarchive), plus a +server-contract entry (`noRequest()` → `{ ok: true }`, exactly like +`unarchive`). Behavior by current env status: + +1. If the thread is archived, unarchive first (one-click restore from an env-gone + archived thread). +2. `retiring` → route to the cheap in-place `retire.cancelled` revive and return + (worktree intact; no new row). *(This is the same revive as Phase 2.)* +3. `destroying` → reject `409` ("Environment is still being torn down; try again + shortly"). Client retries once status flips to `destroyed`. +4. `destroyed` (or `environmentId === null`) → mint a fresh env and re-enter + provisioning: + - `createEnvironment` (`packages/db/src/data/environments.ts:36`) with + `status:'provisioning'`, copying `hostId / projectId / workspaceProvisionType + / branchName / baseBranch / mergeBaseBranch` from the destroyed row. Created + directly in `provisioning` — **no `provision.requested` event** (the lifecycle + has no such cell from `provisioning`; mirror + `createPreparedProvisioningEnvironment`'s "no provision.requested event here", + `thread-provisioning-environment.ts:651-652`). + - Repoint `thread.environmentId` to the new row, then `run.preparing` to move + the thread `→ starting` (so `advanceThreadProvisioning`'s `status==='starting'` + gate at `thread-provisioning.ts:350` is satisfied). + - Build the `environment.provision` command from the **stored** + `environment.branchName` (not a re-derived `buildManagedBranchName` — that would + produce a *different* branch and miss the commits). This is exactly + `dispatchManagedEnvironmentReprovision`'s branch source + (`environment-provisioning-internal.ts:1087-1092`: + `environment.branchName ?? buildManagedBranchName`, base via + `storedBaseBranchNameToSpec`), applied to the **fresh** row. Then drive + `advanceEnvironmentProvisioning` + `advanceThreadProvisioning` to start the + thread once the workspace is ready. (Do **not** call + `dispatchManagedEnvironmentReprovision` itself — it fires `provision.requested` + on an existing non-terminal row.) + - **Rollback:** if create/provision fails before the repoint commits, roll + `thread.environmentId` back (to the destroyed row, or null) so a failed restore + never strands the thread on a half-born env. + - **DAEMON CHANGE (required for committed-work recovery):** today + `createWorktree` (`provisioning.ts:255`) always does `git worktree add -B + `, which **resets** the surviving branch to base and orphans its + commits. Change it to **check out an existing branch in place**: if `branchName` + already exists in the source repo, `git worktree add ` + (no `-B`, preserves the tip); else create from base as today. This is small and + contained, and also makes error-recovery reprovision branch-safe. Without it, + Restore "succeeds" but silently discards the very commits it promised. +5. Guards (verified with provision type): + - `managed-worktree` → full restore (the above). + - `unmanaged` → `409`. The workspace is **user-owned**; bb never created it and + can't safely recreate it (`dispatchManagedEnvironmentReprovision` already 409s + unmanaged at `environment-provisioning-internal.ts:1030`). Hide the Restore + affordance for unmanaged envs. + - `personal` → recreate the empty scratch dir only (personal is a non-git + `mkdir`'d dir, `provision.ts` `provisionPersonalWorkspace`); the confirm dialog + must say "this creates an empty workspace; previous contents are gone". + - offline daemon → provision queues as normal (no special handling). +6. **Post-TTL fallback:** `pruneDestroyedEnvironments` hard-deletes destroyed rows + after `DESTROYED_ENVIRONMENT_TTL_MS` (7 days; `sweeps.ts`), and + `threads.environmentId` is `onDelete:'set null'`. A pruned env leaves the thread + with `environmentId === null` and **no** branch/host metadata (those lived on the + deleted row; the thread row carries only `projectId`). Restore then falls back to + project defaults (default host, default branch, managed worktree) like a + brand-new thread env — or returns `410`. Decide + document; do not imply branch + recovery post-prune. + +No new `EnvironmentLifecycleEvent` or `ThreadLifecycleEvent` types are required. +The only non-server change is the daemon `createWorktree` branch-preservation tweak +in 4. + +### Phase 4 — Frontend Restore affordance + undo toast — ✅ IMPLEMENTED + +Shipped: +- `ThreadPromptEnvironmentGoneSection` gained `onRestore`/`restorePending`; the + read-only banner renders an `EnvironmentRestoreTextAction` in the statusAction + slot — "Restore environment" when `destroyed`, a disabled "Cleaning up…" while + `destroying`. Banner tests cover both. +- `api.restoreThreadEnvironment` + `useRestoreThreadEnvironment` (relies on the + realtime channel to refresh the thread/environment after reprovision); wired + through `ThreadDetailPromptArea`. +- **Archive Undo toast**: `useArchiveThread` now shows a 10s "Thread archived — + Undo" toast; Undo un-archives, which revives a still-`retiring` environment + losslessly. The durable banner Unarchive remains as the fallback after the toast. +- No confirm modal (see decision at top). `SideChatTabContent` was investigated + and needs **no** parity: its "This side chat is no longer available" copy + (`SideChatTabContent.tsx:289`) is a `missingThreadLabel` for a *not-found + thread*, not an environment-gone surface — there is nothing to restore. + +Original sketch (kept for reference): + +- Extend `ThreadPromptEnvironmentGoneSection` (`ThreadPromptContextBanner.tsx:93`) + with `onRestore?` / `restorePending?`, mirroring `ThreadPromptArchivedSection`. + In `ReadOnlyContextBanner` the `statusAction` slot currently renders only for + `archivedSection?.onUnarchive` (lines 575-582) — extend it to render an + `EnvironmentRestoreTextAction` (clone of `ThreadUnarchiveTextAction`) when + `environmentGoneSection?.onRestore` exists: disabled "Cleaning up…" on + `destroying`, active "Restore environment" on `destroyed` (keep `CircleX`). +- **Confirm dialog** (uncommitted work is gone): "Restore environment? We'll create + a fresh workspace and check out branch ``. Committed and pushed work + on that branch is preserved. Uncommitted changes, untracked files, and unpushed + local commits were lost when the old workspace was torn down." +- Client: `api.restoreEnvironment` (`apps/app/src/lib/api.ts`) + `useRestoreEnvironment` + (model on `useUnarchiveThread`, thread-state-mutations.ts:231-255); invalidate + thread + environment queries `onSettled` so the banner flips through provisioning + back to a live composer. Wire `onRestore`/`restorePending` from + `ThreadDetailPromptArea` (~line 970) / `ThreadDetailView`. +- **Undo toast on last-thread archive (the lossless undo):** an `appToast` ("Thread + archived — workspace will be cleaned up" / "Undo", duration **10s** = the grace + window) calling the unarchive mutation. This is the *only* lossless undo — within + 10s it fires `retire.cancelled` and the worktree is intact. Show it only when the + archive actually retired the env (last live thread); a non-last archive keeps the + env and needs no toast. +- **After the 10s window** the worktree is gone, so the recovery affordance is + **Restore** (lossy, committed-only), not lossless Unarchive. An archived thread + whose env is destroyed shows the env-gone banner with "Restore environment" + (Restore unarchives first, then reprovisions — Phase 3 step 1). `ThreadDetailView` + already flags only `destroying`/`destroyed` as env-gone; no new in-grace banner + state is needed since the 10s window is covered by the toast. + +### Phase 5 — Hardening + parity (follow-up) + +- Race/robustness matrix: `retire.cancelled` vs sweep-fired `destroy.started` at the + window edge (CAS picks one winner; better-sqlite3 serialized immediate txns make + it well-defined); shared-env all-gone precondition + destroy CAS re-check; restart + mid-grace; daemon-offline-at-expiry deferral; destroyed-row TTL covers restore-retry. +- ~~Mirror the Restore affordance in `SideChatTabContent`~~ — resolved: that + surface is thread-not-found, not env-gone, so no Restore affordance applies. +- Decide whether to promote the grace clock to a dedicated nullable + `environments.retiredAt` column (stamp on `retire.requested`; clear on + `retire.cancelled`/`destroy.started`) — needed only for a precise frontend + countdown or to immunize the clock from `updateEnvironmentMetadata` writes. + +## Implementation checklist (files to touch) + +Verified call sites for a fast start. Order: Phase 1 → 2 → 3 → 4. + +**Phase 1 — grace gate (server):** +- `apps/server/src/services/system/periodic-sweeps.ts` — add + `MANAGED_ENVIRONMENT_RETIRE_GRACE_MS = 10_000` (~line 60); split the 15-min + throttle so `recoverOrphanedEnvironmentDestroyRequests` stays throttled but + `sweepManagedEnvironments` + `runEnvironmentCleanupAdvance` run each tick. +- `apps/server/src/services/environments/environment-cleanup-internal.ts` — add the + grace early-return in `advanceEnvironmentCleanup` after the refreshed-status recheck + (~lines 392–399), keyed on `refreshedEnvironment.updatedAt`. +- `packages/db/src/data/sweeps.ts` — add `AND updatedAt <= now - grace` to + `sweepManagedEnvironments` (~line 412). +- Tests: `apps/server/test/services/managed-environment-cleanup-recovery.test.ts` (or + a new sibling), in-memory SQLite. + +**Phase 2 — unarchive revives (server):** +- `apps/server/src/routes/threads/actions.ts` — unarchive handler (~lines 383–398): + after `unarchiveThread`, if env is `retiring` fire `retire.cancelled` + (`applyLoggedEnvironmentLifecycleEvent`, pattern at `queued-messages.ts:89-94`); + inspect the outcome and route to Restore if already gone. Update the stale comment + at `actions.ts:380`. + +**Phase 3 — restore route (server) + daemon branch fix:** +- `packages/server-contract/src/public-api.ts` — add `restoreEnvironment` route next + to `archive`/`unarchive` (~line 646): `noRequest()` → `{ ok: true }`. +- `apps/server/src/routes/threads/actions.ts` — new `post(routes.restoreEnvironment, …)` + handler implementing the status routing in Phase 3. Reuses `createEnvironment`, + `attachThreadToEnvironment`/`updateThread`, `run.preparing`, + `advanceEnvironmentProvisioning`, `advanceThreadProvisioning`. +- `packages/host-workspace/src/provisioning.ts` — `createWorktree` (~line 255): + check out an existing branch in place instead of `-B`-resetting it. + +**Phase 4 — UI:** +- `apps/app/src/components/promptbox/banner/ThreadPromptContextBanner.tsx` — extend + `ThreadPromptEnvironmentGoneSection` (~line 93) with `onRestore?`/`restorePending?`; + add an `EnvironmentRestoreTextAction` in the `ReadOnlyContextBanner` statusAction + slot (~lines 575–582). Only surface Restore for managed envs (unmanaged → hidden). +- `apps/app/src/lib/api.ts` — `restoreEnvironment(id)` next to `unarchiveThread` (~1267). +- `apps/app/src/hooks/mutations/thread-state-mutations.ts` — `useRestoreEnvironment` + (model on `useUnarchiveThread`, ~lines 231–255), invalidate thread + environment. +- `apps/app/src/views/thread-detail/ThreadDetailPromptArea.tsx` — wire + `onRestore`/`restorePending` into `environmentGoneSection` (~line 970), mirroring + the `handleUnarchiveCurrentThread` wiring. +- Add the archive Undo `appToast` (10s) and the Restore confirm dialog. + +## T3 checkpointing evaluation + +**Verdict: out of scope for this change; file as an independent follow-up at most.** + +The reported pain is a **timing** problem, not a durability one. The grace period +prevents destruction during the window, and `retire.cancelled`-on-unarchive gives +**loss-free** undo of the *live* worktree (uncommitted + untracked included) — +strictly better than any checkpoint, which can only restore a committed snapshot. +Restore then recovers committed/pushed work after the window for free (the shared +object store survives `git worktree remove`). So checkpointing would only buy back +the narrow residual: *user ignored the grace window AND had valuable uncommitted +edits AND a turn boundary had fired to snapshot them.* + +Cost is a new subsystem, the opposite of smallest-correct-change: none of the +pieces exist (no `workspace.captureCheckpoint`/`restoreCheckpoint` daemon commands, +no `turn/completed` checkpoint reactor, no per-env+turn ref namespace, no +restore-to-arbitrary-SHA op — current workspace reset is hard-reset-to-HEAD), plus +the conversation-rollback coupling T3 itself flags as the real work. + +There is also a **self-conflict**: T3's hygiene rule is "delete checkpoint refs on +archive" (to bound orphan-commit growth), but Restore's value is recovering work +*after* archive/destroy. If archive deletes the refs, they're gone exactly when +Restore would use them. Making checkpoints useful for Restore requires the opposite +— retain env-scoped refs and GC them at the destroyed-env TTL +(`DESTROYED_ENVIRONMENT_TTL_MS`), a deliberate retention design that must be owned +from day one, not bolted on here. (It is technically feasible — refs in the shared +`.git` do survive a worktree destroy — which is why it's a *follow-up*, not +*infeasible*.) + +Recommendation: ship grace + undo + restore first; gate any checkpoint work on +telemetry showing post-grace uncommitted-loss is real and frequent. If pursued, +frame it as "recover uncommitted work after the grace window or a hard crash," own +the ref namespace + TTL-aligned GC, and reuse the Restore confirm dialog built here +as the insertion point ("Recover work from last checkpoint"). Building Restore now +does not foreclose it. + +## Exit criteria + +- **Grace period:** archiving the last live thread leaves the env `retiring` with + the worktree intact for the full window; `destroy.started` fires only after + `window + one sweep tick`; restart mid-window still destroys after the window; + daemon offline at expiry defers (no lost timer); archiving a non-last thread never + enters `retiring`. +- **Undo:** unarchiving within the window returns the env to `ready` with + uncommitted/untracked work preserved and the composer re-enabled; a + `destroy.started`-won race routes to Restore rather than silently succeeding. +- **Restore:** a thread on a `destroyed` managed env gets a new provisioning env on + the same branch; committed/pushed work reappears; `409` on `destroying` with a + documented retry-after-flip; `unmanaged` → `409`; failed provision rolls back the + repoint; double-click is idempotent; pruned-env path falls back to project + defaults (or `410`) per the recorded decision. +- **UI:** env-gone banner shows a working Restore button (disabled during + `destroying`, active on `destroyed`); confirm dialog states what is/ isn't + recoverable; archive shows an Undo toast that reverts within the window, with the + banner Unarchive as durable fallback. + +## Validation + +- New unit tests around `advanceEnvironmentCleanup`: no `destroy.started` within the + window; dispatch after it; `error`-status destroy unaffected by the gate. Reuse + in-memory SQLite (`createConnection(":memory:")` + `migrate(db)`); never mock the DB. +- Lifecycle tests: extend `packages/domain/test/environment-lifecycle.test.ts` only + if new transitions are added (none planned) — otherwise assert the route/sweep + behavior in server tests. +- Integration: archive-last-thread → assert `retiring` + worktree present on disk → + unarchive → assert `ready` + worktree intact; let the window elapse → assert + `destroyed` + worktree gone; restore → assert fresh env provisions on the branch + and committed work is present. +- Race test (Phase 5): interleave unarchive and the sweep at the window edge; assert + a single CAS winner and that the loser path is handled. +- Manual QA via `scripts/bb-dev-app` + `pnpm bb:dev thread …` to watch the banner + transition through provisioning back to a live composer. +- Typecheck/build/test via Turbo per `AGENTS.md` + (`pnpm exec turbo run typecheck|test --filter=@bb/`); pipe slow test output to + a file. + +## Open questions + +1. ~~**Grace window length / configurability**~~ — **resolved**: 10s default, + surfaced as the archive toast's Undo, and now a server config value + (`managedEnvironmentRetireGraceMs`) so it is tunable (and set to 0 in the + integration harness). Revisit the default only if 10s proves too tight. +2. **Grace clock storage** — **no new column** (the `retiring` row's `updatedAt` is + a faithful clock). The 10s window makes this clearly correct: the only in-place + writer that touches `updatedAt`, `updateEnvironmentMetadata` (PATCH + /environments/:id), would have to land within a 10s window to perturb it, and the + only effect is a few seconds' extra delay — never a brick. A dedicated + `environments.retiredAt` column is unnecessary unless a precise countdown UI is + later wanted. +3. ~~**In-grace UI**~~ — **resolved**: the 10s toast Undo covers the in-grace state; + no separate `retiring` banner treatment. After the window, the env-gone banner's + Restore takes over. +4. ~~**Personal/unmanaged Restore**~~ — **resolved**: `managed-worktree` → full + restore; `unmanaged` → `409` + hide the affordance (user-owned workspace bb can't + recreate); `personal` → recreate the empty scratch dir with an explicit "previous + contents are gone" confirmation. +5. ~~**Branch staleness on Restore**~~ — **resolved as a minor, documented + limitation**: `environment.branchName` is set once at provision time from the + daemon's `getCurrentBranch()` and never re-synced + (`environment-provisioning-internal.ts:603`), so a manual terminal `git checkout` + isn't tracked and Restore re-checks-out the *stored* branch. Low stakes: every + branch survives the destroy in the shared source repo, so the restored worktree + can `git checkout` the user's other branch. Name the stored branch in the confirm + dialog; no pre-destroy re-discovery is possible (the worktree is already gone). diff --git a/tests/integration/fake/multi-thread/environment-isolation.test.ts b/tests/integration/fake/multi-thread/environment-isolation.test.ts index 8a3cebf63..ae8a1a623 100644 --- a/tests/integration/fake/multi-thread/environment-isolation.test.ts +++ b/tests/integration/fake/multi-thread/environment-isolation.test.ts @@ -5,6 +5,7 @@ import { archiveThread, getEnvironment, getHosts, + restoreEnvironment, runEnvironmentAction, sendTextMessage, unarchiveThread, @@ -99,6 +100,84 @@ describe.sequential( expect(environment.status).toBe("destroyed"); })); + it("restores a destroyed managed environment, recovering committed work on the branch", () => + withHarness(async (harness) => { + const project = await createProjectFixture(harness, { + name: "Restore Destroyed Environment", + }); + const { thread, environment } = await createReadyHostThread(harness, { + projectId: project.id, + timeoutMs: DEFAULT_TIMEOUT_MS, + workspace: { type: "managed-worktree" }, + }); + const originalPath = environment.path; + const branchName = environment.branchName; + if (!originalPath || !branchName) { + throw new Error("Managed worktree path/branch was not assigned"); + } + + // Commit work in the worktree so there is recoverable history on the + // branch (committed work survives the destroy; uncommitted does not). + await fs.writeFile( + path.join(originalPath, "recovered.txt"), + "keep me\n", + ); + await runGit({ cwd: originalPath, args: ["add", "recovered.txt"] }); + await runGit({ + cwd: originalPath, + args: [ + "-c", + "user.email=test@bb.test", + "-c", + "user.name=BB Test", + "commit", + "-m", + "committed work", + ], + }); + + // Archiving the only thread tears the workspace down (the integration + // harness disables the grace window). + await archiveThread(harness.api, thread.id); + await waitForPathRemoval(originalPath, DEFAULT_TIMEOUT_MS); + await waitForEnvironmentStatus( + harness.api, + environment.id, + "destroyed", + DEFAULT_TIMEOUT_MS, + ); + + // Restore reprovisions a fresh environment on the same branch and + // re-seeds the thread into idle (no automatic turn). + await restoreEnvironment(harness.api, thread.id); + const restoredThread = await waitForThreadStatus( + harness.api, + thread.id, + "idle", + DEFAULT_TIMEOUT_MS, + ); + const restoredEnvironmentId = restoredThread.environmentId; + expect(restoredEnvironmentId).toBeTruthy(); + // A fresh environment row replaces the terminal destroyed one. + expect(restoredEnvironmentId).not.toBe(environment.id); + + const restoredEnvironment = await waitForEnvironmentStatus( + harness.api, + restoredEnvironmentId!, + "ready", + DEFAULT_TIMEOUT_MS, + ); + expect(restoredEnvironment.branchName).toBe(branchName); + expect(restoredEnvironment.path).toBeTruthy(); + + // The committed work on the branch is recovered in the fresh worktree. + const recovered = await fs.readFile( + path.join(restoredEnvironment.path!, "recovered.txt"), + "utf8", + ); + expect(recovered).toBe("keep me\n"); + })); + it("isolates concurrent work across separate environments", () => withHarness(async (harness) => { const secondRepoDir = await createTestGitRepo({ diff --git a/tests/integration/helpers/api.ts b/tests/integration/helpers/api.ts index 7c9ad5a65..f97f43dbe 100644 --- a/tests/integration/helpers/api.ts +++ b/tests/integration/helpers/api.ts @@ -510,6 +510,16 @@ export async function unarchiveThread( await expectStatus(response, 200, `unarchive thread ${threadId}`); } +export async function restoreEnvironment( + api: PublicApiClient, + threadId: string, +): Promise { + const response = await api.threads[":id"]["restore-environment"].$post({ + param: { id: threadId }, + }); + await expectStatus(response, 200, `restore environment for thread ${threadId}`); +} + export async function updateEnvironment( api: PublicApiClient, environmentId: string, diff --git a/tests/integration/helpers/harness.ts b/tests/integration/helpers/harness.ts index fc1ddfcdc..6b526e123 100644 --- a/tests/integration/helpers/harness.ts +++ b/tests/integration/helpers/harness.ts @@ -238,6 +238,11 @@ async function startIntegrationServer( threadStorageRootPath, transcriptionModel: "test/mock-transcription", isDevelopment: false, + // The integration harness runs no periodic sweep and has no time control, so + // the archive grace window is disabled here: archiving the last live thread + // tears down its workspace immediately, as these tests expect. The grace + // window itself is covered by the server-level cleanup tests. + managedEnvironmentRetireGraceMs: 0, }; const machineAuth = await createMachineAuthService({ dataDir: serverDataDir,