fix: recover watcher and serialize database writers

__tests__/mcp-daemon.test.ts

@@ -163,14 +163,56 @@ function countListeningLines(root: string): number {
 
 function killTree(...procs: ChildProcessWithoutNullStreams[]): void {
   for (const p of procs) {
-    if (!p.killed) { try { p.kill('SIGKILL'); } catch { /* gone */ } }
+    if (p.exitCode === null && p.signalCode === null) {
+      try { p.kill('SIGKILL'); } catch { /* gone */ }
+    }
   }
 }
 
 async function waitProcessExit(pid: number, timeoutMs: number): Promise<boolean> {
   return waitFor(() => !isAlive(pid), timeoutMs).then(() => true).catch(() => false);
 }
 
+function waitChildClose(
+  child: ChildProcessWithoutNullStreams,
+  timeoutMs: number,
+): Promise<boolean> {
+  if ((child.exitCode !== null || child.signalCode !== null)
+    && child.stdin.destroyed && child.stdout.destroyed && child.stderr.destroyed) {
+    return Promise.resolve(true);
+  }
+  return new Promise((resolve) => {
+    let settled = false;
+    let timer: NodeJS.Timeout;
+    const finish = (closed: boolean) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      child.removeListener('close', onClose);
+      resolve(closed);
+    };
+    const onClose = () => finish(true);
+    child.once('close', onClose);
+    timer = setTimeout(() => finish(false), timeoutMs);
+  });
+}
+
+async function removeTempDirWhenReleased(dir: string, timeoutMs: number): Promise<void> {
+  const started = Date.now();
+  let lastError: NodeJS.ErrnoException | null = null;
+  do {
+    try {
+      fs.rmSync(dir, { recursive: true, force: true });
+      return;
+    } catch (error) {
+      lastError = error as NodeJS.ErrnoException;
+      if (!['EBUSY', 'EPERM', 'ENOTEMPTY'].includes(lastError.code ?? '')) throw error;
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+  } while (Date.now() - started <= timeoutMs);
+  throw lastError ?? new Error(`Timed out removing ${dir}`);
+}
+
 describe('Shared MCP daemon (issue #411)', () => {
   let tempDir: string;   // the (possibly symlinked) path processes are spawned with
   let realRoot: string;  // its canonical form — what the daemon keys paths on
@@ -184,18 +226,35 @@ describe('Shared MCP daemon (issue #411)', () => {
   });
 
   afterEach(async () => {
+    // Register close listeners before kill so already-fast Windows exits cannot
+    // race past the event that proves child stdio/process handles were released.
+    const proxyCloseWaits = servers.map((server) => waitChildClose(server.child, 5000));
+    // Capture the detached daemon before killing proxies: some shutdown
+    // paths remove the pidfile while Windows still owns process/CWD handles.
+    const daemonPid = readLockPid(realRoot);
     killTree(...servers.map((s) => s.child));
+    const proxyClosed = await Promise.all(proxyCloseWaits);
+    const failedProxyIndex = proxyClosed.findIndex((closed) => !closed);
     // The daemon is detached (not a tracked child) — reap it explicitly via the
     // pid it recorded, so a test can't leak a background daemon. Guard against
     // our own pid: the version-mismatch test plants `pid: process.pid` in the
     // lockfile, and we must never SIGKILL the vitest worker.
-    const daemonPid = readLockPid(realRoot);
     if (daemonPid && daemonPid !== process.pid && isAlive(daemonPid)) {
       try { process.kill(daemonPid, 'SIGKILL'); } catch { /* race */ }
+      if (!(await waitProcessExit(daemonPid, 5000))) {
+        throw new Error(`Detached daemon ${daemonPid} did not exit during teardown`);
+      }
     }
-    await new Promise((r) => setTimeout(r, 50));
+    const failedProxyPid = failedProxyIndex === -1
+      ? null
+      : servers[failedProxyIndex].child.pid ?? 'unknown';
     servers.length = 0;
-    fs.rmSync(tempDir, { recursive: true, force: true });
+    if (failedProxyIndex !== -1) {
+      throw new Error(`Proxy process ${failedProxyPid} did not close during teardown`);
+    }
+    // A detached daemon is not a ChildProcess we can await for `close`. Retry
+    // only transient Windows directory-handle races after its PID is gone.
+    await removeTempDirWhenReleased(tempDir, 5000);
   });
 
   it('two invocations share ONE detached daemon; both attach as proxies', async () => {

__tests__/security.test.ts

@@ -18,6 +18,7 @@ import { ToolHandler, tools } from '../src/mcp/tools';
 import { scanDirectory, isSourceFile } from '../src/extraction';
 import { DatabaseConnection, getDatabasePath } from '../src/db';
 import { QueryBuilder } from '../src/db/queries';
+import { acquireDatabaseWriterLease } from '../src/db/writer-lease';
 
 function createTempDir(): string {
   return fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-security-test-'));
@@ -66,6 +67,29 @@ describe('FileLock', () => {
     lock1.release();
   });
 
+  it('should never steal an old lock from a live process', () => {
+    fs.writeFileSync(lockPath, String(process.pid));
+    const old = new Date(Date.now() - 10 * 60 * 1000);
+    fs.utimesSync(lockPath, old, old);
+
+    const lock = new FileLock(lockPath);
+    expect(() => lock.acquire()).toThrow(/locked by another process/);
+    expect(fs.readFileSync(lockPath, 'utf8').trim()).toBe(String(process.pid));
+  });
+
+  it('should keep the database writer lease exclusive across operation locks', () => {
+    const writerLease = acquireDatabaseWriterLease(tempDir);
+    const operationLock = new FileLock(path.join(tempDir, '.codegraph', 'codegraph.lock'));
+    operationLock.acquire();
+    operationLock.release();
+
+    expect(() => acquireDatabaseWriterLease(tempDir)).toThrow(/locked by another process/);
+    writerLease.release();
+
+    const nextWriter = acquireDatabaseWriterLease(tempDir);
+    nextWriter.release();
+  });
+
   it('should detect and remove stale locks from dead processes', () => {
     // Write a lock file with a PID that doesn't exist
     // PID 99999999 is extremely unlikely to be a real process

__tests__/watcher.test.ts

@@ -264,13 +264,19 @@ describe('FileWatcher', () => {
     });
   });
 
-  describe('lock contention degradation (#876)', () => {
-    it('disables auto-sync after prolonged lock contention, with bounded retries', async () => {
-      const syncFn = vi.fn().mockRejectedValue(new LockUnavailableError());
+  describe('lock contention recovery (#876)', () => {
+    it('keeps retrying with capped backoff and recovers after prolonged lock contention', async () => {
+      let attempts = 0;
+      const syncFn = vi.fn(async () => {
+        attempts += 1;
+        if (attempts <= 8) {
+          throw new LockUnavailableError();
+        }
+        return { filesChanged: 1, durationMs: 5 };
+      });
       const onSyncComplete = vi.fn();
       const onSyncError = vi.fn();
       const onDegraded = vi.fn();
-      const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
       const watcher = newWatcher(syncFn, {
         debounceMs: 25,
         onSyncComplete,
@@ -281,22 +287,22 @@ describe('FileWatcher', () => {
       await watcher.waitUntilReady();
       __emitWatchEventForTests(testDir, 'src/long-lock.ts');
 
-      // 5 backoff retries (25·1,2,4,8,16 ms), then degrade on the 6th attempt.
-      await waitFor(() => !watcher.isActive(), 8000, 20);
+      // Cross the former five-retry budget and prove the watcher remains live
+      // with its stale entry retained while the competing writer owns the lock.
+      await waitFor(() => syncFn.mock.calls.length >= 7, 8000, 20);
+      expect(watcher.isActive()).toBe(true);
+      expect(watcher.isDegraded()).toBe(false);
+      expect(onDegraded).not.toHaveBeenCalled();
+      expect(watcher.getPendingFiles().some((p) => p.path === 'src/long-lock.ts')).toBe(true);
 
-      expect(syncFn.mock.calls.length).toBeGreaterThanOrEqual(6); // MAX_LOCK_RETRIES + 1
-      expect(watcher.isDegraded()).toBe(true);
-      expect(onDegraded).toHaveBeenCalledTimes(1);
-      expect(onDegraded).toHaveBeenCalledWith(expect.stringContaining('auto-sync disabled'));
-      // A held lock is neither a sync error nor a completion.
+      // The ninth attempt succeeds after eight lock-contention failures.
+      await waitFor(() => onSyncComplete.mock.calls.length > 0, 8000, 20);
+      expect(syncFn).toHaveBeenCalledTimes(9);
       expect(onSyncError).not.toHaveBeenCalled();
-      expect(onSyncComplete).not.toHaveBeenCalled();
-      // Degrade stops the watcher, which clears pending state.
-      expect(watcher.getPendingFiles()).toEqual([]);
-      const disableWarnings = warnSpy.mock.calls.filter(
-        (c) => typeof c[0] === 'string' && c[0].includes('File watcher disabled')
-      );
-      expect(disableWarnings).toHaveLength(1);
+      expect(onSyncComplete).toHaveBeenCalledTimes(1);
+      expect(watcher.getPendingFiles().some((p) => p.path === 'src/long-lock.ts')).toBe(false);
+
+      watcher.stop();
     });
 
     it('does NOT degrade on brief contention — backoff resets after a clean sync', async () => {

src/bin/codegraph.ts

@@ -53,6 +53,8 @@ import { relaunchWithWasmRuntimeFlagsIfNeeded } from '../extraction/wasm-runtime
 import { installCommandSupervision } from './command-supervision';
 import { EXTRACTION_VERSION } from '../extraction/extraction-version';
 import { getTelemetry, TELEMETRY_DOCS, recordIndexEvent } from '../telemetry';
+import { FileLock } from '../utils';
+import { acquireDatabaseWriterLease } from '../db/writer-lease';
 
 // Decided once, before `--color`/`--no-color` are stripped from argv below
 // (#1281). Piped/redirected stdout, NO_COLOR, or --no-color -> plain output.
@@ -601,6 +603,7 @@ program
   .action(async (pathArg: string | undefined, options: { index?: boolean; force?: boolean; verbose?: boolean }) => {
     const projectPath = path.resolve(pathArg || process.cwd());
     const clack = await importESM('@clack/prompts');
+    let writerLease: FileLock | null = null;
 
     clack.intro('Initializing CodeGraph');
 
@@ -628,6 +631,7 @@ program
         return;
       }
 
+      writerLease = acquireDatabaseWriterLease(projectPath);
       const { default: CodeGraph, getDatabasePath } = await loadCodeGraph();
       const cg = await CodeGraph.init(projectPath, { index: false });
       clack.log.success(`Initialized in ${projectPath}`);
@@ -676,7 +680,9 @@ program
       cg.destroy();
     } catch (err) {
       clack.log.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
-      process.exit(1);
+      process.exitCode = 1;
+    } finally {
+      writerLease?.release();
     }
   });
 
@@ -689,6 +695,7 @@ program
   .option('-f, --force', 'Skip confirmation prompt')
   .action(async (pathArg: string | undefined, options: { force?: boolean }) => {
     const projectPath = resolveProjectPath(pathArg);
+    let writerLease: FileLock | null = null;
 
     try {
       if (!isInitialized(projectPath)) {
@@ -714,6 +721,7 @@ program
         }
       }
 
+      writerLease = acquireDatabaseWriterLease(projectPath);
       const { default: CodeGraph } = await loadCodeGraph();
       const cg = CodeGraph.openSync(projectPath);
       cg.uninitialize();
@@ -737,7 +745,9 @@ program
       } catch { /* non-fatal */ }
     } catch (err) {
       error(`Failed to uninitialize: ${err instanceof Error ? err.message : String(err)}`);
-      process.exit(1);
+      process.exitCode = 1;
+    } finally {
+      writerLease?.release();
     }
   });
 
@@ -752,22 +762,26 @@ program
   .option('-v, --verbose', 'Show detailed worker lifecycle and memory info')
   .action(async (pathArg: string | undefined, options: { force?: boolean; quiet?: boolean; verbose?: boolean }) => {
     const projectPath = resolveProjectPath(pathArg);
+    let writerLease: FileLock | null = null;
 
     try {
       // Don't (re)index your home directory / a filesystem root (#845). --force
       // doubles as the override.
       const unsafe = unsafeIndexRootReason(projectPath);
       if (unsafe && !options.force) {
         error(`Refusing to index ${projectPath} — it looks like ${unsafe}. Pass --force to override.`);
-        process.exit(1);
+        process.exitCode = 1;
+        return;
       }
 
       if (!isInitialized(projectPath)) {
         error(`CodeGraph not initialized in ${projectPath}`);
         info('Run "codegraph init" first');
-        process.exit(1);
+        process.exitCode = 1;
+        return;
       }
 
+      writerLease = acquireDatabaseWriterLease(projectPath);
       const { default: CodeGraph, getDatabasePath } = await loadCodeGraph();
       // `index` is a FULL re-index — identical to a fresh `init`. RECREATE the
       // database from scratch (discard .codegraph/codegraph.db + its WAL) rather
@@ -790,7 +804,7 @@ program
         if (options.quiet) {
           // Quiet mode: no UI, just run against the freshly-recreated graph.
           const result = await cg.indexAll();
-          if (!result.success) process.exit(1);
+          if (!result.success) process.exitCode = 1;
           cg.destroy();
           return;
         }
@@ -824,7 +838,8 @@ program
         }
 
         if (!finalResult.success) {
-          process.exit(1);
+          process.exitCode = 1;
+          return;
         }
 
         clack.outro('Done');
@@ -834,7 +849,9 @@ program
       }
     } catch (err) {
       error(`Failed to index: ${err instanceof Error ? err.message : String(err)}`);
-      process.exit(1);
+      process.exitCode = 1;
+    } finally {
+      writerLease?.release();
     }
   });
 
@@ -847,15 +864,18 @@ program
   .option('-q, --quiet', 'Suppress output (for git hooks)')
   .action(async (pathArg: string | undefined, options: { quiet?: boolean }) => {
     const projectPath = resolveProjectPath(pathArg);
+    let writerLease: FileLock | null = null;
 
     try {
       if (!isInitialized(projectPath)) {
         if (!options.quiet) {
           error(`CodeGraph not initialized in ${projectPath}`);
         }
-        process.exit(1);
+        process.exitCode = 1;
+        return;
       }
 
+      writerLease = acquireDatabaseWriterLease(projectPath);
       const { default: CodeGraph } = await loadCodeGraph();
       const cg = await CodeGraph.open(projectPath);
 
@@ -896,7 +916,9 @@ program
       if (!options.quiet) {
         error(`Failed to sync: ${err instanceof Error ? err.message : String(err)}`);
       }
-      process.exit(1);
+      process.exitCode = 1;
+    } finally {
+      writerLease?.release();
     }
   });
 
@@ -1822,18 +1844,28 @@ program
         return;
       }
 
-      const lockPath = path.join(getCodeGraphDir(projectPath), 'codegraph.lock');
+      const lockPaths = [
+        path.join(getCodeGraphDir(projectPath), 'codegraph.lock'),
+        path.join(getCodeGraphDir(projectPath), 'database-writer.lock'),
+      ];
+      const existingLocks = lockPaths.filter((lockPath) => fs.existsSync(lockPath));
 
-      if (!fs.existsSync(lockPath)) {
+      if (existingLocks.length === 0) {
         info(`No lock file found ${getGlyphs().dash} nothing to do`);
         return;
       }
 
-      fs.unlinkSync(lockPath);
-      success('Removed lock file. You can now run indexing again.');
+      for (const lockPath of existingLocks) {
+        // FileLock only replaces a dead owner. A live writer is never removed,
+        // even when the lock is old.
+        const staleLock = new FileLock(lockPath);
+        staleLock.acquire();
+        staleLock.release();
+      }
+      success(`Removed ${existingLocks.length} stale lock file(s). You can now run indexing again.`);
     } catch (err) {
       error(`Failed to remove lock: ${err instanceof Error ? err.message : String(err)}`);
-      process.exit(1);
+      process.exitCode = 1;
     }
   });