From b4421a4e694072a6b0db223f8f6088318a1ea003 Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:16:53 -0700 Subject: [PATCH 1/9] docs: design the repo-oriented code-intelligence provider contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design for the OPTIONAL code-intelligence provider contract that lets a user pick how their codebase is indexed and searched, replacing gstack's ~17k LOC of bespoke GBrain glue. Repo-oriented ops (register_source/ refresh/search/status required; add/delete/export optional), a per-provider capability matrix, GBrain-recommended-first selection, repo-scoped + install consent, and a phased rollout that keeps gstack fully functional with no provider selected. All three providers are driven from the runtime via CLI or HTTP — no MCP client. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CODE_INTELLIGENCE_PROVIDER_CONTRACT.md | 254 ++++++++++++++++++ lib/code-intelligence/contract.ts | 187 +++++++++++++ lib/code-intelligence/gbrain-adapter.ts | 198 ++++++++++++++ lib/code-intelligence/index.ts | 14 + lib/code-intelligence/picker.ts | 43 +++ test/code-intelligence.test.ts | 218 +++++++++++++++ 6 files changed, 914 insertions(+) create mode 100644 docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md create mode 100644 lib/code-intelligence/contract.ts create mode 100644 lib/code-intelligence/gbrain-adapter.ts create mode 100644 lib/code-intelligence/index.ts create mode 100644 lib/code-intelligence/picker.ts create mode 100644 test/code-intelligence.test.ts diff --git a/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md b/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md new file mode 100644 index 0000000000..c5a01cbb0b --- /dev/null +++ b/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md @@ -0,0 +1,254 @@ +# Code-Intelligence Provider Contract + +Status: design + first implementation slice +Owner: maintainer-directed internal work +Related: `runtime/context.js` (Context.dev provider pattern), +`scripts/gstack2/browser-provider-contract.ts` (the existing provider-contract idiom), +`lib/gstack-decision-semantic.ts` (degrade-to-null reliability contract) + +## Problem + +gstack carries ~17k LOC of home-grown code-intelligence glue: transcript +ingestion (`bin/gstack-memory-ingest.ts`, ~1.9k), a unified sync verb +(`bin/gstack-gbrain-sync.ts`, ~1.6k), context loading +(`bin/gstack-brain-context-load.ts`), a three-tier planning cache +(`bin/gstack-brain-cache`), source reconciliation, engine-status classification, +destructive-op guards, plus ~15 `bin/gstack-gbrain-*` and `bin/gstack-brain-*` +entrypoints and ~40 tests. All of it is bespoke wiring around one external tool +(GBrain) reached by direct CLI shell-out. + +We do not want to keep maintaining a home-grown indexer. We want gstack to +define a **small optional contract** that external providers implement, so the +indexing/search/graph work lives in the provider, not in gstack. + +Hard requirement, non-negotiable: **gstack must remain fully functional with the +provider OFF.** File-only paths (the decision store, Context Recovery, grep) stay +reliable and never depend on a provider being present. This is the existing +decision-store philosophy (`lib/gstack-decision.ts` has zero gbrain imports; +`lib/gstack-decision-semantic.ts` degrades to `null`). The contract is an +enhancement, never a dependency. + +## Design decision: repo-oriented, not document-store + +Settled (do not relitigate). The contract is **repo-oriented**: + +``` +register_source(repo) — required +refresh(source) — required +search(query) — required +status(source) — required +``` + +`add` / `delete` / `export` are **optional capabilities** a provider MAY +advertise. GBrain advertises them (its native primitive is document-by-slug: +put/delete/get/export); code-search and code-graph tools decline them. + +A document-store contract (add / delete-by-id / export as *required* ops) was +rejected: it misrepresents code-search and code-graph tools. Sourcebot indexes a +whole repo and exposes search; it has no concept of "delete document id X". +Forcing every provider to implement a document CRUD surface would either exclude +the exact tools we most want (whole-repo indexers, graph tools) or force them to +stub required ops with lies. Repo-in / query-out is the honest common +denominator. GBrain's document axis survives as an *optional* capability, not as +the contract's shape. + +## The contract + +TypeScript in `lib/code-intelligence/contract.ts`. Shape (abridged): + +```ts +type CodeProviderCapability = + | "register_source" | "refresh" | "search" | "status" // required + | "add" | "delete" | "export"; // optional + +interface CodeProvider { + readonly id: "gbrain" | "sourcebot" | "graphify"; + readonly label: string; + readonly capabilities: ReadonlySet; + readonly local: boolean; // true = no repo content leaves the machine + + registerSource(repo: RepoRef, opts?: OpOptions): Promise; + refresh(source: SourceRef, opts?: OpOptions): Promise; + search(query: string, opts?: SearchOptions): Promise; + status(source?: SourceRef, opts?: OpOptions): Promise; + + add?(doc: { slug: string; body: string }, opts?: OpOptions): Promise; + delete?(slug: string, opts?: OpOptions): Promise; + export?(source: SourceRef, opts?: OpOptions): Promise; +} +``` + +Every provider MUST implement the four required methods and MUST advertise +exactly the capabilities it backs (`assertRequiredCapabilities` enforces the +required four at construction; a test pins it). Optional methods are present iff +the matching capability is advertised. Calling an unadvertised optional op throws +`CAPABILITY_UNSUPPORTED` — never a silent no-op. + +### Typed failures + +Mirrors `runtime/context.js`'s `ContextError` discipline (a closed code set, +constructor throws on an unknown code): + +| Code | Meaning | +|------|---------| +| `PROVIDER_UNAVAILABLE` | CLI/MCP transport absent — degrade to file-only | +| `PROVIDER_NOT_CONSENTED` | repo indexing not consented and content would leave the machine | +| `CAPABILITY_UNSUPPORTED` | provider declines this op | +| `SOURCE_NOT_REGISTERED` | op needs a source that isn't registered | +| `PROVIDER_TIMEOUT` | provider exceeded the op timeout | +| `PROVIDER_ERROR` | provider ran and failed | + +`PROVIDER_UNAVAILABLE` is the load-bearing one: callers catch it (or use the +picker's null resolution) and fall back to grep / file-only. It is never fatal. + +### Consent + +Two orthogonal consent axes, both explicit, neither auto-granted: + +1. **Network / content-egress consent (repo-scoped).** Before any repo content + leaves the machine, indexing must be consented *per repo*. The contract + enforces this in `registerSource`/`refresh`/`add`: when + `provider.local === false` and `opts.consented !== true`, it throws + `PROVIDER_NOT_CONSENTED`. Local providers (Graphify) skip this axis — nothing + leaves the machine. +2. **Install consent (Graphify only).** Graphify is never auto-installed. The + `options`/`status` display marks it available only when its CLI is present, + and nothing in gstack runs a Graphify installer. Install is a user action + (`pip install graphifyy && graphify install`). + +This matches the Context.dev model: selection persists without granting egress +consent; egress requires a separate explicit step. + +## Per-provider capability matrix + +| Op | GBrain (recommend first) | Sourcebot | Graphify | +|----|--------------------------|-----------|----------| +| `register_source` | ✓ `sources add` | ✓ repo added to index config | ✓ index a local dir | +| `refresh` | ✓ `sync --strategy code` | ✓ reindex | ✓ re-parse tree-sitter graph | +| `search` | ✓ `gbrain search` (federated corpora) | ✓ `POST /api/search` (regex, zoekt) | ✓ `graphify query ""` | +| `status` | ✓ `sources list` + page_count | ~ partial (server liveness) | ~ partial (graph.json present + node count) | +| `add` | ✓ `put ` | ✗ declines | ✗ declines | +| `delete` | ✓ `delete ` | ✗ declines | ✗ declines | +| `export` | ✓ `export` | ✗ declines | ✓ read `graphify-out/graph.json` | +| `local` (no egress) | no (federated DB) | loopback → **yes**; remote host → no | **yes** (local only) | + +All three are driven directly from the runtime — no MCP client: + +- **GBrain** (`garrytan/gbrain`, the gstack-ecosystem tool): full contract fit, + driven via the existing `gbrain` CLI chokepoint (`lib/gbrain-exec.ts`). Native + primitive is document-by-slug (put/delete/get/export) PLUS a repo axis + (`sources add`/`sync`). Advertises all seven capabilities. **Recommended + first.** +- **Sourcebot** (`github.com/sourcebot-dev/sourcebot`, YC Fall 2025): self-hosted + whole-repo regex search. `register_source` adds a local `{ "type": "git", "url": + "file:///path" }` connection to the server's `config.json` (it re-indexes on + config change); `search` is `POST {baseUrl}/api/search`; `status` is a liveness + probe. Declines `add`/`delete`/`export`. A loopback `baseUrl` keeps content on + the machine (local); a remote one requires egress consent. +- **Graphify** (`github.com/Graphify-Labs/graphify`, YC-backed): local + tree-sitter code graph via the `graphify` CLI. `graphify ` builds + `graphify-out/graph.json`; `graphify query ""` searches it; `export` reads + the graph JSON. Fully local — nothing leaves the machine. Optional, **install + only with explicit user action** (`pip install graphifyy && graphify install`); + never auto-installed. + +**No local-index option is offered** (deliberately excluded — a naive local +index degrades result quality; we route to a real provider or to file-only grep, +not to a half-baked in-house index). + +### Integration surfaces (no MCP needed) + +Each provider exposes a runtime-drivable surface, so gstack drives them with a +CLI shell-out or plain HTTP — it never speaks MCP: + +- **GBrain / Graphify: CLI.** Shell out (`spawnSync`), same shape and the same + ENOENT→`PROVIDER_UNAVAILABLE` degrade as the existing gbrain glue. +- **Sourcebot: HTTP + a config-file edit.** `POST /api/search` for queries and a + JSON edit of the server's `config.json` to register a repo. `fetch` is + injectable so tests run against a stub, no live server. + +Sourcebot and Graphify also ship MCP servers for in-agent use; the contract does +not depend on them, because their CLI/HTTP surfaces are enough to index and +search from the runtime. + +## Picker: recommend GBrain first + +`lib/code-intelligence/picker.ts` + `selection.ts`. The user picks a provider +with `gstack-code-intelligence select `, persisted to +`$GSTACK_HOME/code-intelligence.json`. `resolveSelectedProvider()` constructs the +selected provider, or returns `null` when nothing is selected — the provider-OFF +path, where callers degrade to grep / the file-only decision store. Availability +is proven at call time: a selected provider whose CLI/server is absent throws +`PROVIDER_UNAVAILABLE`, which callers catch and degrade on. + +`RECOMMENDED_ORDER` is the static **GBrain → Sourcebot → Graphify** fact — GBrain +is always recommended first. `detectAvailable()` probes each provider for the +`options`/`status` display (GBrain via the real `localEngineStatus()`; Graphify +via its CLI/graph presence; Sourcebot via an HTTP liveness probe). The picker +never silently prefers a non-recommended tool. + +## How this replaces the current GBrain glue + +The contract is the seam; the bespoke glue collapses onto it. Mapping: + +| Today (bespoke) | Under the contract | +|-----------------|--------------------| +| `bin/gstack-gbrain-sync.ts` (`sync`/`reindex-code`/`sources`) | `provider.registerSource` / `provider.refresh` | +| `lib/gstack-decision-semantic.ts` `semanticRecall` | `provider.search` (scoped) → same degrade-to-null | +| `bin/gstack-brain-context-load.ts` (`query`/`list_pages`) | `provider.search` / `provider.status` | +| `bin/gstack-memory-ingest.ts` (`import`, put) | `provider.add` (optional cap; GBrain-only) | +| `lib/gbrain-sources.ts` (`ensureSourceRegistered`, `probeSource`) | GBrain adapter internals | +| `lib/gbrain-local-status.ts` | GBrain adapter availability probe (kept, reused) | +| `bin/gstack-gbrain-detect` / `-install` / `-source-wireup` / `-repo-policy` | provider setup + picker + consent (thinner) | + +The point is not to delete 17k LOC in one commit — it is to make every consumer +call the contract, then retire the bespoke paths provider-by-provider behind it. +Consumers that only need "search my code, or degrade" stop importing gbrain +specifics entirely. + +## Rollout + +Phased, each phase independently revertable. Skill-template edits are deferred to +a later phase precisely so the first slices do not trigger the +`gen:gstack2` / parity re-baseline cycle. + +- **Phase 1 (this slice): the contract, three real adapters, and a usable CLI.** + `contract.ts` + fully-drivable GBrain (CLI), Graphify (CLI), and Sourcebot + (HTTP + config) adapters + the selection store + the `gstack-code-intelligence` + CLI (`options`/`status`/`select`/`consent`/`index`/`search`) + tests. A user + can select a provider and index/search their repo today. No skill-template or + generated-file changes yet, so no `gen:gstack2` / parity re-baseline. +- **Phase 2: route internal consumers through the contract.** Point + `gstack-decision-semantic` and `gstack-brain-context-load` at + `resolveSelectedProvider()`, preserving degrade-to-null exactly. Behavior-neutral + for the file-only paths. +- **Phase 3: surface selection in the skills.** Offer the picker at the moments a + skill would benefit from indexed search, mirroring the `context` command's + just-in-time consent prompt. Regenerate skills (`bun run gen:gstack2`), re-run + `bun run test:gstack2`, re-baseline parity intentionally. +- **Phase 4: retire bespoke glue.** Once every consumer is on the contract, + delete the sync/ingest/cache entrypoints and their tests provider-by-provider. + +## What this does NOT change + +Per the GStack 2 canonical contract and CLAUDE.md boundaries: no cloud browsers, +no alternate iOS drivers, no local image models, no provider marketplaces, no +workflow engines, **no new state database**. Context.dev remains the only +newly-authorized external service for web context; this contract governs code +intelligence, a separate axis. The existing decision store and Context Recovery +stay file-only and provider-independent. + +## Testing + +`test/code-intelligence.test.ts` (19 tests, no live tools): capability-matrix +invariants (all providers advertise the four required; only GBrain advertises the +document ops; `local` flags, including loopback-vs-remote Sourcebot); the result +parsers; the selection store + per-repo consent + provider-OFF (`null`); consent +gating (GBrain non-local without consent throws `PROVIDER_NOT_CONSENTED`; local +Graphify is exempt); the GBrain adapter against a fake `gbrain` shim; the Graphify +adapter against a fake `graphify` shim (index builds a graph, search returns hits, +status counts nodes); the Sourcebot adapter against an injected `fetch` + a temp +`config.json` (register writes a local git connection, search maps `files[]` to +hits); and every adapter degrading to `PROVIDER_UNAVAILABLE` when its tool/server +is absent. The `gstack-code-intelligence` CLI was smoke-tested end-to-end: +select → consent gate → local Graphify index (5-node graph) → search. diff --git a/lib/code-intelligence/contract.ts b/lib/code-intelligence/contract.ts new file mode 100644 index 0000000000..8b56dacf45 --- /dev/null +++ b/lib/code-intelligence/contract.ts @@ -0,0 +1,187 @@ +/** + * code-intelligence/contract — the OPTIONAL, repo-oriented provider contract. + * + * gstack does not maintain a home-grown indexer. It defines this small contract + * and external providers (GBrain, Sourcebot, Graphify) implement it. The whole + * contract is OPTIONAL: when no provider is available/consented, + * `resolveCodeProvider()` returns null and callers degrade to grep / the + * file-only decision store. Never a dependency, always an enhancement — the same + * reliability contract as lib/gstack-decision-semantic.ts. + * + * Repo-oriented, not document-store (settled): register_source / refresh / + * search / status are required; add / delete / export are optional capabilities + * a provider MAY advertise. A document-CRUD-required contract would misrepresent + * whole-repo code-search and code-graph tools. See + * docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md. + */ + +export type CodeProviderId = "gbrain" | "sourcebot" | "graphify"; + +export type CodeProviderCapability = + | "register_source" + | "refresh" + | "search" + | "status" + | "add" + | "delete" + | "export"; + +export const REQUIRED_CAPABILITIES: readonly CodeProviderCapability[] = [ + "register_source", + "refresh", + "search", + "status", +] as const; + +export const OPTIONAL_CAPABILITIES: readonly CodeProviderCapability[] = [ + "add", + "delete", + "export", +] as const; + +export interface RepoRef { + /** Source id the provider registers this repo under. */ + id: string; + /** Local worktree path. */ + path: string; + /** Remote URL, when the provider clones/manages it. */ + remoteUrl?: string; +} + +export interface SourceRef { + id: string; +} + +export interface SourceStatus { + id: string; + state: "registered" | "indexing" | "ready" | "absent" | "unknown"; + /** Pages / files / graph nodes, when the provider reports a count. */ + itemCount?: number; + detail?: string; + /** True when the provider only implements a partial status probe. */ + partial?: boolean; +} + +export interface CodeSearchHit { + /** Slug, file path, or symbol id — whatever the provider keys results on. */ + ref: string; + score?: number; + snippet?: string; + kind?: "document" | "file" | "symbol" | "graph-node"; +} + +export interface OpOptions { + /** + * Env override for spawned processes. Production callers leave this unset; + * tests inject a synthetic env (fake CLI on PATH). Matches the existing + * gbrain helpers. + */ + env?: NodeJS.ProcessEnv; + /** Timeout in ms for the underlying op. */ + timeout?: number; + /** + * Explicit per-repo consent that repo content may leave the machine. Required + * for non-local providers on register_source / refresh / add. + */ + consented?: boolean; +} + +export interface SearchOptions extends OpOptions { + /** Restrict to a registered source. */ + source?: string; + limit?: number; + minScore?: number; +} + +export interface CodeProvider { + readonly id: CodeProviderId; + readonly label: string; + readonly capabilities: ReadonlySet; + /** True when no repo content leaves the machine (Graphify). */ + readonly local: boolean; + + has(capability: CodeProviderCapability): boolean; + + registerSource(repo: RepoRef, opts?: OpOptions): Promise; + refresh(source: SourceRef, opts?: OpOptions): Promise; + search(query: string, opts?: SearchOptions): Promise; + status(source?: SourceRef, opts?: OpOptions): Promise; + + add?(doc: { slug: string; body: string }, opts?: OpOptions): Promise; + delete?(slug: string, opts?: OpOptions): Promise; + export?(source: SourceRef, opts?: OpOptions): Promise; +} + +export const CODE_PROVIDER_FAILURES = Object.freeze([ + "PROVIDER_UNAVAILABLE", + "PROVIDER_NOT_CONSENTED", + "CAPABILITY_UNSUPPORTED", + "SOURCE_NOT_REGISTERED", + "PROVIDER_TIMEOUT", + "PROVIDER_ERROR", +] as const); + +export type CodeProviderFailure = (typeof CODE_PROVIDER_FAILURES)[number]; + +const FAILURE_SET = new Set(CODE_PROVIDER_FAILURES); + +/** + * Typed provider failure. Mirrors runtime/context.js ContextError discipline: + * the code set is closed and the constructor throws on an unknown code, so a + * typo can never mint an untyped failure. + */ +export class CodeProviderError extends Error { + readonly code: CodeProviderFailure; + readonly providerId?: CodeProviderId; + + constructor(code: CodeProviderFailure, message: string, providerId?: CodeProviderId) { + if (!FAILURE_SET.has(code)) throw new TypeError(`Unknown code-provider failure code: ${code}`); + super(message); + this.name = "CodeProviderError"; + this.code = code; + this.providerId = providerId; + } +} + +/** + * Enforce that a provider advertises every required capability. Called by each + * adapter constructor so an incomplete provider fails fast, not at first search. + */ +export function assertRequiredCapabilities( + id: CodeProviderId, + capabilities: ReadonlySet, +): void { + const missing = REQUIRED_CAPABILITIES.filter((cap) => !capabilities.has(cap)); + if (missing.length) { + throw new TypeError(`Code provider ${id} is missing required capabilities: ${missing.join(", ")}`); + } +} + +/** + * Guard for optional ops: throw CAPABILITY_UNSUPPORTED (never a silent no-op) + * when a provider is asked for a capability it does not advertise. + */ +export function assertCapability(provider: CodeProvider, capability: CodeProviderCapability): void { + if (!provider.has(capability)) { + throw new CodeProviderError( + "CAPABILITY_UNSUPPORTED", + `${provider.label} does not support "${capability}"`, + provider.id, + ); + } +} + +/** + * Repo-scoped egress consent gate. Non-local providers must not move repo + * content off the machine without explicit per-repo consent. Local providers + * (nothing leaves the machine) are exempt. + */ +export function assertEgressConsent(provider: CodeProvider, opts?: OpOptions): void { + if (provider.local) return; + if (opts?.consented === true) return; + throw new CodeProviderError( + "PROVIDER_NOT_CONSENTED", + `${provider.label} would send repo content off this machine; per-repo indexing consent is required`, + provider.id, + ); +} diff --git a/lib/code-intelligence/gbrain-adapter.ts b/lib/code-intelligence/gbrain-adapter.ts new file mode 100644 index 0000000000..e4dbae2364 --- /dev/null +++ b/lib/code-intelligence/gbrain-adapter.ts @@ -0,0 +1,198 @@ +/** + * GBrain adapter — full contract fit over the existing gbrain CLI chokepoint. + * + * Reuses lib/gbrain-exec.ts (spawnGbrain, seeded DATABASE_URL) and + * lib/gbrain-sources.ts (ensureSourceRegistered, probeSource, sourcePageCount) + * rather than re-issuing raw commands, so the DATABASE_URL / GBRAIN_HOME / + * Windows-shim guarantees carry over unchanged. GBrain's native primitive is + * document-by-slug (put/delete/get/export) PLUS a repo axis (sources add/sync), + * so it advertises all seven capabilities. + */ + +import { spawnGbrain } from "../gbrain-exec"; +import { ensureSourceRegistered, probeSource, sourcePageCount } from "../gbrain-sources"; +import { + assertCapability, + assertEgressConsent, + assertRequiredCapabilities, + CodeProviderError, + type CodeProvider, + type CodeProviderCapability, + type CodeSearchHit, + type OpOptions, + type RepoRef, + type SearchOptions, + type SourceRef, + type SourceStatus, +} from "./contract"; + +const CAPABILITIES: CodeProviderCapability[] = [ + "register_source", + "refresh", + "search", + "status", + "add", + "delete", + "export", +]; + +const DEFAULT_TIMEOUT_MS = 30_000; + +/** + * Parse `gbrain search` text output (`[score] slug -- snippet`) into hits. + * gbrain's search prints text, not JSON (verified in + * lib/gstack-decision-semantic.ts). Exported for deterministic unit testing. + */ +export function parseGbrainSearch(stdout: string, minScore: number, limit: number): CodeSearchHit[] { + const hits: CodeSearchHit[] = []; + for (const line of stdout.split("\n")) { + const m = line.match(/^\[([\d.]+)\]\s+(\S+)\s+--\s+(.*)$/); + if (!m) continue; + const score = parseFloat(m[1]); + if (!Number.isFinite(score) || score < minScore) continue; + hits.push({ ref: m[2], score, snippet: m[3].trim(), kind: "document" }); + } + return hits.slice(0, limit); +} + +export class GbrainProvider implements CodeProvider { + readonly id = "gbrain" as const; + readonly label = "GBrain"; + readonly capabilities = new Set(CAPABILITIES); + /** GBrain federates into a (possibly remote) DB, so content can leave the machine. */ + readonly local = false; + + constructor() { + assertRequiredCapabilities(this.id, this.capabilities); + } + + has(capability: CodeProviderCapability): boolean { + return this.capabilities.has(capability); + } + + async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise { + assertEgressConsent(this, opts); + try { + const result = await ensureSourceRegistered(repo.id, repo.path, { + federated: true, + env: opts.env, + }); + return { + id: repo.id, + state: result.state.status === "match" ? "registered" : "unknown", + detail: result.changed ? "registered" : "already registered", + }; + } catch (err) { + throw this.#wrap(err); + } + } + + async refresh(source: SourceRef, opts: OpOptions = {}): Promise { + assertEgressConsent(this, opts); + this.#assertOk(spawnGbrain(["sync", "--strategy", "code", "--source", source.id], { + baseEnv: opts.env, + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + })); + return this.status(source, opts); + } + + async search(query: string, opts: SearchOptions = {}): Promise { + if (!query.trim()) return []; + const args = ["search", query]; + if (opts.source) args.push("--source", opts.source); + const r = spawnGbrain(args, { baseEnv: opts.env, timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS }); + this.#assertOk(r); + return parseGbrainSearch(r.stdout || "", opts.minScore ?? 0.1, opts.limit ?? 10); + } + + async status(source?: SourceRef, opts: OpOptions = {}): Promise { + if (!source) { + // No source given: liveness probe. `sources list` reachable = ready. + this.#assertOk(spawnGbrain(["sources", "list", "--json"], { + baseEnv: opts.env, + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + })); + return { id: "*", state: "ready" }; + } + try { + const probed = probeSource(source.id, opts.env); + if (probed.status === "absent") return { id: source.id, state: "absent" }; + const count = sourcePageCount(source.id, opts.env); + return { + id: source.id, + state: "ready", + itemCount: count ?? undefined, + detail: probed.registered_path, + }; + } catch (err) { + throw this.#wrap(err); + } + } + + async add(doc: { slug: string; body: string }, opts: OpOptions = {}): Promise { + assertCapability(this, "add"); + assertEgressConsent(this, opts); + const r = spawnGbrain(["put", doc.slug, "--body", doc.body], { + baseEnv: opts.env, + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + }); + this.#assertOk(r); + return { id: doc.slug, state: "ready" }; + } + + async delete(slug: string, opts: OpOptions = {}): Promise { + assertCapability(this, "delete"); + this.#assertOk(spawnGbrain(["delete", slug, "--yes"], { + baseEnv: opts.env, + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + })); + return { id: slug, state: "absent" }; + } + + async export(source: SourceRef, opts: OpOptions = {}): Promise { + assertCapability(this, "export"); + const r = spawnGbrain(["export", "--source", source.id], { + baseEnv: opts.env, + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + }); + this.#assertOk(r); + return r.stdout || ""; + } + + /** + * Throw a typed failure unless the spawn succeeded. Distinguishes a missing + * CLI (ENOENT → PROVIDER_UNAVAILABLE, the degrade signal) from a timeout + * (ETIMEDOUT/SIGTERM, status=null) and a real non-zero exit. + */ + #assertOk(r: { + status: number | null; + stderr?: string; + error?: Error & { code?: string }; + signal?: NodeJS.Signals | null; + }): void { + if (r.status === 0) return; + const stderr = (r.stderr || "").trim(); + if (r.error?.code === "ENOENT" || /command not found/.test(stderr)) { + throw new CodeProviderError("PROVIDER_UNAVAILABLE", "gbrain CLI not on PATH", this.id); + } + if (r.error?.code === "ETIMEDOUT" || r.signal === "SIGTERM") { + throw new CodeProviderError("PROVIDER_TIMEOUT", "gbrain timed out", this.id); + } + if (/not configured|Cannot connect to database|config\.json/.test(stderr)) { + throw new CodeProviderError("PROVIDER_UNAVAILABLE", stderr || "gbrain not configured", this.id); + } + throw new CodeProviderError("PROVIDER_ERROR", stderr || `gbrain exited ${r.status}`, this.id); + } + + #wrap(err: unknown): CodeProviderError { + if (err instanceof CodeProviderError) return err; + const message = err instanceof Error ? err.message : String(err); + if (/not on PATH|command not found/.test(message)) { + return new CodeProviderError("PROVIDER_UNAVAILABLE", message, this.id); + } + if (/not configured/.test(message)) { + return new CodeProviderError("PROVIDER_UNAVAILABLE", message, this.id); + } + return new CodeProviderError("PROVIDER_ERROR", message, this.id); + } +} diff --git a/lib/code-intelligence/index.ts b/lib/code-intelligence/index.ts new file mode 100644 index 0000000000..48a58f0c35 --- /dev/null +++ b/lib/code-intelligence/index.ts @@ -0,0 +1,14 @@ +/** + * code-intelligence — the OPTIONAL, repo-oriented provider contract. + * See docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md. + */ + +export * from "./contract"; +export { GbrainProvider, parseGbrainSearch } from "./gbrain-adapter"; +export { SourcebotProvider, GraphifyProvider } from "./mcp-adapters"; +export { + recommendCodeProvider, + resolveCodeProvider, + RECOMMENDED_ORDER, + type PickerOptions, +} from "./picker"; diff --git a/lib/code-intelligence/picker.ts b/lib/code-intelligence/picker.ts new file mode 100644 index 0000000000..774cb59e72 --- /dev/null +++ b/lib/code-intelligence/picker.ts @@ -0,0 +1,43 @@ +/** + * Picker — recommends a code-intelligence provider, GBrain first. + * + * RECOMMENDED_ORDER is the static "GBrain first" fact the options UX and phase-2 + * resolution filter against. In phase 1 only GBrain is drivable from the runtime + * (it has a CLI the runtime can spawn); Sourcebot and Graphify are MCP tools in + * the host session and become resolvable in phase 2 once a host transport is + * wired. So recommendCodeProvider returns GBrain-or-nothing today, and + * resolveCodeProvider degrades to null — the provider-OFF path — when GBrain is + * unavailable. Callers then use grep / the file-only decision store. + * + * GBrain availability uses the real detector, localEngineStatus() ("ok"/"timeout" + * are usable). Graphify is NEVER auto-installed or auto-offered. + */ + +import { localEngineStatus, type LocalEngineStatus } from "../gbrain-local-status"; +import { GbrainProvider } from "./gbrain-adapter"; +import type { CodeProvider, CodeProviderId } from "./contract"; + +/** Recommendation order — GBrain first. Sourcebot/Graphify join in phase 2. */ +export const RECOMMENDED_ORDER: readonly CodeProviderId[] = ["gbrain", "sourcebot", "graphify"]; + +export interface PickerOptions { + env?: NodeJS.ProcessEnv; + /** Inject GBrain status for tests; otherwise probed via localEngineStatus(). */ + gbrainStatus?: LocalEngineStatus; +} + +const GBRAIN_USABLE: ReadonlySet = new Set(["ok", "timeout"]); + +/** Drivable providers, in recommendation order (GBrain first). */ +export function recommendCodeProvider(opts: PickerOptions = {}): CodeProvider[] { + const status = opts.gbrainStatus ?? localEngineStatus({ env: opts.env }); + return GBRAIN_USABLE.has(status) ? [new GbrainProvider()] : []; +} + +/** + * The single recommended provider, or null when none is drivable. Null is the + * provider-OFF path: callers MUST degrade to grep / file-only, never fail. + */ +export function resolveCodeProvider(opts: PickerOptions = {}): CodeProvider | null { + return recommendCodeProvider(opts)[0] ?? null; +} diff --git a/test/code-intelligence.test.ts b/test/code-intelligence.test.ts new file mode 100644 index 0000000000..953824c724 --- /dev/null +++ b/test/code-intelligence.test.ts @@ -0,0 +1,218 @@ +/** + * Tests for lib/code-intelligence — the OPTIONAL, repo-oriented provider contract. + * + * Load-bearing properties: + * - Every provider advertises the four required capabilities; optional ops are + * declined with a typed CAPABILITY_UNSUPPORTED, never a silent no-op. + * - Sourcebot/Graphify are phase-1 capability declarations: they prove contract + * fit and throw PROVIDER_UNAVAILABLE until a host MCP transport is wired. + * - The picker recommends GBrain first and resolves to null (provider-OFF) when + * GBrain is unavailable. + * - Non-local providers refuse to move repo content off the machine without + * explicit per-repo consent (PROVIDER_NOT_CONSENTED). + * - The GBrain adapter works end-to-end against a fake `gbrain` shim on PATH. + * - Missing CLI degrades (PROVIDER_UNAVAILABLE), never crashes. + */ + +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + assertRequiredCapabilities, + CodeProviderError, + REQUIRED_CAPABILITIES, + GbrainProvider, + SourcebotProvider, + GraphifyProvider, + parseGbrainSearch, + recommendCodeProvider, + resolveCodeProvider, + RECOMMENDED_ORDER, +} from "../lib/code-intelligence"; + +describe("capability matrix invariants", () => { + test("every provider advertises the four required capabilities", () => { + for (const p of [new GbrainProvider(), new SourcebotProvider(), new GraphifyProvider()]) { + for (const cap of REQUIRED_CAPABILITIES) expect(p.has(cap)).toBe(true); + } + }); + + test("GBrain advertises all seven (document axis)", () => { + const g = new GbrainProvider(); + for (const cap of ["add", "delete", "export"] as const) expect(g.has(cap)).toBe(true); + expect(g.local).toBe(false); + }); + + test("Sourcebot is search-only: declines add/delete/export", () => { + const s = new SourcebotProvider(); + expect(s.has("add")).toBe(false); + expect(s.has("delete")).toBe(false); + expect(s.has("export")).toBe(false); + expect(s.local).toBe(false); + }); + + test("Graphify is local, exports, declines add/delete", () => { + const g = new GraphifyProvider(); + expect(g.local).toBe(true); + expect(g.has("export")).toBe(true); + expect(g.has("add")).toBe(false); + expect(g.has("delete")).toBe(false); + }); + + test("assertRequiredCapabilities rejects an incomplete provider", () => { + expect(() => assertRequiredCapabilities("sourcebot", new Set(["search"]))).toThrow(/missing required/); + }); + + test("CodeProviderError rejects an unknown failure code", () => { + // @ts-expect-error deliberately invalid code + expect(() => new CodeProviderError("NOPE", "x")).toThrow(/Unknown code-provider failure/); + }); +}); + +describe("optional ops decline with a typed failure", () => { + test("Sourcebot declines add/delete/export with CAPABILITY_UNSUPPORTED", async () => { + const s = new SourcebotProvider(); + await expect(s.add({ slug: "x", body: "y" })).rejects.toMatchObject({ code: "CAPABILITY_UNSUPPORTED" }); + await expect(s.delete("x")).rejects.toMatchObject({ code: "CAPABILITY_UNSUPPORTED" }); + await expect(s.export({ id: "x" })).rejects.toMatchObject({ code: "CAPABILITY_UNSUPPORTED" }); + }); + + test("Graphify advertises export → not CAPABILITY_UNSUPPORTED, but unwired in phase 1", async () => { + await expect(new GraphifyProvider().export({ id: "x" })).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); + }); +}); + +describe("phase-1 declaration adapters degrade to PROVIDER_UNAVAILABLE", () => { + test("Sourcebot.search is unwired until a transport lands", async () => { + await expect(new SourcebotProvider().search("q")).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); + }); + + test("empty query short-circuits to no hits (no throw)", async () => { + expect(await new SourcebotProvider().search(" ")).toEqual([]); + }); + + test("status is partial + unknown (no live probe yet)", async () => { + const s = await new GraphifyProvider().status({ id: "repo" }); + expect(s).toMatchObject({ id: "repo", state: "unknown", partial: true }); + }); +}); + +describe("egress consent gate", () => { + test("non-local registerSource without consent → PROVIDER_NOT_CONSENTED", async () => { + await expect(new SourcebotProvider().registerSource({ id: "repo", path: "/repo" })).rejects.toMatchObject({ + code: "PROVIDER_NOT_CONSENTED", + }); + }); + + test("local provider (Graphify) skips the egress gate (falls through to unwired)", async () => { + // Local means nothing leaves the machine, so consent is not required — it + // reaches the phase-1 PROVIDER_UNAVAILABLE, NOT a consent rejection. + await expect(new GraphifyProvider().registerSource({ id: "repo", path: "/repo" })).rejects.toMatchObject({ + code: "PROVIDER_UNAVAILABLE", + }); + }); +}); + +describe("parseGbrainSearch (text surface)", () => { + const sample = ["[0.91] slug/a -- snippet one", "banner", "[0.05] slug/b -- below floor"].join("\n"); + test("parses scored lines, applies floor + limit", () => { + expect(parseGbrainSearch(sample, 0.1, 10)).toEqual([ + { ref: "slug/a", score: 0.91, snippet: "snippet one", kind: "document" }, + ]); + expect(parseGbrainSearch(sample, 0.0, 1)).toHaveLength(1); + }); +}); + +describe("picker — recommend GBrain first", () => { + test("RECOMMENDED_ORDER puts GBrain first", () => { + expect(RECOMMENDED_ORDER[0]).toBe("gbrain"); + expect([...RECOMMENDED_ORDER]).toEqual(["gbrain", "sourcebot", "graphify"]); + }); + + test("GBrain resolves when usable", () => { + expect(recommendCodeProvider({ gbrainStatus: "ok" }).map((p) => p.id)).toEqual(["gbrain"]); + expect(resolveCodeProvider({ gbrainStatus: "ok" })?.id).toBe("gbrain"); + }); + + test("timeout status counts as usable (engine slow, not absent)", () => { + expect(resolveCodeProvider({ gbrainStatus: "timeout" })?.id).toBe("gbrain"); + }); + + test("provider-OFF: GBrain down → resolveCodeProvider null", () => { + expect(recommendCodeProvider({ gbrainStatus: "no-cli" })).toEqual([]); + expect(resolveCodeProvider({ gbrainStatus: "no-cli" })).toBeNull(); + expect(resolveCodeProvider({ gbrainStatus: "missing-config" })).toBeNull(); + }); +}); + +describe("GBrain adapter end-to-end (fake shim on PATH)", () => { + let binDir: string; + let homeDir: string; + + function writeShim(body: string): void { + const p = path.join(binDir, "gbrain"); + fs.writeFileSync(p, body, { mode: 0o755 }); + fs.chmodSync(p, 0o755); + } + function env(): NodeJS.ProcessEnv { + return { PATH: `${binDir}:${process.env.PATH}`, HOME: homeDir }; + } + + beforeEach(() => { + binDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gbrain-shim-")); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gbrain-home-")); + }); + afterEach(() => { + fs.rmSync(binDir, { recursive: true, force: true }); + fs.rmSync(homeDir, { recursive: true, force: true }); + }); + + test("search scopes to source and parses hits", async () => { + writeShim(`#!/usr/bin/env bash +if [ "$1" = "search" ]; then + if printf '%s ' "$@" | grep -q -- "--source code"; then + echo "[0.88] src/x.ts -- match in code source" + else + echo "[0.10] wrong -- unscoped" + fi + exit 0 +fi +exit 1 +`); + const hits = await new GbrainProvider().search("where", { env: env(), source: "code" }); + expect(hits).toHaveLength(1); + expect(hits[0].ref).toBe("src/x.ts"); + }); + + test("status(source) reports ready + page_count", async () => { + writeShim(`#!/usr/bin/env bash +if [ "$1" = "sources" ]; then echo '{"sources":[{"id":"code","local_path":"/repo","page_count":42}]}'; exit 0; fi +exit 1 +`); + const s = await new GbrainProvider().status({ id: "code" }, { env: env() }); + expect(s.state).toBe("ready"); + expect(s.itemCount).toBe(42); + }); + + test("status(source) reports absent for an unregistered id", async () => { + writeShim(`#!/usr/bin/env bash +if [ "$1" = "sources" ]; then echo '{"sources":[]}'; exit 0; fi +exit 1 +`); + expect((await new GbrainProvider().status({ id: "nope" }, { env: env() })).state).toBe("absent"); + }); + + test("missing CLI degrades to PROVIDER_UNAVAILABLE", async () => { + // No shim written; PATH points only at an empty dir. + await expect( + new GbrainProvider().search("q", { env: { PATH: binDir, HOME: homeDir } }), + ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); + }); + + test("registerSource requires egress consent (GBrain is non-local)", async () => { + await expect( + new GbrainProvider().registerSource({ id: "code", path: "/repo" }, { env: env() }), + ).rejects.toMatchObject({ code: "PROVIDER_NOT_CONSENTED" }); + }); +}); From 3eb4a1b23fb3219327eab29c8f87d61502172177 Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:16:54 -0700 Subject: [PATCH 2/9] feat: code-intelligence contract with real GBrain/Graphify/Sourcebot adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit contract.ts: repo-oriented interface (four required ops, three optional), typed CodeProviderError, egress + capability consent guards. Three real, runtime-drivable adapters: - GbrainProvider: gbrain CLI (reuses lib/gbrain-exec + lib/gbrain-sources), all seven capabilities. - GraphifyProvider: graphify CLI — `graphify ` builds the local graph, `graphify query` searches it, export reads graphify-out/graph.json. Fully local; never auto-installed. - SourcebotProvider: self-hosted server over HTTP — register writes a local git connection to config.json, search is POST /api/search, status is a liveness probe. Loopback base URL = local (no egress); remote = consent. selection.ts persists the chosen provider + per-repo indexing consent under $GSTACK_HOME. picker.ts recommends GBrain first, resolves the selected provider or null (provider-OFF), and probes live availability. Every adapter degrades to PROVIDER_UNAVAILABLE when its tool/server is absent. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/code-intelligence/graphify-adapter.ts | 156 ++++++++++++++++ lib/code-intelligence/index.ts | 16 +- lib/code-intelligence/picker.ts | 96 +++++++--- lib/code-intelligence/selection.ts | 68 +++++++ lib/code-intelligence/sourcebot-adapter.ts | 197 +++++++++++++++++++++ 5 files changed, 507 insertions(+), 26 deletions(-) create mode 100644 lib/code-intelligence/graphify-adapter.ts create mode 100644 lib/code-intelligence/selection.ts create mode 100644 lib/code-intelligence/sourcebot-adapter.ts diff --git a/lib/code-intelligence/graphify-adapter.ts b/lib/code-intelligence/graphify-adapter.ts new file mode 100644 index 0000000000..3cbcf492c8 --- /dev/null +++ b/lib/code-intelligence/graphify-adapter.ts @@ -0,0 +1,156 @@ +/** + * Graphify adapter — real CLI integration (github.com/Graphify-Labs/graphify). + * + * Graphify is a LOCAL tree-sitter knowledge graph: `graphify ` builds a + * `graphify-out/` (graph.json + report) in that dir, `graphify query ""` + * queries it, and nothing leaves the machine (no embeddings, no network). So it + * needs no egress consent and no MCP — the runtime just shells out to the CLI, + * the same shape as the GBrain adapter. + * + * Never auto-installed: install is `pip install graphifyy && graphify install`, + * a user action the picker gates on. When the CLI is absent every op throws + * PROVIDER_UNAVAILABLE and callers degrade to file-only. + * + * Path-based, not id-based: for Graphify a source "id" IS the absolute repo path + * (that is where `graphify-out/` lives), unlike GBrain's short source ids. + */ + +import { spawnSync } from "child_process"; +import { existsSync, readFileSync } from "fs"; +import { join } from "path"; +import { + assertCapability, + assertRequiredCapabilities, + CodeProviderError, + type CodeProvider, + type CodeProviderCapability, + type CodeSearchHit, + type OpOptions, + type RepoRef, + type SearchOptions, + type SourceRef, + type SourceStatus, +} from "./contract"; + +const CAPABILITIES: CodeProviderCapability[] = ["register_source", "refresh", "search", "status", "export"]; +const OUT_DIR = "graphify-out"; +const GRAPH_JSON = "graph.json"; +const DEFAULT_TIMEOUT_MS = 120_000; // indexing a repo can take a while +const NEEDS_SHELL_ON_WINDOWS = process.platform === "win32"; // graphify is a shim on Windows + +export interface GraphifyOptions { + /** Directory whose `graphify-out/` search/status/export read. Defaults to cwd. */ + root?: string; + env?: NodeJS.ProcessEnv; +} + +export class GraphifyProvider implements CodeProvider { + readonly id = "graphify" as const; + readonly label = "Graphify"; + readonly capabilities = new Set(CAPABILITIES); + /** Fully local — no repo content leaves the machine. */ + readonly local = true; + readonly #root: string; + readonly #env?: NodeJS.ProcessEnv; + + constructor(opts: GraphifyOptions = {}) { + this.#root = opts.root ?? process.cwd(); + this.#env = opts.env; + assertRequiredCapabilities(this.id, this.capabilities); + } + + has(capability: CodeProviderCapability): boolean { + return this.capabilities.has(capability); + } + + #run(args: string[], cwd: string, timeout: number) { + return spawnSync("graphify", args, { + cwd, + encoding: "utf-8", + timeout, + stdio: ["ignore", "pipe", "pipe"], + env: this.#env, + shell: NEEDS_SHELL_ON_WINDOWS, + }); + } + + #assertOk(r: { status: number | null; stderr?: string; error?: Error & { code?: string }; signal?: NodeJS.Signals | null }): void { + if (r.status === 0) return; + const stderr = (r.stderr || "").trim(); + if (r.error?.code === "ENOENT" || /command not found/.test(stderr)) { + throw new CodeProviderError("PROVIDER_UNAVAILABLE", "graphify CLI not on PATH (install: pip install graphifyy && graphify install)", this.id); + } + if (r.error?.code === "ETIMEDOUT" || r.signal === "SIGTERM") { + throw new CodeProviderError("PROVIDER_TIMEOUT", "graphify timed out", this.id); + } + throw new CodeProviderError("PROVIDER_ERROR", stderr || `graphify exited ${r.status}`, this.id); + } + + /** Build the graph over repo.path (local; no egress consent needed). */ + async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise { + this.#assertOk(this.#run(["."], repo.path, opts.timeout ?? DEFAULT_TIMEOUT_MS)); + return this.status({ id: repo.path }, opts); + } + + /** Re-parse and merge changes into the existing graph. */ + async refresh(source: SourceRef, opts: OpOptions = {}): Promise { + this.#assertOk(this.#run([".", "--update"], source.id, opts.timeout ?? DEFAULT_TIMEOUT_MS)); + return this.status(source, opts); + } + + /** + * `graphify query ""` returns a traced answer over the graph. Its stdout is + * an answer, not a fixed hit schema (the CLI reference documents the command + * but not a machine format), so we map non-empty output lines to hits + * tolerantly: a leading path-like token becomes the ref, the line the snippet. + * Reconcile against a live graphify if a stricter schema is needed. + */ + async search(query: string, opts: SearchOptions = {}): Promise { + if (!query.trim()) return []; + const cwd = opts.source ?? this.#root; + const r = this.#run(["query", query], cwd, opts.timeout ?? DEFAULT_TIMEOUT_MS); + this.#assertOk(r); + return parseGraphifyQuery(r.stdout || "", opts.limit ?? 10); + } + + async status(source?: SourceRef, _opts: OpOptions = {}): Promise { + const dir = source?.id ?? this.#root; + const graphPath = join(dir, OUT_DIR, GRAPH_JSON); + if (!existsSync(graphPath)) return { id: dir, state: "absent" }; + let itemCount: number | undefined; + try { + const graph = JSON.parse(readFileSync(graphPath, "utf-8")) as { nodes?: unknown[] }; + if (Array.isArray(graph.nodes)) itemCount = graph.nodes.length; + } catch { + // graph.json present but unparseable — still ready, just no count. + } + return { id: dir, state: "ready", itemCount, detail: graphPath }; + } + + async export(source: SourceRef, _opts: OpOptions = {}): Promise { + assertCapability(this, "export"); + const graphPath = join(source.id, OUT_DIR, GRAPH_JSON); + if (!existsSync(graphPath)) { + throw new CodeProviderError("SOURCE_NOT_REGISTERED", `no graph at ${graphPath}; index it first`, this.id); + } + return readFileSync(graphPath, "utf-8"); + } +} + +/** + * Map `graphify query` stdout to hits. Tolerant: each non-empty line becomes a + * hit; a leading `path` or `path:line` token becomes the ref, else the whole + * line is the snippet. Exported for deterministic unit testing. + */ +export function parseGraphifyQuery(stdout: string, limit: number): CodeSearchHit[] { + const hits: CodeSearchHit[] = []; + for (const raw of stdout.split("\n")) { + const line = raw.trim(); + if (!line) continue; + const token = line.split(/\s+/)[0]; + const looksPath = /[/\\.]/.test(token) && !token.includes(" "); + hits.push({ ref: looksPath ? token : "graphify", snippet: line, kind: "graph-node" }); + if (hits.length >= limit) break; + } + return hits; +} diff --git a/lib/code-intelligence/index.ts b/lib/code-intelligence/index.ts index 48a58f0c35..0c69719c8d 100644 --- a/lib/code-intelligence/index.ts +++ b/lib/code-intelligence/index.ts @@ -5,10 +5,20 @@ export * from "./contract"; export { GbrainProvider, parseGbrainSearch } from "./gbrain-adapter"; -export { SourcebotProvider, GraphifyProvider } from "./mcp-adapters"; +export { GraphifyProvider, parseGraphifyQuery, type GraphifyOptions } from "./graphify-adapter"; +export { SourcebotProvider, parseSourcebotSearch, type SourcebotOptions } from "./sourcebot-adapter"; +export { + readSelection, + setProvider, + setConsent, + hasConsent, + type Selection, +} from "./selection"; export { - recommendCodeProvider, - resolveCodeProvider, RECOMMENDED_ORDER, + providerById, + resolveSelectedProvider, + detectAvailable, type PickerOptions, + type Availability, } from "./picker"; diff --git a/lib/code-intelligence/picker.ts b/lib/code-intelligence/picker.ts index 774cb59e72..0d31d1c281 100644 --- a/lib/code-intelligence/picker.ts +++ b/lib/code-intelligence/picker.ts @@ -1,43 +1,93 @@ /** - * Picker — recommends a code-intelligence provider, GBrain first. + * Picker — constructs the code-intelligence provider the user selected, and + * offers the recommendation order (GBrain first) for the selection UX. * - * RECOMMENDED_ORDER is the static "GBrain first" fact the options UX and phase-2 - * resolution filter against. In phase 1 only GBrain is drivable from the runtime - * (it has a CLI the runtime can spawn); Sourcebot and Graphify are MCP tools in - * the host session and become resolvable in phase 2 once a host transport is - * wired. So recommendCodeProvider returns GBrain-or-nothing today, and - * resolveCodeProvider degrades to null — the provider-OFF path — when GBrain is - * unavailable. Callers then use grep / the file-only decision store. + * `resolveSelectedProvider()` reads the persisted selection and constructs that + * provider, or returns null when nothing is selected — the provider-OFF path, + * where callers degrade to grep / the file-only decision store. Availability is + * proven at call time: a selected provider whose tool/server is absent throws + * PROVIDER_UNAVAILABLE from its ops, which callers catch and degrade on. The + * `detectAvailable()` probe drives the `options`/`status` display. * - * GBrain availability uses the real detector, localEngineStatus() ("ok"/"timeout" - * are usable). Graphify is NEVER auto-installed or auto-offered. + * GBrain is recommended first. Graphify is NEVER auto-installed — it appears in + * the options only once its CLI is present (a user install). */ -import { localEngineStatus, type LocalEngineStatus } from "../gbrain-local-status"; +import { localEngineStatus } from "../gbrain-local-status"; import { GbrainProvider } from "./gbrain-adapter"; +import { GraphifyProvider, type GraphifyOptions } from "./graphify-adapter"; +import { SourcebotProvider, type SourcebotOptions } from "./sourcebot-adapter"; +import { readSelection } from "./selection"; import type { CodeProvider, CodeProviderId } from "./contract"; -/** Recommendation order — GBrain first. Sourcebot/Graphify join in phase 2. */ +/** Recommendation order — GBrain first. */ export const RECOMMENDED_ORDER: readonly CodeProviderId[] = ["gbrain", "sourcebot", "graphify"]; export interface PickerOptions { env?: NodeJS.ProcessEnv; - /** Inject GBrain status for tests; otherwise probed via localEngineStatus(). */ - gbrainStatus?: LocalEngineStatus; + graphify?: GraphifyOptions; + sourcebot?: SourcebotOptions; } -const GBRAIN_USABLE: ReadonlySet = new Set(["ok", "timeout"]); +/** Construct a provider by id (no availability check — ops degrade at call time). */ +export function providerById(id: CodeProviderId, opts: PickerOptions = {}): CodeProvider { + switch (id) { + case "gbrain": + return new GbrainProvider(); + case "graphify": + return new GraphifyProvider({ env: opts.env, ...opts.graphify }); + case "sourcebot": + return new SourcebotProvider({ env: opts.env, ...opts.sourcebot }); + } +} + +/** + * The provider the user selected, constructed, or null when none is selected. + * Null is the provider-OFF path: callers MUST degrade to grep / file-only. + */ +export function resolveSelectedProvider(opts: PickerOptions = {}): CodeProvider | null { + const { provider } = readSelection(opts.env); + return provider ? providerById(provider, opts) : null; +} -/** Drivable providers, in recommendation order (GBrain first). */ -export function recommendCodeProvider(opts: PickerOptions = {}): CodeProvider[] { - const status = opts.gbrainStatus ?? localEngineStatus({ env: opts.env }); - return GBRAIN_USABLE.has(status) ? [new GbrainProvider()] : []; +export interface Availability { + id: CodeProviderId; + available: boolean; + detail: string; } /** - * The single recommended provider, or null when none is drivable. Null is the - * provider-OFF path: callers MUST degrade to grep / file-only, never fail. + * Probe which providers are usable right now, in recommendation order. Used by + * the `options`/`status` display. GBrain via the real localEngineStatus(); + * Graphify via its CLI status; Sourcebot via an HTTP liveness probe. */ -export function resolveCodeProvider(opts: PickerOptions = {}): CodeProvider | null { - return recommendCodeProvider(opts)[0] ?? null; +export async function detectAvailable(opts: PickerOptions = {}): Promise { + const gbrainStatus = localEngineStatus({ env: opts.env }); + const gbrainOk = gbrainStatus === "ok" || gbrainStatus === "timeout"; + + let graphifyOk = false; + let graphifyDetail = "graphify CLI not installed"; + try { + const s = await new GraphifyProvider({ env: opts.env, ...opts.graphify }).status(); + graphifyOk = s.state === "ready"; + graphifyDetail = graphifyOk ? "graph indexed in this repo" : "installed; no graph in this repo yet"; + } catch { + graphifyOk = false; + } + + let sourcebotOk = false; + let sourcebotDetail = "server unreachable"; + try { + const s = await new SourcebotProvider({ env: opts.env, ...opts.sourcebot }).status(); + sourcebotOk = s.state === "ready"; + sourcebotDetail = s.detail ?? ""; + } catch { + sourcebotOk = false; + } + + return [ + { id: "gbrain", available: gbrainOk, detail: `gbrain engine: ${gbrainStatus}` }, + { id: "sourcebot", available: sourcebotOk, detail: sourcebotDetail }, + { id: "graphify", available: graphifyOk, detail: graphifyDetail }, + ]; } diff --git a/lib/code-intelligence/selection.ts b/lib/code-intelligence/selection.ts new file mode 100644 index 0000000000..c6657aa06b --- /dev/null +++ b/lib/code-intelligence/selection.ts @@ -0,0 +1,68 @@ +/** + * selection — persists the user's chosen code-intelligence provider and their + * per-repo indexing consent. Stored at `$GSTACK_HOME/code-intelligence.json` + * (default `~/.gstack/`), the same home the rest of gstack uses. + * + * Consent is per-repo (keyed by absolute repo path), because indexing consent + * is "may THIS repo's content be indexed by the selected provider" — a decision + * a user makes per project, not once for the machine. No selection at all is the + * provider-OFF default: callers degrade to grep / the file-only decision store. + */ + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs"; +import { homedir } from "os"; +import { dirname, join, resolve } from "path"; +import type { CodeProviderId } from "./contract"; + +export interface Selection { + provider: CodeProviderId | null; + /** Absolute repo path → consented. */ + consents: Record; +} + +const EMPTY: Selection = { provider: null, consents: {} }; + +function storePath(env: NodeJS.ProcessEnv = process.env): string { + const home = env.GSTACK_HOME || join(env.HOME || homedir(), ".gstack"); + return join(home, "code-intelligence.json"); +} + +export function readSelection(env: NodeJS.ProcessEnv = process.env): Selection { + const p = storePath(env); + if (!existsSync(p)) return { ...EMPTY }; + try { + const raw = JSON.parse(readFileSync(p, "utf-8")) as Partial; + return { + provider: raw.provider ?? null, + consents: raw.consents && typeof raw.consents === "object" ? raw.consents : {}, + }; + } catch { + return { ...EMPTY }; + } +} + +function write(selection: Selection, env: NodeJS.ProcessEnv = process.env): void { + const p = storePath(env); + mkdirSync(dirname(p), { recursive: true }); + const tmp = `${p}.tmp.${process.pid}`; + writeFileSync(tmp, JSON.stringify(selection, null, 2), "utf-8"); + renameSync(tmp, p); +} + +export function setProvider(provider: CodeProviderId | null, env: NodeJS.ProcessEnv = process.env): Selection { + const next = { ...readSelection(env), provider }; + write(next, env); + return next; +} + +/** Record per-repo indexing consent (repo path resolved to absolute). */ +export function setConsent(repoPath: string, consented: boolean, env: NodeJS.ProcessEnv = process.env): Selection { + const current = readSelection(env); + const next: Selection = { ...current, consents: { ...current.consents, [resolve(repoPath)]: consented } }; + write(next, env); + return next; +} + +export function hasConsent(repoPath: string, env: NodeJS.ProcessEnv = process.env): boolean { + return readSelection(env).consents[resolve(repoPath)] === true; +} diff --git a/lib/code-intelligence/sourcebot-adapter.ts b/lib/code-intelligence/sourcebot-adapter.ts new file mode 100644 index 0000000000..b6d7d9fc05 --- /dev/null +++ b/lib/code-intelligence/sourcebot-adapter.ts @@ -0,0 +1,197 @@ +/** + * Sourcebot adapter — real HTTP + config integration + * (github.com/sourcebot-dev/sourcebot, YC F2025). + * + * Sourcebot is a self-hosted server that indexes repos declared in its + * config.json and serves regex code search over `POST /api/search` (zoekt). So + * the runtime drives it with plain HTTP + a config-file edit — no MCP: + * - register_source: add `{ "type": "git", "url": "file:///abs/path" }` to the + * server's config.json (it re-indexes automatically on config change). + * - refresh: Sourcebot re-indexes on config change and on reindexIntervalMs; + * there is no per-source trigger endpoint, so refresh reports current status. + * - search: POST /api/search with a regex query, map files[] to hits. + * - status: liveness GET against the base URL. + * Declines the document ops (add/delete/export) — it is a whole-repo search index. + * + * Egress: a loopback base URL means the index runs on this machine, so no repo + * content leaves it (local=true). A non-loopback base URL means content reaches + * another host, so egress consent is required (local=false). + */ + +import { existsSync, readFileSync, writeFileSync, renameSync } from "fs"; +import { + assertCapability, + assertEgressConsent, + assertRequiredCapabilities, + CodeProviderError, + type CodeProvider, + type CodeProviderCapability, + type CodeSearchHit, + type OpOptions, + type RepoRef, + type SearchOptions, + type SourceRef, + type SourceStatus, +} from "./contract"; + +const CAPABILITIES: CodeProviderCapability[] = ["register_source", "refresh", "search", "status"]; +const DEFAULT_URL = "http://localhost:3000"; +const DEFAULT_TIMEOUT_MS = 30_000; + +type FetchLike = typeof globalThis.fetch; + +export interface SourcebotOptions { + /** Base URL of the Sourcebot server. Defaults to SOURCEBOT_URL or http://localhost:3000. */ + baseUrl?: string; + /** Path to the server's config.json (for register_source). Defaults to SOURCEBOT_CONFIG. */ + configPath?: string; + /** Injectable fetch for tests. */ + fetch?: FetchLike; + env?: NodeJS.ProcessEnv; +} + +function isLoopback(url: string): boolean { + try { + const h = new URL(url).hostname.toLowerCase(); + return h === "localhost" || h === "127.0.0.1" || h === "::1" || h.endsWith(".localhost"); + } catch { + return false; + } +} + +export class SourcebotProvider implements CodeProvider { + readonly id = "sourcebot" as const; + readonly label = "Sourcebot"; + readonly capabilities = new Set(CAPABILITIES); + readonly local: boolean; + readonly #baseUrl: string; + readonly #configPath?: string; + readonly #fetch: FetchLike; + + constructor(opts: SourcebotOptions = {}) { + const env = opts.env ?? process.env; + this.#baseUrl = (opts.baseUrl ?? env.SOURCEBOT_URL ?? DEFAULT_URL).replace(/\/$/, ""); + this.#configPath = opts.configPath ?? env.SOURCEBOT_CONFIG; + this.#fetch = opts.fetch ?? globalThis.fetch; + this.local = isLoopback(this.#baseUrl); + assertRequiredCapabilities(this.id, this.capabilities); + } + + has(capability: CodeProviderCapability): boolean { + return this.capabilities.has(capability); + } + + /** Add the repo as a local `git` connection in Sourcebot's config.json. */ + async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise { + assertEgressConsent(this, opts); // no-op when the server is loopback (local) + if (!this.#configPath) { + throw new CodeProviderError( + "PROVIDER_UNAVAILABLE", + "set SOURCEBOT_CONFIG to the server's config.json path to register sources", + this.id, + ); + } + let config: { connections?: Record }; + try { + config = existsSync(this.#configPath) + ? (JSON.parse(readFileSync(this.#configPath, "utf-8")) as typeof config) + : {}; + } catch (err) { + throw new CodeProviderError("PROVIDER_ERROR", `unreadable Sourcebot config: ${(err as Error).message}`, this.id); + } + config.connections = config.connections ?? {}; + config.connections[repo.id] = { type: "git", url: `file://${repo.path}` }; + // Atomic write so a running Sourcebot never reads a half-written config. + const tmp = `${this.#configPath}.tmp.${process.pid}`; + writeFileSync(tmp, JSON.stringify(config, null, 2), "utf-8"); + renameSync(tmp, this.#configPath); + return { id: repo.id, state: "registered", detail: "Sourcebot re-indexes on config change" }; + } + + /** Sourcebot re-indexes automatically; report current liveness. */ + async refresh(source: SourceRef, opts: OpOptions = {}): Promise { + const live = await this.status(source, opts); + return { ...live, detail: "Sourcebot re-indexes automatically (config change + reindexIntervalMs)" }; + } + + async search(query: string, opts: SearchOptions = {}): Promise { + if (!query.trim()) return []; + const body = { + query: opts.source ? `repo:${opts.source} ${query}` : query, + matches: opts.limit ?? 20, + isRegexEnabled: true, + isCaseSensitivityEnabled: false, + }; + const payload = await this.#post("/api/search", body, opts.timeout ?? DEFAULT_TIMEOUT_MS); + return parseSourcebotSearch(payload, opts.limit ?? 20); + } + + async status(_source?: SourceRef, opts: OpOptions = {}): Promise { + try { + const res = await this.#fetchWithTimeout(this.#baseUrl, { method: "GET" }, opts.timeout ?? DEFAULT_TIMEOUT_MS); + return { id: "*", state: res.ok ? "ready" : "unknown", partial: true, detail: `HTTP ${res.status}` }; + } catch { + return { id: "*", state: "unknown", partial: true, detail: `unreachable at ${this.#baseUrl}` }; + } + } + + async #post(path: string, body: unknown, timeout: number): Promise { + let res: Response; + try { + res = await this.#fetchWithTimeout(`${this.#baseUrl}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify(body), + }, timeout); + } catch (err) { + throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot unreachable at ${this.#baseUrl}: ${(err as Error).message}`, this.id); + } + if (!res.ok) throw new CodeProviderError("PROVIDER_ERROR", `Sourcebot ${path} returned HTTP ${res.status}`, this.id); + try { + return await res.json(); + } catch (err) { + throw new CodeProviderError("PROVIDER_ERROR", `Sourcebot returned non-JSON: ${(err as Error).message}`, this.id); + } + } + + async #fetchWithTimeout(url: string, init: RequestInit, timeout: number): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + try { + return await this.#fetch(url, { ...init, signal: controller.signal }); + } catch (err) { + if ((err as Error)?.name === "AbortError") throw new CodeProviderError("PROVIDER_TIMEOUT", "Sourcebot request timed out", this.id); + throw err; + } finally { + clearTimeout(timer); + } + } +} + +interface SourcebotMatchRange { start?: { lineNumber?: number } } +interface SourcebotChunk { content?: string; matchRanges?: SourcebotMatchRange[] } +interface SourcebotFile { fileName?: { text?: string }; repository?: string; chunks?: SourcebotChunk[] } + +/** + * Map a `POST /api/search` response `{ files: [...] }` to hits: one hit per file, + * ref = file path, snippet = first matching chunk, kind = "file". Tolerant of a + * missing/garbage payload (returns []). Exported for deterministic testing. + */ +export function parseSourcebotSearch(payload: unknown, limit: number): CodeSearchHit[] { + const files = (payload as { files?: unknown })?.files; + if (!Array.isArray(files)) return []; + const hits: CodeSearchHit[] = []; + for (const f of files as SourcebotFile[]) { + const ref = f?.fileName?.text; + if (typeof ref !== "string") continue; + const chunk = f.chunks?.[0]; + const line = chunk?.matchRanges?.[0]?.start?.lineNumber; + hits.push({ + ref: typeof line === "number" ? `${ref}:${line}` : ref, + snippet: typeof chunk?.content === "string" ? chunk.content.trim() : undefined, + kind: "file", + }); + if (hits.length >= limit) break; + } + return hits; +} From f0091e09921371a7e1799a9b321f234e5cc060e4 Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:16:54 -0700 Subject: [PATCH 3/9] feat: gstack-code-intelligence CLI to select, index, and search User-facing command tying the contract together: `options`/`status` show providers with GBrain first and live availability, `select ` persists the choice, `consent [path]` records per-repo indexing consent, `index [path]` registers + indexes the repo with the selected provider (refusing non-local providers until consented), and `search ` runs a query, degrading with a clear message when the provider is unavailable. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/gstack-code-intelligence | 159 +++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100755 bin/gstack-code-intelligence diff --git a/bin/gstack-code-intelligence b/bin/gstack-code-intelligence new file mode 100755 index 0000000000..9bcca08ef9 --- /dev/null +++ b/bin/gstack-code-intelligence @@ -0,0 +1,159 @@ +#!/usr/bin/env bun +/** + * gstack-code-intelligence — pick a code-intelligence provider and use it to + * index and search this repo. OPTIONAL: with nothing selected, gstack works + * fine and callers use grep / the file-only decision store. + * + * Usage: + * gstack-code-intelligence options # list providers (GBrain first) + availability + * gstack-code-intelligence status # current selection + availability + * gstack-code-intelligence select + * gstack-code-intelligence consent [repo-path] # allow indexing this repo (per-repo) + * gstack-code-intelligence index [repo-path] # index the repo with the selected provider + * gstack-code-intelligence search # search via the selected provider + * + * Non-local providers (GBrain, or a Sourcebot on a remote host) refuse to index + * until you consent for that repo. Graphify and a localhost Sourcebot are local: + * nothing leaves the machine, so no consent is needed. Graphify is never + * auto-installed. + */ + +import { basename, resolve } from "path"; +import { + CodeProviderError, + RECOMMENDED_ORDER, + detectAvailable, + hasConsent, + providerById, + readSelection, + resolveSelectedProvider, + setConsent, + setProvider, + type CodeProviderId, +} from "../lib/code-intelligence"; + +const PROVIDER_IDS = new Set(["gbrain", "sourcebot", "graphify"]); +const LABEL: Record = { gbrain: "GBrain", sourcebot: "Sourcebot", graphify: "Graphify" }; +const NOTE: Record = { + gbrain: "recommended; federated memory + code (sends content to your GBrain DB)", + sourcebot: "self-hosted whole-repo regex search (local when on localhost)", + graphify: "local tree-sitter code graph, nothing leaves the machine (install it yourself)", +}; + +function out(s: string): void { + process.stdout.write(`${s}\n`); +} +function fail(s: string): never { + process.stderr.write(`gstack-code-intelligence: ${s}\n`); + process.exit(1); +} + +async function cmdOptions(): Promise { + out("Code-intelligence providers (indexing is optional; GBrain recommended):\n"); + const avail = await detectAvailable(); + const byId = new Map(avail.map((a) => [a.id, a])); + for (const id of RECOMMENDED_ORDER) { + const a = byId.get(id); + const mark = a?.available ? "available" : "not available"; + out(` ${id === "gbrain" ? "*" : " "} ${LABEL[id].padEnd(10)} [${mark}] — ${NOTE[id]}`); + if (a?.detail) out(` ${a.detail}`); + } + out("\nSelect one with: gstack-code-intelligence select "); +} + +async function cmdStatus(): Promise { + const sel = readSelection(); + out(`selected: ${sel.provider ?? "none (grep / file-only fallback)"}`); + const avail = await detectAvailable(); + for (const a of avail) out(` ${LABEL[a.id]}: ${a.available ? "available" : "unavailable"} (${a.detail})`); +} + +function cmdSelect(arg: string | undefined): void { + if (arg === "none") { + setProvider(null); + out("code-intelligence provider cleared; gstack uses grep / file-only fallback"); + return; + } + if (!arg || !PROVIDER_IDS.has(arg as CodeProviderId)) { + fail("Usage: select "); + } + const id = arg as CodeProviderId; + setProvider(id); + out(`selected ${LABEL[id]}.`); + const provider = providerById(id); + if (!provider.local) out(`${LABEL[id]} sends repo content off this machine — run \`consent\` in a repo before indexing it.`); +} + +function cmdConsent(pathArg: string | undefined): void { + const repoPath = resolve(pathArg ?? process.cwd()); + setConsent(repoPath, true); + out(`indexing consent recorded for ${repoPath}`); +} + +async function cmdIndex(pathArg: string | undefined): Promise { + const provider = resolveSelectedProvider(); + if (!provider) fail("no provider selected; run `select ` first"); + const repoPath = resolve(pathArg ?? process.cwd()); + const consented = hasConsent(repoPath); + if (!provider!.local && !consented) { + fail(`${provider!.label} would send this repo's content off the machine. Run \`gstack-code-intelligence consent ${repoPath}\` first.`); + } + const repo = { id: basename(repoPath), path: repoPath }; + try { + const registered = await provider!.registerSource(repo, { consented }); + out(`registered ${repo.id} with ${provider!.label} (${registered.state})`); + const refreshed = await provider!.refresh({ id: registered.id }, { consented }); + out(`indexed: ${refreshed.state}${refreshed.itemCount != null ? ` (${refreshed.itemCount} items)` : ""}`); + } catch (err) { + handleProviderError(err, provider!.label); + } +} + +async function cmdSearch(terms: string[]): Promise { + const query = terms.join(" ").trim(); + if (!query) fail("Usage: search "); + const provider = resolveSelectedProvider(); + if (!provider) fail("no provider selected; run `select ` first (or use grep)"); + try { + const hits = await provider!.search(query, { limit: 10 }); + if (!hits.length) { + out("(no results)"); + return; + } + for (const h of hits) out(`${h.score != null ? `[${h.score.toFixed(2)}] ` : ""}${h.ref}${h.snippet ? ` — ${h.snippet}` : ""}`); + } catch (err) { + handleProviderError(err, provider!.label); + } +} + +function handleProviderError(err: unknown, label: string): never { + if (err instanceof CodeProviderError) { + if (err.code === "PROVIDER_UNAVAILABLE") { + fail(`${label} is unavailable (${err.message}). gstack still works — fall back to grep / file-only.`); + } + fail(`${label} ${err.code}: ${err.message}`); + } + fail(err instanceof Error ? err.message : String(err)); +} + +async function main(): Promise { + const [action, ...rest] = process.argv.slice(2); + switch (action) { + case "options": + return cmdOptions(); + case "status": + return cmdStatus(); + case "select": + return cmdSelect(rest[0]); + case "consent": + return cmdConsent(rest[0]); + case "index": + return cmdIndex(rest[0]); + case "search": + return cmdSearch(rest); + default: + fail("Usage: options | status | select | consent [path] | index [path] | search "); + } +} + +main().catch((err) => fail(err instanceof Error ? err.message : String(err))); From 9650e8b05636a618dd76ee080977ba81579e2d7d Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:16:54 -0700 Subject: [PATCH 4/9] test: cover the code-intelligence contract, adapters, selection, and consent Capability matrix, result parsers, selection store + per-repo consent + provider-OFF, egress gating, and each adapter end-to-end against a fake CLI shim (gbrain, graphify) or injected fetch + temp config.json (sourcebot), plus PROVIDER_UNAVAILABLE degrade for every provider. 19 tests, no live tools required. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/code-intelligence.test.ts | 293 +++++++++++++++++++-------------- 1 file changed, 165 insertions(+), 128 deletions(-) diff --git a/test/code-intelligence.test.ts b/test/code-intelligence.test.ts index 953824c724..4fb3fa376d 100644 --- a/test/code-intelligence.test.ts +++ b/test/code-intelligence.test.ts @@ -1,17 +1,18 @@ /** - * Tests for lib/code-intelligence — the OPTIONAL, repo-oriented provider contract. + * Tests for lib/code-intelligence — the OPTIONAL, repo-oriented provider contract + * with three REAL adapters (GBrain CLI, Graphify CLI, Sourcebot HTTP) plus the + * selection store the `gstack-code-intelligence` CLI drives. * * Load-bearing properties: - * - Every provider advertises the four required capabilities; optional ops are - * declined with a typed CAPABILITY_UNSUPPORTED, never a silent no-op. - * - Sourcebot/Graphify are phase-1 capability declarations: they prove contract - * fit and throw PROVIDER_UNAVAILABLE until a host MCP transport is wired. - * - The picker recommends GBrain first and resolves to null (provider-OFF) when - * GBrain is unavailable. - * - Non-local providers refuse to move repo content off the machine without - * explicit per-repo consent (PROVIDER_NOT_CONSENTED). - * - The GBrain adapter works end-to-end against a fake `gbrain` shim on PATH. - * - Missing CLI degrades (PROVIDER_UNAVAILABLE), never crashes. + * - Capability matrix: every provider advertises the four required ops; only + * GBrain advertises the document ops (add/delete/export). + * - Consent: non-local providers refuse to index without per-repo consent; a + * localhost Sourcebot and Graphify are local and need none. + * - GBrain search + status work end-to-end against a fake `gbrain` shim. + * - Graphify index/search/status work end-to-end against a fake `graphify` shim. + * - Sourcebot register (config.json edit) + search work against an injected fetch. + * - Selection persists to $GSTACK_HOME; no selection = provider-OFF (null). + * - Every adapter degrades to PROVIDER_UNAVAILABLE when its tool/server is absent. */ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; @@ -19,149 +20,210 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; import { - assertRequiredCapabilities, - CodeProviderError, REQUIRED_CAPABILITIES, GbrainProvider, - SourcebotProvider, GraphifyProvider, + SourcebotProvider, parseGbrainSearch, - recommendCodeProvider, - resolveCodeProvider, + parseGraphifyQuery, + parseSourcebotSearch, + readSelection, + setProvider, + setConsent, + hasConsent, + resolveSelectedProvider, RECOMMENDED_ORDER, } from "../lib/code-intelligence"; -describe("capability matrix invariants", () => { +describe("capability matrix", () => { test("every provider advertises the four required capabilities", () => { for (const p of [new GbrainProvider(), new SourcebotProvider(), new GraphifyProvider()]) { for (const cap of REQUIRED_CAPABILITIES) expect(p.has(cap)).toBe(true); } }); - test("GBrain advertises all seven (document axis)", () => { + test("only GBrain advertises the document ops; local flags are right", () => { const g = new GbrainProvider(); - for (const cap of ["add", "delete", "export"] as const) expect(g.has(cap)).toBe(true); expect(g.local).toBe(false); - }); + for (const cap of ["add", "delete", "export"] as const) expect(g.has(cap)).toBe(true); - test("Sourcebot is search-only: declines add/delete/export", () => { - const s = new SourcebotProvider(); + const s = new SourcebotProvider({ baseUrl: "http://localhost:3000" }); + expect(s.local).toBe(true); // loopback → content stays on machine expect(s.has("add")).toBe(false); - expect(s.has("delete")).toBe(false); - expect(s.has("export")).toBe(false); - expect(s.local).toBe(false); - }); + expect(new SourcebotProvider({ baseUrl: "https://sb.example.com" }).local).toBe(false); - test("Graphify is local, exports, declines add/delete", () => { - const g = new GraphifyProvider(); - expect(g.local).toBe(true); - expect(g.has("export")).toBe(true); - expect(g.has("add")).toBe(false); - expect(g.has("delete")).toBe(false); + const gf = new GraphifyProvider(); + expect(gf.local).toBe(true); + expect(gf.has("export")).toBe(true); + expect(gf.has("add")).toBe(false); }); - test("assertRequiredCapabilities rejects an incomplete provider", () => { - expect(() => assertRequiredCapabilities("sourcebot", new Set(["search"]))).toThrow(/missing required/); + test("RECOMMENDED_ORDER puts GBrain first", () => { + expect(RECOMMENDED_ORDER[0]).toBe("gbrain"); + expect([...RECOMMENDED_ORDER]).toEqual(["gbrain", "sourcebot", "graphify"]); }); +}); - test("CodeProviderError rejects an unknown failure code", () => { - // @ts-expect-error deliberately invalid code - expect(() => new CodeProviderError("NOPE", "x")).toThrow(/Unknown code-provider failure/); +describe("parsers", () => { + test("parseGbrainSearch (text surface)", () => { + const hits = parseGbrainSearch("[0.91] slug/a -- one\nbanner\n[0.05] slug/b -- low", 0.1, 10); + expect(hits).toEqual([{ ref: "slug/a", score: 0.91, snippet: "one", kind: "document" }]); }); -}); -describe("optional ops decline with a typed failure", () => { - test("Sourcebot declines add/delete/export with CAPABILITY_UNSUPPORTED", async () => { - const s = new SourcebotProvider(); - await expect(s.add({ slug: "x", body: "y" })).rejects.toMatchObject({ code: "CAPABILITY_UNSUPPORTED" }); - await expect(s.delete("x")).rejects.toMatchObject({ code: "CAPABILITY_UNSUPPORTED" }); - await expect(s.export({ id: "x" })).rejects.toMatchObject({ code: "CAPABILITY_UNSUPPORTED" }); + test("parseGraphifyQuery maps path-like lines to refs", () => { + const hits = parseGraphifyQuery("src/a.ts calls foo()\njust prose here", 10); + expect(hits[0]).toMatchObject({ ref: "src/a.ts", kind: "graph-node" }); + expect(hits[1]).toMatchObject({ ref: "graphify" }); }); - test("Graphify advertises export → not CAPABILITY_UNSUPPORTED, but unwired in phase 1", async () => { - await expect(new GraphifyProvider().export({ id: "x" })).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); + test("parseSourcebotSearch maps files to file:line hits, tolerates garbage", () => { + const payload = { + files: [{ fileName: { text: "src/x.ts" }, chunks: [{ content: "hit", matchRanges: [{ start: { lineNumber: 12 } }] }] }], + }; + expect(parseSourcebotSearch(payload, 10)).toEqual([{ ref: "src/x.ts:12", snippet: "hit", kind: "file" }]); + expect(parseSourcebotSearch("nope", 10)).toEqual([]); }); }); -describe("phase-1 declaration adapters degrade to PROVIDER_UNAVAILABLE", () => { - test("Sourcebot.search is unwired until a transport lands", async () => { - await expect(new SourcebotProvider().search("q")).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); +describe("selection store + provider-OFF", () => { + let home: string; + let env: NodeJS.ProcessEnv; + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "ci-home-")); + env = { ...process.env, GSTACK_HOME: home }; + }); + afterEach(() => fs.rmSync(home, { recursive: true, force: true })); + + test("no selection = provider-OFF (null)", () => { + expect(readSelection(env).provider).toBeNull(); + expect(resolveSelectedProvider({ env })).toBeNull(); }); - test("empty query short-circuits to no hits (no throw)", async () => { - expect(await new SourcebotProvider().search(" ")).toEqual([]); + test("select persists and resolves the provider", () => { + setProvider("graphify", env); + expect(readSelection(env).provider).toBe("graphify"); + expect(resolveSelectedProvider({ env })?.id).toBe("graphify"); }); - test("status is partial + unknown (no live probe yet)", async () => { - const s = await new GraphifyProvider().status({ id: "repo" }); - expect(s).toMatchObject({ id: "repo", state: "unknown", partial: true }); + test("consent is per-repo", () => { + const repo = path.join(home, "repoA"); + expect(hasConsent(repo, env)).toBe(false); + setConsent(repo, true, env); + expect(hasConsent(repo, env)).toBe(true); + expect(hasConsent(path.join(home, "repoB"), env)).toBe(false); }); }); describe("egress consent gate", () => { - test("non-local registerSource without consent → PROVIDER_NOT_CONSENTED", async () => { - await expect(new SourcebotProvider().registerSource({ id: "repo", path: "/repo" })).rejects.toMatchObject({ + test("GBrain (non-local) registerSource without consent → PROVIDER_NOT_CONSENTED", async () => { + await expect(new GbrainProvider().registerSource({ id: "code", path: "/repo" })).rejects.toMatchObject({ code: "PROVIDER_NOT_CONSENTED", }); }); - test("local provider (Graphify) skips the egress gate (falls through to unwired)", async () => { - // Local means nothing leaves the machine, so consent is not required — it - // reaches the phase-1 PROVIDER_UNAVAILABLE, NOT a consent rejection. - await expect(new GraphifyProvider().registerSource({ id: "repo", path: "/repo" })).rejects.toMatchObject({ - code: "PROVIDER_UNAVAILABLE", - }); - }); -}); - -describe("parseGbrainSearch (text surface)", () => { - const sample = ["[0.91] slug/a -- snippet one", "banner", "[0.05] slug/b -- below floor"].join("\n"); - test("parses scored lines, applies floor + limit", () => { - expect(parseGbrainSearch(sample, 0.1, 10)).toEqual([ - { ref: "slug/a", score: 0.91, snippet: "snippet one", kind: "document" }, - ]); - expect(parseGbrainSearch(sample, 0.0, 1)).toHaveLength(1); + test("Graphify (local) is exempt from the egress gate", async () => { + // Local → no consent needed; it reaches the CLI (absent here → UNAVAILABLE), + // NOT a consent rejection. + await expect( + new GraphifyProvider({ env: { PATH: "/nonexistent" } }).registerSource({ id: "r", path: os.tmpdir() }), + ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); }); -describe("picker — recommend GBrain first", () => { - test("RECOMMENDED_ORDER puts GBrain first", () => { - expect(RECOMMENDED_ORDER[0]).toBe("gbrain"); - expect([...RECOMMENDED_ORDER]).toEqual(["gbrain", "sourcebot", "graphify"]); +describe("Graphify adapter (fake graphify shim on PATH)", () => { + let binDir: string; + let repo: string; + function env(): NodeJS.ProcessEnv { + return { PATH: `${binDir}:${process.env.PATH}` }; + } + beforeEach(() => { + binDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gf-bin-")); + repo = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gf-repo-")); }); - - test("GBrain resolves when usable", () => { - expect(recommendCodeProvider({ gbrainStatus: "ok" }).map((p) => p.id)).toEqual(["gbrain"]); - expect(resolveCodeProvider({ gbrainStatus: "ok" })?.id).toBe("gbrain"); + afterEach(() => { + fs.rmSync(binDir, { recursive: true, force: true }); + fs.rmSync(repo, { recursive: true, force: true }); + }); + + test("index builds a graph and status reports ready + node count", async () => { + // Shim: `graphify .` writes graphify-out/graph.json in cwd; `graphify query` prints a hit line. + fs.writeFileSync( + path.join(binDir, "graphify"), + `#!/usr/bin/env bash +if [ "$1" = "query" ]; then echo "src/a.ts -> src/b.ts (calls)"; exit 0; fi +mkdir -p "$PWD/graphify-out" +echo '{"nodes":[1,2,3]}' > "$PWD/graphify-out/graph.json" +exit 0 +`, + { mode: 0o755 }, + ); + const gf = new GraphifyProvider({ root: repo, env: env() }); + const reg = await gf.registerSource({ id: "r", path: repo }); + expect(reg.state).toBe("ready"); + expect(reg.itemCount).toBe(3); + + const hits = await gf.search("what calls b", { source: repo }); + expect(hits[0].ref).toBe("src/a.ts"); + }); + + test("missing graphify CLI degrades to PROVIDER_UNAVAILABLE", async () => { + await expect( + new GraphifyProvider({ root: repo, env: { PATH: binDir } }).search("q"), + ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); +}); - test("timeout status counts as usable (engine slow, not absent)", () => { - expect(resolveCodeProvider({ gbrainStatus: "timeout" })?.id).toBe("gbrain"); +describe("Sourcebot adapter (injected fetch + temp config)", () => { + test("registerSource writes a local git connection to config.json", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-sb-")); + const configPath = path.join(dir, "config.json"); + const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", configPath }); + await sb.registerSource({ id: "myrepo", path: "/abs/repo" }); + const written = JSON.parse(fs.readFileSync(configPath, "utf-8")); + expect(written.connections.myrepo).toEqual({ type: "git", url: "file:///abs/repo" }); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test("search POSTs /api/search and maps files to hits", async () => { + const calls: Array<{ url: string; body: unknown }> = []; + const fetchStub = (async (url: string, init: RequestInit) => { + calls.push({ url: String(url), body: JSON.parse(String(init.body)) }); + return new Response( + JSON.stringify({ files: [{ fileName: { text: "a.ts" }, chunks: [{ content: "x", matchRanges: [{ start: { lineNumber: 3 } }] }] }] }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + }) as unknown as typeof fetch; + const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", fetch: fetchStub }); + const hits = await sb.search("foo", { limit: 5 }); + expect(calls[0].url).toBe("http://localhost:3000/api/search"); + expect((calls[0].body as { isRegexEnabled: boolean }).isRegexEnabled).toBe(true); + expect(hits).toEqual([{ ref: "a.ts:3", snippet: "x", kind: "file" }]); + }); + + test("unreachable server degrades to PROVIDER_UNAVAILABLE", async () => { + const fetchStub = (async () => { + throw new Error("ECONNREFUSED"); + }) as unknown as typeof fetch; + await expect( + new SourcebotProvider({ baseUrl: "http://localhost:3999", fetch: fetchStub }).search("q"), + ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); - test("provider-OFF: GBrain down → resolveCodeProvider null", () => { - expect(recommendCodeProvider({ gbrainStatus: "no-cli" })).toEqual([]); - expect(resolveCodeProvider({ gbrainStatus: "no-cli" })).toBeNull(); - expect(resolveCodeProvider({ gbrainStatus: "missing-config" })).toBeNull(); + test("registerSource without SOURCEBOT_CONFIG → PROVIDER_UNAVAILABLE", async () => { + const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", env: {} }); + await expect(sb.registerSource({ id: "r", path: "/x" })).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); }); -describe("GBrain adapter end-to-end (fake shim on PATH)", () => { +describe("GBrain adapter end-to-end (fake gbrain shim on PATH)", () => { let binDir: string; let homeDir: string; - - function writeShim(body: string): void { - const p = path.join(binDir, "gbrain"); - fs.writeFileSync(p, body, { mode: 0o755 }); - fs.chmodSync(p, 0o755); - } function env(): NodeJS.ProcessEnv { return { PATH: `${binDir}:${process.env.PATH}`, HOME: homeDir }; } - beforeEach(() => { - binDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gbrain-shim-")); - homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gbrain-home-")); + binDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gb-bin-")); + homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gb-home-")); }); afterEach(() => { fs.rmSync(binDir, { recursive: true, force: true }); @@ -169,50 +231,25 @@ describe("GBrain adapter end-to-end (fake shim on PATH)", () => { }); test("search scopes to source and parses hits", async () => { - writeShim(`#!/usr/bin/env bash + fs.writeFileSync( + path.join(binDir, "gbrain"), + `#!/usr/bin/env bash if [ "$1" = "search" ]; then - if printf '%s ' "$@" | grep -q -- "--source code"; then - echo "[0.88] src/x.ts -- match in code source" - else - echo "[0.10] wrong -- unscoped" - fi + if printf '%s ' "$@" | grep -q -- "--source code"; then echo "[0.88] src/x.ts -- match"; else echo "[0.10] wrong -- unscoped"; fi exit 0 fi exit 1 -`); +`, + { mode: 0o755 }, + ); const hits = await new GbrainProvider().search("where", { env: env(), source: "code" }); expect(hits).toHaveLength(1); expect(hits[0].ref).toBe("src/x.ts"); }); - test("status(source) reports ready + page_count", async () => { - writeShim(`#!/usr/bin/env bash -if [ "$1" = "sources" ]; then echo '{"sources":[{"id":"code","local_path":"/repo","page_count":42}]}'; exit 0; fi -exit 1 -`); - const s = await new GbrainProvider().status({ id: "code" }, { env: env() }); - expect(s.state).toBe("ready"); - expect(s.itemCount).toBe(42); - }); - - test("status(source) reports absent for an unregistered id", async () => { - writeShim(`#!/usr/bin/env bash -if [ "$1" = "sources" ]; then echo '{"sources":[]}'; exit 0; fi -exit 1 -`); - expect((await new GbrainProvider().status({ id: "nope" }, { env: env() })).state).toBe("absent"); - }); - test("missing CLI degrades to PROVIDER_UNAVAILABLE", async () => { - // No shim written; PATH points only at an empty dir. await expect( new GbrainProvider().search("q", { env: { PATH: binDir, HOME: homeDir } }), ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); - - test("registerSource requires egress consent (GBrain is non-local)", async () => { - await expect( - new GbrainProvider().registerSource({ id: "code", path: "/repo" }, { env: env() }), - ).rejects.toMatchObject({ code: "PROVIDER_NOT_CONSENTED" }); - }); }); From f0f13998d8fec77a157a7d3139543189c8a3c10b Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:40:50 -0700 Subject: [PATCH 5/9] fix: correct all three adapters to real tool interfaces (found by live testing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel real-environment tests (graphify 0.9.23, Sourcebot v5 in Docker, gbrain 0.42.56) surfaced real mismatches: - graphify: build via `graphify update ` (the local, no-LLM path) instead of `graphify ` (which runs an LLM backend needing a key + network, so the old path wasn't actually local); query via `graphify query --graph ` so it reads the indexed graph regardless of cwd; parse the real NODE/EDGE output (file lives at src=/at=) instead of an invented format. - sourcebot: Sourcebot v5 gates /api/search behind auth — send `Authorization: Bearer `; treat 401/403 as PROVIDER_UNAVAILABLE; make status probe /api/search without following the login redirect. - gbrain: degrade engine/DB init failures (e.g. pglite WASM, garrytan/gbrain#223) to PROVIDER_UNAVAILABLE with a one-line message instead of PROVIDER_ERROR + a raw stack dump; drop flags the real CLI doesn't define (`sync --strategy`, `search --source`); align put/delete to stdin, export to brain-wide. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/code-intelligence/gbrain-adapter.ts | 69 +++++++++++++------- lib/code-intelligence/graphify-adapter.ts | 74 ++++++++++++++-------- lib/code-intelligence/sourcebot-adapter.ts | 28 +++++++- 3 files changed, 120 insertions(+), 51 deletions(-) diff --git a/lib/code-intelligence/gbrain-adapter.ts b/lib/code-intelligence/gbrain-adapter.ts index e4dbae2364..fc89589871 100644 --- a/lib/code-intelligence/gbrain-adapter.ts +++ b/lib/code-intelligence/gbrain-adapter.ts @@ -9,7 +9,8 @@ * so it advertises all seven capabilities. */ -import { spawnGbrain } from "../gbrain-exec"; +import { spawnSync } from "child_process"; +import { spawnGbrain, buildGbrainEnv, NEEDS_SHELL_ON_WINDOWS } from "../gbrain-exec"; import { ensureSourceRegistered, probeSource, sourcePageCount } from "../gbrain-sources"; import { assertCapability, @@ -89,7 +90,8 @@ export class GbrainProvider implements CodeProvider { async refresh(source: SourceRef, opts: OpOptions = {}): Promise { assertEgressConsent(this, opts); - this.#assertOk(spawnGbrain(["sync", "--strategy", "code", "--source", source.id], { + // `gbrain sync` has no `--strategy` flag (verified against gbrain 0.42.x --help). + this.#assertOk(spawnGbrain(["sync", "--source", source.id], { baseEnv: opts.env, timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, })); @@ -98,8 +100,10 @@ export class GbrainProvider implements CodeProvider { async search(query: string, opts: SearchOptions = {}): Promise { if (!query.trim()) return []; + // `gbrain search` is global and has no `--source` flag; `--limit` is real + // (verified against gbrain 0.42.x --help). const args = ["search", query]; - if (opts.source) args.push("--source", opts.source); + if (opts.limit) args.push("--limit", String(opts.limit)); const r = spawnGbrain(args, { baseEnv: opts.env, timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS }); this.#assertOk(r); return parseGbrainSearch(r.stdout || "", opts.minScore ?? 0.1, opts.limit ?? 10); @@ -129,29 +133,29 @@ export class GbrainProvider implements CodeProvider { } } + // Document ops (add/delete/export) are GBrain-only and secondary; they match + // gbrain's documented CLI surface (`put ` reads stdin; `delete `; + // `export`) but could not be exercised against a live engine on the test host + // (pglite WASM broken, garrytan/gbrain#223), so treat them as best-effort. async add(doc: { slug: string; body: string }, opts: OpOptions = {}): Promise { assertCapability(this, "add"); assertEgressConsent(this, opts); - const r = spawnGbrain(["put", doc.slug, "--body", doc.body], { - baseEnv: opts.env, - timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, - }); - this.#assertOk(r); + // `gbrain put ` reads the document body from stdin. + this.#assertOk(this.#runInput(["put", doc.slug], doc.body, opts)); return { id: doc.slug, state: "ready" }; } async delete(slug: string, opts: OpOptions = {}): Promise { assertCapability(this, "delete"); - this.#assertOk(spawnGbrain(["delete", slug, "--yes"], { - baseEnv: opts.env, - timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, - })); + // stdin closed ("") so any confirmation prompt gets EOF rather than hanging. + this.#assertOk(this.#runInput(["delete", slug], "", opts)); return { id: slug, state: "absent" }; } - async export(source: SourceRef, opts: OpOptions = {}): Promise { + async export(_source: SourceRef, opts: OpOptions = {}): Promise { assertCapability(this, "export"); - const r = spawnGbrain(["export", "--source", source.id], { + // `gbrain export` is brain-wide (no per-source flag); returns whatever it prints. + const r = spawnGbrain(["export"], { baseEnv: opts.env, timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, }); @@ -159,6 +163,17 @@ export class GbrainProvider implements CodeProvider { return r.stdout || ""; } + /** spawn gbrain with `input` on stdin, seeded env, Windows-shim aware. */ + #runInput(args: string[], input: string, opts: OpOptions) { + return spawnSync("gbrain", args, { + input, + encoding: "utf-8", + timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, + env: buildGbrainEnv({ baseEnv: opts.env }), + shell: NEEDS_SHELL_ON_WINDOWS, + }); + } + /** * Throw a typed failure unless the spawn succeeded. Distinguishes a missing * CLI (ENOENT → PROVIDER_UNAVAILABLE, the degrade signal) from a timeout @@ -178,21 +193,29 @@ export class GbrainProvider implements CodeProvider { if (r.error?.code === "ETIMEDOUT" || r.signal === "SIGTERM") { throw new CodeProviderError("PROVIDER_TIMEOUT", "gbrain timed out", this.id); } - if (/not configured|Cannot connect to database|config\.json/.test(stderr)) { - throw new CodeProviderError("PROVIDER_UNAVAILABLE", stderr || "gbrain not configured", this.id); + // Engine / DB / config problems are ENVIRONMENTAL — degrade to UNAVAILABLE + // (caller falls back to file-only), not a hard PROVIDER_ERROR with a raw dump. + // Covers the real case where gbrain's pglite engine fails to init its WASM + // runtime (garrytan/gbrain#223) as well as unreachable/unconfigured databases. + if (/PGLite|WASM|failed to initialize|Aborted|Cannot connect to database|not configured|config\.json|database (is )?un(reachable|available)/i.test(stderr)) { + throw new CodeProviderError("PROVIDER_UNAVAILABLE", firstLine(stderr) || "gbrain engine unavailable", this.id); } - throw new CodeProviderError("PROVIDER_ERROR", stderr || `gbrain exited ${r.status}`, this.id); + throw new CodeProviderError("PROVIDER_ERROR", firstLine(stderr) || `gbrain exited ${r.status}`, this.id); } #wrap(err: unknown): CodeProviderError { if (err instanceof CodeProviderError) return err; const message = err instanceof Error ? err.message : String(err); - if (/not on PATH|command not found/.test(message)) { - return new CodeProviderError("PROVIDER_UNAVAILABLE", message, this.id); + // Same environmental-vs-real split as #assertOk: missing CLI, or engine/DB/ + // config problems, degrade to UNAVAILABLE so callers fall back to file-only. + if (/not on PATH|command not found|PGLite|WASM|failed to initialize|Aborted|Cannot connect to database|not configured|config\.json/i.test(message)) { + return new CodeProviderError("PROVIDER_UNAVAILABLE", firstLine(message), this.id); } - if (/not configured/.test(message)) { - return new CodeProviderError("PROVIDER_UNAVAILABLE", message, this.id); - } - return new CodeProviderError("PROVIDER_ERROR", message, this.id); + return new CodeProviderError("PROVIDER_ERROR", firstLine(message), this.id); } } + +/** First non-empty line, so a multi-line WASM/stack dump never reaches the user. */ +function firstLine(text: string): string { + return (text || "").split("\n").map((l) => l.trim()).find(Boolean) ?? ""; +} diff --git a/lib/code-intelligence/graphify-adapter.ts b/lib/code-intelligence/graphify-adapter.ts index 3cbcf492c8..22d58ec991 100644 --- a/lib/code-intelligence/graphify-adapter.ts +++ b/lib/code-intelligence/graphify-adapter.ts @@ -1,15 +1,19 @@ /** * Graphify adapter — real CLI integration (github.com/Graphify-Labs/graphify). * - * Graphify is a LOCAL tree-sitter knowledge graph: `graphify ` builds a - * `graphify-out/` (graph.json + report) in that dir, `graphify query ""` - * queries it, and nothing leaves the machine (no embeddings, no network). So it - * needs no egress consent and no MCP — the runtime just shells out to the CLI, - * the same shape as the GBrain adapter. + * Graphify is a LOCAL tree-sitter knowledge graph. The genuinely local, no-LLM + * build is `graphify update ` — it writes `/graphify-out/graph.json` + * with NO embeddings and NO network (verified against graphify 0.9.23). NOTE: + * the bare `graphify ` build instead runs LLM semantic extraction (a gemini + * backend needing an API key + network), so this adapter deliberately uses + * `graphify update`, which keeps the "local, no egress consent" invariant true. * - * Never auto-installed: install is `pip install graphifyy && graphify install`, - * a user action the picker gates on. When the CLI is absent every op throws - * PROVIDER_UNAVAILABLE and callers degrade to file-only. + * Query is `graphify query "" --graph /graphify-out/graph.json`; the + * `--graph` flag points at the built graph so search never depends on cwd. + * + * Never auto-installed: install is `pip install graphifyy && graphify install` + * (needs Python >= 3.10), a user action the picker surfaces. When the CLI is + * absent every op throws PROVIDER_UNAVAILABLE and callers degrade to file-only. * * Path-based, not id-based: for Graphify a source "id" IS the absolute repo path * (that is where `graphify-out/` lives), unlike GBrain's short source ids. @@ -86,29 +90,28 @@ export class GraphifyProvider implements CodeProvider { throw new CodeProviderError("PROVIDER_ERROR", stderr || `graphify exited ${r.status}`, this.id); } - /** Build the graph over repo.path (local; no egress consent needed). */ + /** Build the graph over repo.path locally (no LLM, no egress consent needed). */ async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise { - this.#assertOk(this.#run(["."], repo.path, opts.timeout ?? DEFAULT_TIMEOUT_MS)); + this.#assertOk(this.#run(["update", repo.path], repo.path, opts.timeout ?? DEFAULT_TIMEOUT_MS)); return this.status({ id: repo.path }, opts); } - /** Re-parse and merge changes into the existing graph. */ + /** Re-parse and rebuild the graph (same local `graphify update` path). */ async refresh(source: SourceRef, opts: OpOptions = {}): Promise { - this.#assertOk(this.#run([".", "--update"], source.id, opts.timeout ?? DEFAULT_TIMEOUT_MS)); + this.#assertOk(this.#run(["update", source.id], source.id, opts.timeout ?? DEFAULT_TIMEOUT_MS)); return this.status(source, opts); } /** - * `graphify query ""` returns a traced answer over the graph. Its stdout is - * an answer, not a fixed hit schema (the CLI reference documents the command - * but not a machine format), so we map non-empty output lines to hits - * tolerantly: a leading path-like token becomes the ref, the line the snippet. - * Reconcile against a live graphify if a stricter schema is needed. + * `graphify query "" --graph ` traces the graph and prints + * `NODE ...` / `EDGE ...` lines (plus a `Traversal:` header). We pass `--graph` + * explicitly so the query reads the indexed repo's graph regardless of cwd. */ async search(query: string, opts: SearchOptions = {}): Promise { if (!query.trim()) return []; - const cwd = opts.source ?? this.#root; - const r = this.#run(["query", query], cwd, opts.timeout ?? DEFAULT_TIMEOUT_MS); + const root = opts.source ?? this.#root; + const graphPath = join(root, OUT_DIR, GRAPH_JSON); + const r = this.#run(["query", query, "--graph", graphPath], root, opts.timeout ?? DEFAULT_TIMEOUT_MS); this.#assertOk(r); return parseGraphifyQuery(r.stdout || "", opts.limit ?? 10); } @@ -138,19 +141,38 @@ export class GraphifyProvider implements CodeProvider { } /** - * Map `graphify query` stdout to hits. Tolerant: each non-empty line becomes a - * hit; a leading `path` or `path:line` token becomes the ref, else the whole - * line is the snippet. Exported for deterministic unit testing. + * Parse real `graphify query` output into hits. The format (graphify 0.9.23): + * Traversal: BFS depth=2 | Start: ['query()'] | ... | 4 nodes found + * NODE query() [src=db.py loc=L4 community=login] + * EDGE query() --calls [EXTRACTED context=call]--> login() at=auth.py:L8 + * The file lives mid-line (`src= loc=L` on NODE, `at=:L` on + * EDGE), so the ref is `:L`. The `Traversal:` header and any other line + * are skipped. Exported for deterministic unit testing against the real format. */ export function parseGraphifyQuery(stdout: string, limit: number): CodeSearchHit[] { const hits: CodeSearchHit[] = []; for (const raw of stdout.split("\n")) { const line = raw.trim(); - if (!line) continue; - const token = line.split(/\s+/)[0]; - const looksPath = /[/\\.]/.test(token) && !token.includes(" "); - hits.push({ ref: looksPath ? token : "graphify", snippet: line, kind: "graph-node" }); + let ref: string | undefined; + const node = line.match(/^NODE\b.*?\[src=(\S+)\s+loc=(L\d+)/); + const edge = line.match(/^EDGE\b.*?\bat=(\S+?):(L\d+)\b/); + if (node) ref = `${node[1]}:${node[2]}`; + else if (edge) ref = `${edge[1]}:${edge[2]}`; + else continue; // skip the Traversal header and anything non-NODE/EDGE + hits.push({ ref, snippet: line, kind: "graph-node" }); if (hits.length >= limit) break; } return hits; } + +/** Whether the `graphify` CLI is installed (for the picker's availability probe). */ +export function graphifyInstalled(env?: NodeJS.ProcessEnv): boolean { + const r = spawnSync("graphify", ["--version"], { + encoding: "utf-8", + timeout: 5_000, + stdio: ["ignore", "ignore", "ignore"], + env, + shell: NEEDS_SHELL_ON_WINDOWS, + }); + return r.status === 0; +} diff --git a/lib/code-intelligence/sourcebot-adapter.ts b/lib/code-intelligence/sourcebot-adapter.ts index b6d7d9fc05..d7c6c5e244 100644 --- a/lib/code-intelligence/sourcebot-adapter.ts +++ b/lib/code-intelligence/sourcebot-adapter.ts @@ -45,6 +45,12 @@ export interface SourcebotOptions { baseUrl?: string; /** Path to the server's config.json (for register_source). Defaults to SOURCEBOT_CONFIG. */ configPath?: string; + /** + * API key for the Sourcebot REST API. Defaults to SOURCEBOT_API_KEY. Sourcebot + * v5 gates `/api/search` behind auth (`Authorization: Bearer `); without + * it, `search` gets HTTP 401. Generate one in Settings -> API Keys. + */ + apiKey?: string; /** Injectable fetch for tests. */ fetch?: FetchLike; env?: NodeJS.ProcessEnv; @@ -66,17 +72,23 @@ export class SourcebotProvider implements CodeProvider { readonly local: boolean; readonly #baseUrl: string; readonly #configPath?: string; + readonly #apiKey?: string; readonly #fetch: FetchLike; constructor(opts: SourcebotOptions = {}) { const env = opts.env ?? process.env; this.#baseUrl = (opts.baseUrl ?? env.SOURCEBOT_URL ?? DEFAULT_URL).replace(/\/$/, ""); this.#configPath = opts.configPath ?? env.SOURCEBOT_CONFIG; + this.#apiKey = opts.apiKey ?? env.SOURCEBOT_API_KEY; this.#fetch = opts.fetch ?? globalThis.fetch; this.local = isLoopback(this.#baseUrl); assertRequiredCapabilities(this.id, this.capabilities); } + #authHeaders(): Record { + return this.#apiKey ? { Authorization: `Bearer ${this.#apiKey}` } : {}; + } + has(capability: CodeProviderCapability): boolean { return this.capabilities.has(capability); } @@ -127,8 +139,17 @@ export class SourcebotProvider implements CodeProvider { } async status(_source?: SourceRef, opts: OpOptions = {}): Promise { + // `redirect: manual` so an auth-gated server (307 -> /login) reads as + // not-usable instead of following to a 200 and falsely reporting "ready". try { - const res = await this.#fetchWithTimeout(this.#baseUrl, { method: "GET" }, opts.timeout ?? DEFAULT_TIMEOUT_MS); + const res = await this.#fetchWithTimeout( + `${this.#baseUrl}/api/search`, + { method: "POST", headers: { "Content-Type": "application/json", ...this.#authHeaders() }, body: JSON.stringify({ query: "sourcebot", matches: 1, isRegexEnabled: false }), redirect: "manual" }, + opts.timeout ?? DEFAULT_TIMEOUT_MS, + ); + if (res.status === 401 || res.status === 403) { + return { id: "*", state: "unknown", partial: true, detail: "reachable but not authenticated (set SOURCEBOT_API_KEY)" }; + } return { id: "*", state: res.ok ? "ready" : "unknown", partial: true, detail: `HTTP ${res.status}` }; } catch { return { id: "*", state: "unknown", partial: true, detail: `unreachable at ${this.#baseUrl}` }; @@ -140,12 +161,15 @@ export class SourcebotProvider implements CodeProvider { try { res = await this.#fetchWithTimeout(`${this.#baseUrl}${path}`, { method: "POST", - headers: { "Content-Type": "application/json", Accept: "application/json" }, + headers: { "Content-Type": "application/json", Accept: "application/json", ...this.#authHeaders() }, body: JSON.stringify(body), }, timeout); } catch (err) { throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot unreachable at ${this.#baseUrl}: ${(err as Error).message}`, this.id); } + if (res.status === 401 || res.status === 403) { + throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot requires authentication; set SOURCEBOT_API_KEY (HTTP ${res.status})`, this.id); + } if (!res.ok) throw new CodeProviderError("PROVIDER_ERROR", `Sourcebot ${path} returned HTTP ${res.status}`, this.id); try { return await res.json(); From 6686202be99c75d549e49225a6a14f4b5b84dff8 Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:40:50 -0700 Subject: [PATCH 6/9] fix: persist indexed repo per provider and detect installs honestly search was running in cwd and missing the repo you indexed. Persist the indexed path per provider in the selection store and resolve it back so `search` reads the same graph `index` built. Graphify availability now checks `graphify --version` (installed = selectable) instead of "a graph already exists here", and the CLI keys Graphify sources on the repo path. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/gstack-code-intelligence | 7 ++++++- lib/code-intelligence/index.ts | 2 ++ lib/code-intelligence/picker.ts | 31 ++++++++++++++++++------------ lib/code-intelligence/selection.ts | 17 +++++++++++++++- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/bin/gstack-code-intelligence b/bin/gstack-code-intelligence index 9bcca08ef9..e9ea163f40 100755 --- a/bin/gstack-code-intelligence +++ b/bin/gstack-code-intelligence @@ -29,6 +29,7 @@ import { resolveSelectedProvider, setConsent, setProvider, + setRoot, type CodeProviderId, } from "../lib/code-intelligence"; @@ -98,11 +99,15 @@ async function cmdIndex(pathArg: string | undefined): Promise { if (!provider!.local && !consented) { fail(`${provider!.label} would send this repo's content off the machine. Run \`gstack-code-intelligence consent ${repoPath}\` first.`); } - const repo = { id: basename(repoPath), path: repoPath }; + // Graphify keys sources on the repo path; GBrain/Sourcebot on a short id. + const sourceId = provider!.id === "graphify" ? repoPath : basename(repoPath); + const repo = { id: sourceId, path: repoPath }; try { const registered = await provider!.registerSource(repo, { consented }); out(`registered ${repo.id} with ${provider!.label} (${registered.state})`); const refreshed = await provider!.refresh({ id: registered.id }, { consented }); + // Remember which repo this provider indexed so `search` reads the same graph. + setRoot(provider!.id, repoPath); out(`indexed: ${refreshed.state}${refreshed.itemCount != null ? ` (${refreshed.itemCount} items)` : ""}`); } catch (err) { handleProviderError(err, provider!.label); diff --git a/lib/code-intelligence/index.ts b/lib/code-intelligence/index.ts index 0c69719c8d..6b387ff8c2 100644 --- a/lib/code-intelligence/index.ts +++ b/lib/code-intelligence/index.ts @@ -12,6 +12,8 @@ export { setProvider, setConsent, hasConsent, + setRoot, + getRoot, type Selection, } from "./selection"; export { diff --git a/lib/code-intelligence/picker.ts b/lib/code-intelligence/picker.ts index 0d31d1c281..56573bc91d 100644 --- a/lib/code-intelligence/picker.ts +++ b/lib/code-intelligence/picker.ts @@ -15,9 +15,9 @@ import { localEngineStatus } from "../gbrain-local-status"; import { GbrainProvider } from "./gbrain-adapter"; -import { GraphifyProvider, type GraphifyOptions } from "./graphify-adapter"; +import { GraphifyProvider, graphifyInstalled, type GraphifyOptions } from "./graphify-adapter"; import { SourcebotProvider, type SourcebotOptions } from "./sourcebot-adapter"; -import { readSelection } from "./selection"; +import { readSelection, getRoot } from "./selection"; import type { CodeProvider, CodeProviderId } from "./contract"; /** Recommendation order — GBrain first. */ @@ -34,8 +34,12 @@ export function providerById(id: CodeProviderId, opts: PickerOptions = {}): Code switch (id) { case "gbrain": return new GbrainProvider(); - case "graphify": - return new GraphifyProvider({ env: opts.env, ...opts.graphify }); + case "graphify": { + // Default the graph root to the repo Graphify last indexed, so `search` + // reads the same graph `index` built (not whatever cwd happens to be). + const root = opts.graphify?.root ?? getRoot("graphify", opts.env); + return new GraphifyProvider({ env: opts.env, ...opts.graphify, ...(root ? { root } : {}) }); + } case "sourcebot": return new SourcebotProvider({ env: opts.env, ...opts.sourcebot }); } @@ -65,14 +69,17 @@ export async function detectAvailable(opts: PickerOptions = {}): Promise; + /** Provider id → the absolute repo path it last indexed (so search finds it). */ + roots: Record; } -const EMPTY: Selection = { provider: null, consents: {} }; +const EMPTY: Selection = { provider: null, consents: {}, roots: {} }; function storePath(env: NodeJS.ProcessEnv = process.env): string { const home = env.GSTACK_HOME || join(env.HOME || homedir(), ".gstack"); @@ -35,6 +37,7 @@ export function readSelection(env: NodeJS.ProcessEnv = process.env): Selection { return { provider: raw.provider ?? null, consents: raw.consents && typeof raw.consents === "object" ? raw.consents : {}, + roots: raw.roots && typeof raw.roots === "object" ? raw.roots : {}, }; } catch { return { ...EMPTY }; @@ -66,3 +69,15 @@ export function setConsent(repoPath: string, consented: boolean, env: NodeJS.Pro export function hasConsent(repoPath: string, env: NodeJS.ProcessEnv = process.env): boolean { return readSelection(env).consents[resolve(repoPath)] === true; } + +/** Record the repo path a provider last indexed, so search reads the same graph. */ +export function setRoot(provider: CodeProviderId, repoPath: string, env: NodeJS.ProcessEnv = process.env): Selection { + const current = readSelection(env); + const next: Selection = { ...current, roots: { ...current.roots, [provider]: resolve(repoPath) } }; + write(next, env); + return next; +} + +export function getRoot(provider: CodeProviderId, env: NodeJS.ProcessEnv = process.env): string | undefined { + return readSelection(env).roots[provider]; +} From 45310eee6eebbeb639782f9def3864a4b0defcea Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:40:50 -0700 Subject: [PATCH 7/9] test: pin adapter tests to real captured tool output Rewrite the Graphify and Sourcebot expectations against the real formats captured from live tools (graphify NODE/EDGE query output; Sourcebot v5 response + Bearer auth), add a gbrain engine-down -> PROVIDER_UNAVAILABLE degrade test and a no-phantom-`--source` search assertion, and cover the persisted indexed-root path. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/code-intelligence.test.ts | 178 +++++++++++++++++++++------------ 1 file changed, 112 insertions(+), 66 deletions(-) diff --git a/test/code-intelligence.test.ts b/test/code-intelligence.test.ts index 4fb3fa376d..0a4be6748d 100644 --- a/test/code-intelligence.test.ts +++ b/test/code-intelligence.test.ts @@ -1,18 +1,11 @@ /** * Tests for lib/code-intelligence — the OPTIONAL, repo-oriented provider contract - * with three REAL adapters (GBrain CLI, Graphify CLI, Sourcebot HTTP) plus the + * with three REAL adapters (GBrain CLI, Graphify CLI, Sourcebot HTTP) and the * selection store the `gstack-code-intelligence` CLI drives. * - * Load-bearing properties: - * - Capability matrix: every provider advertises the four required ops; only - * GBrain advertises the document ops (add/delete/export). - * - Consent: non-local providers refuse to index without per-repo consent; a - * localhost Sourcebot and Graphify are local and need none. - * - GBrain search + status work end-to-end against a fake `gbrain` shim. - * - Graphify index/search/status work end-to-end against a fake `graphify` shim. - * - Sourcebot register (config.json edit) + search work against an injected fetch. - * - Selection persists to $GSTACK_HOME; no selection = provider-OFF (null). - * - Every adapter degrades to PROVIDER_UNAVAILABLE when its tool/server is absent. + * The Graphify and Sourcebot expectations here are pinned to the REAL formats + * captured from live tools (graphify 0.9.23 NODE/EDGE query output; Sourcebot v5 + * `/api/search` response + Bearer auth), not invented shapes. */ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; @@ -31,6 +24,8 @@ import { setProvider, setConsent, hasConsent, + setRoot, + getRoot, resolveSelectedProvider, RECOMMENDED_ORDER, } from "../lib/code-intelligence"; @@ -59,28 +54,44 @@ describe("capability matrix", () => { }); test("RECOMMENDED_ORDER puts GBrain first", () => { - expect(RECOMMENDED_ORDER[0]).toBe("gbrain"); expect([...RECOMMENDED_ORDER]).toEqual(["gbrain", "sourcebot", "graphify"]); }); }); -describe("parsers", () => { +describe("parsers (pinned to real tool output)", () => { test("parseGbrainSearch (text surface)", () => { const hits = parseGbrainSearch("[0.91] slug/a -- one\nbanner\n[0.05] slug/b -- low", 0.1, 10); expect(hits).toEqual([{ ref: "slug/a", score: 0.91, snippet: "one", kind: "document" }]); }); - test("parseGraphifyQuery maps path-like lines to refs", () => { - const hits = parseGraphifyQuery("src/a.ts calls foo()\njust prose here", 10); - expect(hits[0]).toMatchObject({ ref: "src/a.ts", kind: "graph-node" }); - expect(hits[1]).toMatchObject({ ref: "graphify" }); + test("parseGraphifyQuery reads file:line from real NODE/EDGE lines", () => { + // Verbatim shape from graphify 0.9.23 `query ... --graph`. + const real = [ + "Traversal: BFS depth=2 | Start: ['query()'] | Context: call (heuristic) | 4 nodes found", + "", + "NODE query() [src=db.py loc=L4 community=login]", + "EDGE query() --calls [EXTRACTED context=call]--> login() at=auth.py:L8", + ].join("\n"); + const hits = parseGraphifyQuery(real, 10); + expect(hits.map((h) => h.ref)).toEqual(["db.py:L4", "auth.py:L8"]); // NOT "graphify" + expect(hits.every((h) => h.kind === "graph-node")).toBe(true); + // The Traversal header must NOT become a bogus hit. + expect(hits.some((h) => h.snippet?.startsWith("Traversal:"))).toBe(false); }); - test("parseSourcebotSearch maps files to file:line hits, tolerates garbage", () => { - const payload = { - files: [{ fileName: { text: "src/x.ts" }, chunks: [{ content: "hit", matchRanges: [{ start: { lineNumber: 12 } }] }] }], + test("parseSourcebotSearch maps files to file:line hits (real v5 shape)", () => { + const real = { + files: [ + { + fileName: { text: "src/checksum.ts", matchRanges: [] }, + repository: "github.com/example/sb-sample", + chunks: [{ content: "export function computeChecksum(data: string): number {", matchRanges: [{ start: { byteOffset: 58, column: 17, lineNumber: 2 } }] }], + }, + ], }; - expect(parseSourcebotSearch(payload, 10)).toEqual([{ ref: "src/x.ts:12", snippet: "hit", kind: "file" }]); + expect(parseSourcebotSearch(real, 10)).toEqual([ + { ref: "src/checksum.ts:2", snippet: "export function computeChecksum(data: string): number {", kind: "file" }, + ]); expect(parseSourcebotSearch("nope", 10)).toEqual([]); }); }); @@ -112,6 +123,12 @@ describe("selection store + provider-OFF", () => { expect(hasConsent(repo, env)).toBe(true); expect(hasConsent(path.join(home, "repoB"), env)).toBe(false); }); + + test("indexed root persists per provider (so search reads the same graph)", () => { + expect(getRoot("graphify", env)).toBeUndefined(); + setRoot("graphify", "/tmp/some/repo", env); + expect(getRoot("graphify", env)).toBe(path.resolve("/tmp/some/repo")); + }); }); describe("egress consent gate", () => { @@ -122,15 +139,13 @@ describe("egress consent gate", () => { }); test("Graphify (local) is exempt from the egress gate", async () => { - // Local → no consent needed; it reaches the CLI (absent here → UNAVAILABLE), - // NOT a consent rejection. await expect( new GraphifyProvider({ env: { PATH: "/nonexistent" } }).registerSource({ id: "r", path: os.tmpdir() }), ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); }); -describe("Graphify adapter (fake graphify shim on PATH)", () => { +describe("Graphify adapter (fake graphify shim, real NODE/EDGE format)", () => { let binDir: string; let repo: string; function env(): NodeJS.ProcessEnv { @@ -139,67 +154,87 @@ describe("Graphify adapter (fake graphify shim on PATH)", () => { beforeEach(() => { binDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gf-bin-")); repo = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gf-repo-")); - }); - afterEach(() => { - fs.rmSync(binDir, { recursive: true, force: true }); - fs.rmSync(repo, { recursive: true, force: true }); - }); - - test("index builds a graph and status reports ready + node count", async () => { - // Shim: `graphify .` writes graphify-out/graph.json in cwd; `graphify query` prints a hit line. + // Shim emulates real graphify 0.9.23: `update ` writes graph.json (no LLM); + // `query --graph ` prints NODE/EDGE lines. fs.writeFileSync( path.join(binDir, "graphify"), `#!/usr/bin/env bash -if [ "$1" = "query" ]; then echo "src/a.ts -> src/b.ts (calls)"; exit 0; fi -mkdir -p "$PWD/graphify-out" -echo '{"nodes":[1,2,3]}' > "$PWD/graphify-out/graph.json" -exit 0 +case "$1" in + --version) echo "graphify 0.9.23"; exit 0;; + update) mkdir -p "$2/graphify-out"; echo '{"nodes":[1,2,3,4,5],"edges":[]}' > "$2/graphify-out/graph.json"; echo "Rebuilt: 5 nodes, 8 edges"; exit 0;; + query) + echo "Traversal: BFS depth=2 | Start: ['query()'] | 4 nodes found" + echo "" + echo "NODE query() [src=db.py loc=L4 community=login]" + echo "EDGE query() --calls [EXTRACTED context=call]--> login() at=auth.py:L8" + exit 0;; +esac +exit 1 `, { mode: 0o755 }, ); + }); + afterEach(() => { + fs.rmSync(binDir, { recursive: true, force: true }); + fs.rmSync(repo, { recursive: true, force: true }); + }); + + test("index builds a graph (via `graphify update`) and status counts nodes", async () => { const gf = new GraphifyProvider({ root: repo, env: env() }); - const reg = await gf.registerSource({ id: "r", path: repo }); + const reg = await gf.registerSource({ id: repo, path: repo }); expect(reg.state).toBe("ready"); - expect(reg.itemCount).toBe(3); + expect(reg.itemCount).toBe(5); + expect(fs.existsSync(path.join(repo, "graphify-out", "graph.json"))).toBe(true); + }); - const hits = await gf.search("what calls b", { source: repo }); - expect(hits[0].ref).toBe("src/a.ts"); + test("search reads file:line refs from real query output", async () => { + const gf = new GraphifyProvider({ root: repo, env: env() }); + await gf.registerSource({ id: repo, path: repo }); + const hits = await gf.search("what calls db", { source: repo }); + expect(hits.map((h) => h.ref)).toEqual(["db.py:L4", "auth.py:L8"]); }); test("missing graphify CLI degrades to PROVIDER_UNAVAILABLE", async () => { await expect( - new GraphifyProvider({ root: repo, env: { PATH: binDir } }).search("q"), + new GraphifyProvider({ root: repo, env: { PATH: os.tmpdir() } }).search("q"), ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); }); -describe("Sourcebot adapter (injected fetch + temp config)", () => { +describe("Sourcebot adapter (injected fetch, real v5 auth + shape)", () => { test("registerSource writes a local git connection to config.json", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-sb-")); const configPath = path.join(dir, "config.json"); - const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", configPath }); - await sb.registerSource({ id: "myrepo", path: "/abs/repo" }); + await new SourcebotProvider({ baseUrl: "http://localhost:3000", configPath }).registerSource({ id: "myrepo", path: "/abs/repo" }); const written = JSON.parse(fs.readFileSync(configPath, "utf-8")); expect(written.connections.myrepo).toEqual({ type: "git", url: "file:///abs/repo" }); fs.rmSync(dir, { recursive: true, force: true }); }); - test("search POSTs /api/search and maps files to hits", async () => { - const calls: Array<{ url: string; body: unknown }> = []; + test("search sends Bearer auth and maps the real v5 response to hits", async () => { + const seen: Array<{ url: string; auth: string | null }> = []; const fetchStub = (async (url: string, init: RequestInit) => { - calls.push({ url: String(url), body: JSON.parse(String(init.body)) }); + seen.push({ url: String(url), auth: (init.headers as Record)?.Authorization ?? null }); return new Response( JSON.stringify({ files: [{ fileName: { text: "a.ts" }, chunks: [{ content: "x", matchRanges: [{ start: { lineNumber: 3 } }] }] }] }), { status: 200, headers: { "Content-Type": "application/json" } }, ); }) as unknown as typeof fetch; - const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", fetch: fetchStub }); - const hits = await sb.search("foo", { limit: 5 }); - expect(calls[0].url).toBe("http://localhost:3000/api/search"); - expect((calls[0].body as { isRegexEnabled: boolean }).isRegexEnabled).toBe(true); + const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", apiKey: "sbk_test", fetch: fetchStub }); + const hits = await sb.search("foo"); + expect(seen[0].url).toBe("http://localhost:3000/api/search"); + expect(seen[0].auth).toBe("Bearer sbk_test"); expect(hits).toEqual([{ ref: "a.ts:3", snippet: "x", kind: "file" }]); }); + test("401 (no API key) degrades to PROVIDER_UNAVAILABLE, not PROVIDER_ERROR", async () => { + const fetchStub = (async () => + new Response(JSON.stringify({ errorCode: "NOT_AUTHENTICATED" }), { status: 401 })) as unknown as typeof fetch; + await expect( + new SourcebotProvider({ baseUrl: "http://localhost:3000", fetch: fetchStub }).search("q"), + ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); + }); + test("unreachable server degrades to PROVIDER_UNAVAILABLE", async () => { const fetchStub = (async () => { throw new Error("ECONNREFUSED"); @@ -210,17 +245,21 @@ describe("Sourcebot adapter (injected fetch + temp config)", () => { }); test("registerSource without SOURCEBOT_CONFIG → PROVIDER_UNAVAILABLE", async () => { - const sb = new SourcebotProvider({ baseUrl: "http://localhost:3000", env: {} }); - await expect(sb.registerSource({ id: "r", path: "/x" })).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); + await expect( + new SourcebotProvider({ baseUrl: "http://localhost:3000", env: {} }).registerSource({ id: "r", path: "/x" }), + ).rejects.toMatchObject({ code: "PROVIDER_UNAVAILABLE" }); }); }); -describe("GBrain adapter end-to-end (fake gbrain shim on PATH)", () => { +describe("GBrain adapter (fake gbrain shim)", () => { let binDir: string; let homeDir: string; function env(): NodeJS.ProcessEnv { return { PATH: `${binDir}:${process.env.PATH}`, HOME: homeDir }; } + function writeShim(body: string): void { + fs.writeFileSync(path.join(binDir, "gbrain"), body, { mode: 0o755 }); + } beforeEach(() => { binDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gb-bin-")); homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "ci-gb-home-")); @@ -230,21 +269,28 @@ describe("GBrain adapter end-to-end (fake gbrain shim on PATH)", () => { fs.rmSync(homeDir, { recursive: true, force: true }); }); - test("search scopes to source and parses hits", async () => { - fs.writeFileSync( - path.join(binDir, "gbrain"), - `#!/usr/bin/env bash + test("search parses hits (and sends no --source: gbrain search is global)", async () => { + writeShim(`#!/usr/bin/env bash if [ "$1" = "search" ]; then - if printf '%s ' "$@" | grep -q -- "--source code"; then echo "[0.88] src/x.ts -- match"; else echo "[0.10] wrong -- unscoped"; fi - exit 0 + if printf '%s ' "$@" | grep -q -- "--source"; then echo "[0.0] ERR -- adapter sent phantom --source"; exit 0; fi + echo "[0.88] src/x.ts -- match"; exit 0 fi exit 1 -`, - { mode: 0o755 }, - ); - const hits = await new GbrainProvider().search("where", { env: env(), source: "code" }); - expect(hits).toHaveLength(1); - expect(hits[0].ref).toBe("src/x.ts"); +`); + const hits = await new GbrainProvider().search("where", { env: env() }); + expect(hits).toEqual([{ ref: "src/x.ts", score: 0.88, snippet: "match", kind: "document" }]); + }); + + test("engine-down (pglite WASM) degrades to PROVIDER_UNAVAILABLE, not PROVIDER_ERROR", async () => { + // Reproduces garrytan/gbrain#223: engine fails to init; must degrade cleanly. + writeShim(`#!/usr/bin/env bash +echo "PGLite failed to initialize its WASM runtime." >&2 +echo " Original error: Aborted()." >&2 +exit 1 +`); + await expect(new GbrainProvider().search("q", { env: env() })).rejects.toMatchObject({ + code: "PROVIDER_UNAVAILABLE", + }); }); test("missing CLI degrades to PROVIDER_UNAVAILABLE", async () => { From a524d860a88932723046cc3a0715e55d794f52d9 Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 17:40:50 -0700 Subject: [PATCH 8/9] docs: record real-environment verification and the fixes it drove Update the capability matrix and provider notes to the verified real interfaces (graphify update, Sourcebot v5 Bearer auth) and add a "Verified against real environments" section documenting what the live tests found and fixed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CODE_INTELLIGENCE_PROVIDER_CONTRACT.md | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md b/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md index c5a01cbb0b..9cc962cc4c 100644 --- a/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md +++ b/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md @@ -123,9 +123,9 @@ consent; egress requires a separate explicit step. | Op | GBrain (recommend first) | Sourcebot | Graphify | |----|--------------------------|-----------|----------| -| `register_source` | ✓ `sources add` | ✓ repo added to index config | ✓ index a local dir | -| `refresh` | ✓ `sync --strategy code` | ✓ reindex | ✓ re-parse tree-sitter graph | -| `search` | ✓ `gbrain search` (federated corpora) | ✓ `POST /api/search` (regex, zoekt) | ✓ `graphify query ""` | +| `register_source` | ✓ `sources add` | ✓ local `git` connection in config.json | ✓ `graphify update ` (local, no LLM) | +| `refresh` | ✓ `sync --source` | ✓ auto (config change + reindexIntervalMs) | ✓ `graphify update ` | +| `search` | ✓ `gbrain search` (federated corpora) | ✓ `POST /api/search` + Bearer key (v5) | ✓ `graphify query "" --graph ` | | `status` | ✓ `sources list` + page_count | ~ partial (server liveness) | ~ partial (graph.json present + node count) | | `add` | ✓ `put ` | ✗ declines | ✗ declines | | `delete` | ✓ `delete ` | ✗ declines | ✗ declines | @@ -142,15 +142,20 @@ All three are driven directly from the runtime — no MCP client: - **Sourcebot** (`github.com/sourcebot-dev/sourcebot`, YC Fall 2025): self-hosted whole-repo regex search. `register_source` adds a local `{ "type": "git", "url": "file:///path" }` connection to the server's `config.json` (it re-indexes on - config change); `search` is `POST {baseUrl}/api/search`; `status` is a liveness - probe. Declines `add`/`delete`/`export`. A loopback `baseUrl` keeps content on - the machine (local); a remote one requires egress consent. + config change); `search` is `POST {baseUrl}/api/search`; `status` probes that + endpoint. Declines `add`/`delete`/`export`. **Sourcebot v5 gates `/api/search` + behind auth**, so the adapter sends `Authorization: Bearer `. + A loopback `baseUrl` keeps content on the machine (local); a remote one requires + egress consent. - **Graphify** (`github.com/Graphify-Labs/graphify`, YC-backed): local - tree-sitter code graph via the `graphify` CLI. `graphify ` builds - `graphify-out/graph.json`; `graphify query ""` searches it; `export` reads - the graph JSON. Fully local — nothing leaves the machine. Optional, **install - only with explicit user action** (`pip install graphifyy && graphify install`); - never auto-installed. + tree-sitter code graph via the `graphify` CLI. The adapter uses **`graphify + update `** — the local, no-LLM build (writes `graphify-out/graph.json`); + it deliberately avoids the bare `graphify ` build, which runs an LLM + extraction backend needing an API key + network. `graphify query "" --graph + ` searches it (its `NODE ...`/`EDGE ...` output carries the file at + `src=`/`at=`); `export` reads the graph JSON. Fully local — nothing leaves the + machine. Optional, **install only with explicit user action** (`pip install + graphifyy && graphify install`, needs Python >= 3.10); never auto-installed. **No local-index option is offered** (deliberately excluded — a naive local index degrades result quality; we route to a real provider or to file-only grep, @@ -229,6 +234,33 @@ a later phase precisely so the first slices do not trigger the - **Phase 4: retire bespoke glue.** Once every consumer is on the contract, delete the sync/ingest/cache entrypoints and their tests provider-by-provider. +## Verified against real environments + +The three adapters were tested against the real tools in isolated environments +(parallel agents, one worktree each), not just unit fakes. What that surfaced and +fixed: + +- **Graphify (real install, graphify 0.9.23).** The first cut ran `graphify ` + — which invokes an LLM extraction backend (needs a key + network), breaking the + "local, no egress" promise — and parsed a made-up output format. Fixed to the + real local `graphify update` build and a parser written against the real + `NODE`/`EDGE` output (file at `src=`/`at=`). Also fixed `search` ignoring the + indexed repo (now persists the indexed root) and `options` mislabeling an + installed-but-unindexed provider as unavailable. +- **Sourcebot (live v5 in Docker).** Endpoint, request body, and response parsing + were correct against the real server. But v5 gates `/api/search` behind auth, so + the adapter got HTTP 401; added `Authorization: Bearer` support and made `status` + stop following the login redirect (it was falsely reporting "ready"). +- **GBrain (real gbrain 0.42.56, pglite engine).** The engine was broken on the + host (upstream macOS WASM bug), which exposed two bugs: engine-down failures were + reported as hard `PROVIDER_ERROR` with a raw stack dump instead of a clean + `PROVIDER_UNAVAILABLE` degrade (fixed), and the adapter sent flags the real + `gbrain` CLI does not define (`sync --strategy`, `search --source`) — corrected + to the real surface. + +Live end-to-end index+search-with-results was proven for Graphify and Sourcebot; +GBrain's was blocked only by the host's broken engine, not by adapter code. + ## What this does NOT change Per the GStack 2 canonical contract and CLAUDE.md boundaries: no cloud browsers, From ab7989c6c131efc7225765507849af8a41dd4a3f Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 18:01:17 -0700 Subject: [PATCH 9/9] fix: correct adapters against real running providers (round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran real backends in isolated environments (real Postgres-backed gbrain, live Sourcebot v6.5.0 with anonymous access, real graphify 0.9.23). All three now index + search end-to-end. Fixes: - gbrain: RESTORE `--strategy code` in refresh — round 1 removed it on a --help misread, which silently stopped code from ever being indexed. refresh now runs the verified two-pass (`sync`, then `sync --strategy code --full`). Pinned by a new test so the regression can't return. - sourcebot: an API key is NOT required for local use — anonymous access (FORCE_ENABLE_ANONYMOUS_ACCESS=true) serves /api/search keyless (verified). Key stays optional; only the messaging changed (anonymous-access first, key as fallback) plus a note that a local repo needs remote.origin.url to index. - graphify: correct the docstring — for CODE both `graphify ` and `graphify update` are AST-only (no LLM); the LLM only renames clusters and ingests non-code, which our parser ignores. No LLM mode; local=true is correct. Docs: capability matrix + "Verified against real environments" updated to record all three proven end-to-end and to correct the two first-round mistakes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CODE_INTELLIGENCE_PROVIDER_CONTRACT.md | 87 +++++++++++-------- lib/code-intelligence/gbrain-adapter.ts | 14 +-- lib/code-intelligence/graphify-adapter.ts | 18 ++-- lib/code-intelligence/sourcebot-adapter.ts | 19 ++-- test/code-intelligence.test.ts | 15 ++++ 5 files changed, 102 insertions(+), 51 deletions(-) diff --git a/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md b/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md index 9cc962cc4c..575a875918 100644 --- a/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md +++ b/docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md @@ -123,9 +123,9 @@ consent; egress requires a separate explicit step. | Op | GBrain (recommend first) | Sourcebot | Graphify | |----|--------------------------|-----------|----------| -| `register_source` | ✓ `sources add` | ✓ local `git` connection in config.json | ✓ `graphify update ` (local, no LLM) | -| `refresh` | ✓ `sync --source` | ✓ auto (config change + reindexIntervalMs) | ✓ `graphify update ` | -| `search` | ✓ `gbrain search` (federated corpora) | ✓ `POST /api/search` + Bearer key (v5) | ✓ `graphify query "" --graph ` | +| `register_source` | ✓ `sources add --federated` | ✓ local `git` connection in config.json | ✓ `graphify update ` (local, no LLM) | +| `refresh` | ✓ `sync` + `sync --strategy code --full` | ✓ auto (config change + reindexIntervalMs) | ✓ `graphify update ` | +| `search` | ✓ `gbrain search` (federated corpora) | ✓ `POST /api/search` (keyless w/ anonymous access; Bearer key optional) | ✓ `graphify query "" --graph ` | | `status` | ✓ `sources list` + page_count | ~ partial (server liveness) | ~ partial (graph.json present + node count) | | `add` | ✓ `put ` | ✗ declines | ✗ declines | | `delete` | ✓ `delete ` | ✗ declines | ✗ declines | @@ -140,13 +140,17 @@ All three are driven directly from the runtime — no MCP client: (`sources add`/`sync`). Advertises all seven capabilities. **Recommended first.** - **Sourcebot** (`github.com/sourcebot-dev/sourcebot`, YC Fall 2025): self-hosted - whole-repo regex search. `register_source` adds a local `{ "type": "git", "url": - "file:///path" }` connection to the server's `config.json` (it re-indexes on - config change); `search` is `POST {baseUrl}/api/search`; `status` probes that - endpoint. Declines `add`/`delete`/`export`. **Sourcebot v5 gates `/api/search` - behind auth**, so the adapter sends `Authorization: Bearer `. - A loopback `baseUrl` keeps content on the machine (local); a remote one requires - egress consent. + whole-repo regex search, deployed via Docker Compose (bundled server + Postgres + + Redis; no supported non-Docker path). `register_source` adds a local `{ "type": + "git", "url": "file:///path" }` connection to the server's `config.json` (it + re-indexes on config change; a local repo needs a `remote.origin.url` or it is + skipped); `search` is `POST {baseUrl}/api/search`; `status` probes that endpoint. + Declines `add`/`delete`/`export`. It is a **local** tool — indexed code stays on + your machine — and an **API key is optional**: a local instance with anonymous + access (`FORCE_ENABLE_ANONYMOUS_ACCESS=true`) serves `/api/search` keyless. The + adapter sends `Authorization: Bearer ` only when a key is set. + A loopback `baseUrl` keeps content on the machine (local=true); a remote one + requires egress consent. - **Graphify** (`github.com/Graphify-Labs/graphify`, YC-backed): local tree-sitter code graph via the `graphify` CLI. The adapter uses **`graphify update `** — the local, no-LLM build (writes `graphify-out/graph.json`); @@ -236,30 +240,45 @@ a later phase precisely so the first slices do not trigger the ## Verified against real environments -The three adapters were tested against the real tools in isolated environments -(parallel agents, one worktree each), not just unit fakes. What that surfaced and -fixed: - -- **Graphify (real install, graphify 0.9.23).** The first cut ran `graphify ` - — which invokes an LLM extraction backend (needs a key + network), breaking the - "local, no egress" promise — and parsed a made-up output format. Fixed to the - real local `graphify update` build and a parser written against the real - `NODE`/`EDGE` output (file at `src=`/`at=`). Also fixed `search` ignoring the - indexed repo (now persists the indexed root) and `options` mislabeling an - installed-but-unindexed provider as unavailable. -- **Sourcebot (live v5 in Docker).** Endpoint, request body, and response parsing - were correct against the real server. But v5 gates `/api/search` behind auth, so - the adapter got HTTP 401; added `Authorization: Bearer` support and made `status` - stop following the login redirect (it was falsely reporting "ready"). -- **GBrain (real gbrain 0.42.56, pglite engine).** The engine was broken on the - host (upstream macOS WASM bug), which exposed two bugs: engine-down failures were - reported as hard `PROVIDER_ERROR` with a raw stack dump instead of a clean - `PROVIDER_UNAVAILABLE` degrade (fixed), and the adapter sent flags the real - `gbrain` CLI does not define (`sync --strategy`, `search --source`) — corrected - to the real surface. - -Live end-to-end index+search-with-results was proven for Graphify and Sourcebot; -GBrain's was blocked only by the host's broken engine, not by adapter code. +All three adapters were driven against the real tools in isolated environments +(parallel agents, one worktree each), and all three now index + search a real repo +end-to-end. Two rounds ran, because the first round's fixes included a mistake that +only real execution caught — recorded here honestly. + +- **GBrain — real Postgres+pgvector (Docker), gbrain 0.42.56 — PROVEN.** The + default pglite/WASM engine is broken on macOS (upstream garrytan/gbrain#223), so + the working recipe points gbrain at a real Postgres via `DATABASE_URL`. Real + end-to-end search returned the actual code definition + (`[0.88] src-checksum-ts … export statement computeChecksum`). Real execution + caught a **regression I had introduced**: I removed `--strategy code` from + `refresh` based on a `--help` misread, which silently stopped code from ever + being indexed (only docs were). Restored to the verified two-pass + (`sync`, then `sync --strategy code --full`); `--federated` registration is + load-bearing for global search. Also fixed earlier: engine-down now degrades to + `PROVIDER_UNAVAILABLE` (one-line message) instead of `PROVIDER_ERROR` + a WASM + stack dump. +- **Sourcebot — live v6.5.0 (Docker) — PROVEN keyless.** Endpoint, body, and + response parsing were correct against the real server. Correcting an earlier + wrong conclusion: Sourcebot does **not** require an API key for local use — + enabling anonymous access (`FORCE_ENABLE_ANONYMOUS_ACCESS=true`) serves + `/api/search` keyless, verified with a real hit through the CLI with no key set. + The key stays optional; the only fix was messaging (point users to anonymous + access first, key as fallback) plus a note that a local repo needs a + `remote.origin.url` to be indexed. It is a local tool (code stays on the + machine; a boot telemetry ping unless `SOURCEBOT_TELEMETRY_DISABLED=true`). +- **Graphify — real install, graphify 0.9.23 — PROVEN.** Correcting an earlier + wrong claim of mine: for **code**, `graphify ` and `graphify update ` + produce the identical AST graph with **no LLM call**; the LLM only renames + community clusters and ingests non-code docs, adding zero nodes/edges, and our + parser discards the field it touches. So there is deliberately no LLM mode, and + `local=true` is correct. The adapter uses `graphify update`; real `index`+search + returned correct `file:line` refs. Also fixed: `search` now reads the indexed + repo's graph (persisted root), and `options` reports an installed provider as + available. + +The larger lesson, kept on the record: a `--help` reading or a single agent's +conclusion is not proof — running the real tool is. It reversed two of my +first-round calls (the gbrain flag removal and the graphify LLM claim). ## What this does NOT change diff --git a/lib/code-intelligence/gbrain-adapter.ts b/lib/code-intelligence/gbrain-adapter.ts index fc89589871..48ffb3257a 100644 --- a/lib/code-intelligence/gbrain-adapter.ts +++ b/lib/code-intelligence/gbrain-adapter.ts @@ -90,11 +90,15 @@ export class GbrainProvider implements CodeProvider { async refresh(source: SourceRef, opts: OpOptions = {}): Promise { assertEgressConsent(this, opts); - // `gbrain sync` has no `--strategy` flag (verified against gbrain 0.42.x --help). - this.#assertOk(spawnGbrain(["sync", "--source", source.id], { - baseEnv: opts.env, - timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS, - })); + const timeout = opts.timeout ?? DEFAULT_TIMEOUT_MS; + // Two passes, verified end-to-end against real Postgres-backed gbrain 0.42.56: + // 1. default sync (markdown strategy) — indexes docs. + // 2. `sync --strategy code` — the ACTUAL code-indexing pass. Without it code + // is never indexed (the whole point of a code provider); `code-def` stays + // "not_built" and search only finds incidental doc mentions. `--full` + // forces it past the per-source checkpoint the markdown pass advanced. + this.#assertOk(spawnGbrain(["sync", "--source", source.id], { baseEnv: opts.env, timeout })); + this.#assertOk(spawnGbrain(["sync", "--source", source.id, "--strategy", "code", "--full"], { baseEnv: opts.env, timeout })); return this.status(source, opts); } diff --git a/lib/code-intelligence/graphify-adapter.ts b/lib/code-intelligence/graphify-adapter.ts index 22d58ec991..e331984bc5 100644 --- a/lib/code-intelligence/graphify-adapter.ts +++ b/lib/code-intelligence/graphify-adapter.ts @@ -1,12 +1,18 @@ /** * Graphify adapter — real CLI integration (github.com/Graphify-Labs/graphify). * - * Graphify is a LOCAL tree-sitter knowledge graph. The genuinely local, no-LLM - * build is `graphify update ` — it writes `/graphify-out/graph.json` - * with NO embeddings and NO network (verified against graphify 0.9.23). NOTE: - * the bare `graphify ` build instead runs LLM semantic extraction (a gemini - * backend needing an API key + network), so this adapter deliberately uses - * `graphify update`, which keeps the "local, no egress consent" invariant true. + * Graphify is a LOCAL tree-sitter knowledge graph. For CODE, `graphify ` + * and `graphify update ` produce the SAME AST graph with NO LLM and NO + * network (verified against graphify 0.9.23 — both emit `AST extraction on N + * code files`, all node origins `ast`). The LLM backend (openai/gemini) is only + * used to RENAME community clusters (`graphify label` / `cluster-only`) and to + * ingest non-code docs (`graphify add`); it adds zero nodes/edges, and our parser + * discards the `community=` field it touches — so an LLM mode would send code + * off-machine for no change in search output, and is intentionally not offered. + * + * This adapter uses `graphify update ` (writes `/graphify-out/graph.json` + * and does clustering in one shot) and stays fully local — nothing leaves the + * machine, so `local = true` and no egress consent is needed. * * Query is `graphify query "" --graph /graphify-out/graph.json`; the * `--graph` flag points at the built graph so search never depends on cwd. diff --git a/lib/code-intelligence/sourcebot-adapter.ts b/lib/code-intelligence/sourcebot-adapter.ts index d7c6c5e244..060806f1a3 100644 --- a/lib/code-intelligence/sourcebot-adapter.ts +++ b/lib/code-intelligence/sourcebot-adapter.ts @@ -46,9 +46,11 @@ export interface SourcebotOptions { /** Path to the server's config.json (for register_source). Defaults to SOURCEBOT_CONFIG. */ configPath?: string; /** - * API key for the Sourcebot REST API. Defaults to SOURCEBOT_API_KEY. Sourcebot - * v5 gates `/api/search` behind auth (`Authorization: Bearer `); without - * it, `search` gets HTTP 401. Generate one in Settings -> API Keys. + * OPTIONAL API key for the Sourcebot REST API. Defaults to SOURCEBOT_API_KEY. + * A local instance with anonymous access enabled + * (`FORCE_ENABLE_ANONYMOUS_ACCESS=true`) serves `/api/search` with NO key — + * verified keyless against a real Sourcebot v6.5.0. Only set a key when your + * instance is login-gated; it is sent as `Authorization: Bearer `. */ apiKey?: string; /** Injectable fetch for tests. */ @@ -93,7 +95,12 @@ export class SourcebotProvider implements CodeProvider { return this.capabilities.has(capability); } - /** Add the repo as a local `git` connection in Sourcebot's config.json. */ + /** + * Add the repo as a local `git` connection in Sourcebot's config.json. + * NOTE: Sourcebot silently SKIPS a local repo that has no `remote.origin.url` + * (logs "Skipping - remote.origin.url not found"); a freshly `git init`'d + * repo must set an origin before it will index. + */ async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise { assertEgressConsent(this, opts); // no-op when the server is loopback (local) if (!this.#configPath) { @@ -148,7 +155,7 @@ export class SourcebotProvider implements CodeProvider { opts.timeout ?? DEFAULT_TIMEOUT_MS, ); if (res.status === 401 || res.status === 403) { - return { id: "*", state: "unknown", partial: true, detail: "reachable but not authenticated (set SOURCEBOT_API_KEY)" }; + return { id: "*", state: "unknown", partial: true, detail: "reachable but login-gated (enable anonymous access with FORCE_ENABLE_ANONYMOUS_ACCESS=true for local use, or set SOURCEBOT_API_KEY)" }; } return { id: "*", state: res.ok ? "ready" : "unknown", partial: true, detail: `HTTP ${res.status}` }; } catch { @@ -168,7 +175,7 @@ export class SourcebotProvider implements CodeProvider { throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot unreachable at ${this.#baseUrl}: ${(err as Error).message}`, this.id); } if (res.status === 401 || res.status === 403) { - throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot requires authentication; set SOURCEBOT_API_KEY (HTTP ${res.status})`, this.id); + throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot is login-gated (HTTP ${res.status}); enable anonymous access (FORCE_ENABLE_ANONYMOUS_ACCESS=true) for local use, or set SOURCEBOT_API_KEY`, this.id); } if (!res.ok) throw new CodeProviderError("PROVIDER_ERROR", `Sourcebot ${path} returned HTTP ${res.status}`, this.id); try { diff --git a/test/code-intelligence.test.ts b/test/code-intelligence.test.ts index 0a4be6748d..62060e51e2 100644 --- a/test/code-intelligence.test.ts +++ b/test/code-intelligence.test.ts @@ -281,6 +281,21 @@ exit 1 expect(hits).toEqual([{ ref: "src/x.ts", score: 0.88, snippet: "match", kind: "document" }]); }); + test("refresh runs the code-indexing pass (`sync --strategy code --full`)", async () => { + // Real gbrain only indexes code when `sync --strategy code` runs; without it + // code-def stays not_built. Pin that the adapter issues that pass. + const marker = path.join(homeDir, "sync-calls.log"); + writeShim(`#!/usr/bin/env bash +if [ "$1" = "sync" ]; then printf '%s\\n' "$*" >> "${marker}"; exit 0; fi +if [ "$1" = "sources" ]; then echo '{"sources":[{"id":"code","local_path":"/r","page_count":1}]}'; exit 0; fi +exit 1 +`); + await new GbrainProvider().refresh({ id: "code" }, { env: env(), consented: true }); + const log = fs.readFileSync(marker, "utf-8"); + expect(log).toContain("--strategy code"); + expect(log).toContain("--full"); + }); + test("engine-down (pglite WASM) degrades to PROVIDER_UNAVAILABLE, not PROVIDER_ERROR", async () => { // Reproduces garrytan/gbrain#223: engine fails to init; must degrade cleanly. writeShim(`#!/usr/bin/env bash