From 1c62fcf2d86663f7e3eede5d6cd8fc1cc18918ef Mon Sep 17 00:00:00 2001 From: betegon Date: Tue, 11 Aug 2026 09:23:14 +0200 Subject: [PATCH] feat(init): report repository scan omissions --- packages/cli/src/lib/init/tools/list-dir.ts | 69 ++++++++++++--- packages/cli/src/lib/init/tools/read-files.ts | 87 ++++++++++++++----- packages/cli/src/lib/init/types.ts | 35 ++++++++ packages/cli/src/lib/init/wizard-runner.ts | 9 +- packages/cli/src/lib/init/workflow-inputs.ts | 13 ++- .../lib/init/tools/filesystem-tools.test.ts | 29 ++++++- .../cli/test/lib/init/tools/list-dir.test.ts | 39 ++++++++- .../cli/test/lib/init/wizard-runner.test.ts | 17 +++- 8 files changed, 256 insertions(+), 42 deletions(-) diff --git a/packages/cli/src/lib/init/tools/list-dir.ts b/packages/cli/src/lib/init/tools/list-dir.ts index 85f00f5d59..a79ec475ef 100644 --- a/packages/cli/src/lib/init/tools/list-dir.ts +++ b/packages/cli/src/lib/init/tools/list-dir.ts @@ -1,7 +1,12 @@ import fs from "node:fs"; import path from "node:path"; import { DEFAULT_SKIP_DIRS, normalizePath } from "../../scan/index.js"; -import type { DirEntry, ListDirPayload, ToolResult } from "../types.js"; +import type { + DirEntry, + ListDirPayload, + OmittedDirectoryReason, + ToolResult, +} from "../types.js"; import { safePath } from "./shared.js"; import type { InitToolDefinition } from "./types.js"; @@ -35,11 +40,22 @@ export async function listDir(payload: ListDirPayload): Promise { entries: [], maxDepth, maxEntries, + omittedDirectories: [], recursive, + truncated: false, }; await walkDirectory(targetPath, 0, state); - return { ok: true, data: { entries: state.entries } }; + return { + ok: true, + data: { + entries: state.entries, + metadata: { + truncated: state.truncated, + omittedDirectories: state.omittedDirectories, + }, + }, + }; } type WalkState = { @@ -48,7 +64,12 @@ type WalkState = { entries: DirEntry[]; maxDepth: number; maxEntries: number; + omittedDirectories: Array<{ + path: string; + reason: OmittedDirectoryReason; + }>; recursive: boolean; + truncated: boolean; }; async function readDirEntries(dir: string): Promise { @@ -59,14 +80,27 @@ async function readDirEntries(dir: string): Promise { } } -function shouldRecurseInto(entry: fs.Dirent, state: WalkState): boolean { - return ( - state.recursive && - entry.isDirectory() && - !entry.isSymbolicLink() && - !entry.name.startsWith(".") && - !INIT_SKIP_DIRS.has(entry.name) - ); +function omissionReason( + entry: fs.Dirent, + state: WalkState, + depth: number +): OmittedDirectoryReason | undefined { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + return; + } + if (!state.recursive) { + return "non-recursive"; + } + if (entry.name.startsWith(".")) { + return "hidden"; + } + if (INIT_SKIP_DIRS.has(entry.name)) { + return "excluded"; + } + if (depth >= state.maxDepth) { + return "max-depth"; + } + return; } /** @@ -105,12 +139,17 @@ async function walkDirectory( depth: number, state: WalkState ): Promise { - if (depth > state.maxDepth || state.entries.length >= state.maxEntries) { + if (depth > state.maxDepth) { + return; + } + if (state.entries.length >= state.maxEntries) { + state.truncated = true; return; } for (const entry of await readDirEntries(dir)) { if (state.entries.length >= state.maxEntries) { + state.truncated = true; return; } const nextEntry = toDirEntry(state, dir, entry); @@ -118,7 +157,13 @@ async function walkDirectory( continue; } state.entries.push(nextEntry); - if (shouldRecurseInto(entry, state)) { + const omittedBecause = omissionReason(entry, state, depth); + if (omittedBecause) { + state.omittedDirectories.push({ + path: nextEntry.path, + reason: omittedBecause, + }); + } else if (entry.isDirectory() && !entry.isSymbolicLink()) { await walkDirectory(dir + NATIVE_SEP + entry.name, depth + 1, state); } } diff --git a/packages/cli/src/lib/init/tools/read-files.ts b/packages/cli/src/lib/init/tools/read-files.ts index a331f0cf50..0822ca977d 100644 --- a/packages/cli/src/lib/init/tools/read-files.ts +++ b/packages/cli/src/lib/init/tools/read-files.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import { MAX_FILE_BYTES } from "../constants.js"; -import type { ReadFilesPayload, ToolResult } from "../types.js"; +import type { FileReadResult, ReadFilesPayload, ToolResult } from "../types.js"; import { safePath } from "./shared.js"; import type { InitToolDefinition } from "./types.js"; @@ -15,24 +15,31 @@ export async function readFiles( const maxBytes = payload.params.maxBytes ?? MAX_FILE_BYTES; const results = await Promise.all( payload.params.paths.map(async (filePath) => { - const content = await readSingleFile(payload.cwd, filePath, maxBytes); - return [filePath, content] as const; + const result = await readSingleFile(payload.cwd, filePath, maxBytes); + return [filePath, result] as const; }) ); const files: Record = {}; - for (const [filePath, content] of results) { - files[filePath] = content; + const readResults: Record = {}; + for (const [filePath, result] of results) { + files[filePath] = result.content; + readResults[filePath] = result.metadata; } - return { ok: true, data: { files } }; + return { ok: true, data: { files, readResults } }; } +type SingleFileRead = { + content: string | null; + metadata: FileReadResult; +}; + async function readSingleFile( cwd: string, filePath: string, maxBytes: number -): Promise { +): Promise { try { const absPath = safePath(cwd, filePath); const stat = await fs.promises.stat(absPath); @@ -40,22 +47,60 @@ async function readSingleFile( // `open("r")` block indefinitely on a FIFO waiting for a writer. // `stat` follows symlinks, so symlink → FIFO is caught too. if (!stat.isFile()) { - return null; - } - if (stat.size <= maxBytes) { - return await fs.promises.readFile(absPath, "utf-8"); + return { + content: null, + metadata: { status: "skipped", reason: "not-regular-file" }, + }; } + const buffer = + stat.size <= maxBytes + ? await fs.promises.readFile(absPath) + : await readFilePrefix(absPath, maxBytes); - const handle = await fs.promises.open(absPath, "r"); - try { - const buffer = Buffer.alloc(maxBytes); - await handle.read(buffer, 0, maxBytes, 0); - return buffer.toString("utf-8"); - } finally { - await handle.close(); - } - } catch { - return null; + return { + content: buffer.toString("utf-8"), + metadata: { + status: stat.size > maxBytes ? "truncated" : "read", + bytesRead: buffer.byteLength, + totalBytes: stat.size, + }, + }; + } catch (error) { + return { + content: null, + metadata: { status: "error", reason: readFailureReason(error) }, + }; + } +} + +function readFailureReason(error: unknown): FileReadResult["reason"] { + const code = + typeof error === "object" && error !== null && "code" in error + ? String(error.code) + : undefined; + if (code === "ENOENT") { + return "not-found"; + } + if (code === "EACCES" || code === "EPERM") { + return "permission-denied"; + } + if (error instanceof Error && error.message.includes("outside project")) { + return "outside-project"; + } + return "read-failed"; +} + +async function readFilePrefix( + absPath: string, + maxBytes: number +): Promise { + const handle = await fs.promises.open(absPath, "r"); + try { + const buffer = Buffer.alloc(maxBytes); + const { bytesRead } = await handle.read(buffer, 0, maxBytes, 0); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close(); } } diff --git a/packages/cli/src/lib/init/types.ts b/packages/cli/src/lib/init/types.ts index 7ea6a87219..f96e241556 100644 --- a/packages/cli/src/lib/init/types.ts +++ b/packages/cli/src/lib/init/types.ts @@ -4,6 +4,41 @@ export type DirEntry = { type: "file" | "directory"; }; +export type OmittedDirectoryReason = + | "excluded" + | "hidden" + | "max-depth" + | "non-recursive"; + +export type OmittedDirectory = { + path: string; + reason: OmittedDirectoryReason; +}; + +export type DirectoryScanMetadata = { + /** True when maxEntries prevented the scanner from inspecting more entries. */ + truncated: boolean; + /** Directories present in the listing whose contents were not inspected. */ + omittedDirectories: OmittedDirectory[]; +}; + +export type DirectoryScanResult = { + entries: DirEntry[]; + metadata: DirectoryScanMetadata; +}; + +export type FileReadResult = { + status: "read" | "truncated" | "skipped" | "error"; + bytesRead?: number; + totalBytes?: number; + reason?: + | "not-found" + | "not-regular-file" + | "outside-project" + | "permission-denied" + | "read-failed"; +}; + export type ExistingProjectData = { orgSlug: string; projectSlug: string; diff --git a/packages/cli/src/lib/init/wizard-runner.ts b/packages/cli/src/lib/init/wizard-runner.ts index 5ef492164c..3bd6816837 100644 --- a/packages/cli/src/lib/init/wizard-runner.ts +++ b/packages/cli/src/lib/init/wizard-runner.ts @@ -950,10 +950,16 @@ export async function runWizard(initialOptions: WizardOptions): Promise { let run: Awaited>; let result: WorkflowRunResult; try { - const [dirListing, existingSentry] = await Promise.all([ + const [directoryScan, existingSentry] = await Promise.all([ precomputeDirListing(directory), precomputeSentryDetection(directory).catch(() => null), ]); + const { entries: dirListing, metadata: dirListingMetadata } = directoryScan; + setTag("wizard.scan.truncated", dirListingMetadata.truncated); + setTag( + "wizard.scan.omitted_directory_count", + dirListingMetadata.omittedDirectories.length + ); const fileCache = await preReadCommonFiles(directory, dirListing); ui.setIntroMode?.(false); spin.message("Connecting to wizard..."); @@ -981,6 +987,7 @@ export async function runWizard(initialOptions: WizardOptions): Promise { }, initialState: { dirListing, + dirListingMetadata, fileCache, existingSentry: existingSentry?.data, knownPlatform: context.existingProject?.platform, diff --git a/packages/cli/src/lib/init/workflow-inputs.ts b/packages/cli/src/lib/init/workflow-inputs.ts index b2cff4c6f3..40f16589ba 100644 --- a/packages/cli/src/lib/init/workflow-inputs.ts +++ b/packages/cli/src/lib/init/workflow-inputs.ts @@ -4,7 +4,7 @@ import { isLikelyBinary } from "../scan/index.js"; import { MAX_FILE_BYTES } from "./constants.js"; import { detectSentry } from "./tools/detect-sentry.js"; import { listDir } from "./tools/list-dir.js"; -import type { DirEntry } from "./types.js"; +import type { DirEntry, DirectoryScanResult } from "./types.js"; /** * Common config files that multiple init steps frequently inspect. @@ -87,14 +87,21 @@ const MAX_PREREAD_TOTAL_BYTES = 512 * 1024; */ export async function precomputeDirListing( directory: string -): Promise { +): Promise { const result = await listDir({ type: "tool", operation: "list-dir", cwd: directory, params: { path: ".", recursive: true, maxDepth: 3, maxEntries: 500 }, }); - return (result.data as { entries?: DirEntry[] } | undefined)?.entries ?? []; + const data = result.data as DirectoryScanResult | undefined; + return { + entries: data?.entries ?? [], + metadata: data?.metadata ?? { + truncated: false, + omittedDirectories: [], + }, + }; } /** diff --git a/packages/cli/test/lib/init/tools/filesystem-tools.test.ts b/packages/cli/test/lib/init/tools/filesystem-tools.test.ts index 5a7d8f18b7..eba414474d 100644 --- a/packages/cli/test/lib/init/tools/filesystem-tools.test.ts +++ b/packages/cli/test/lib/init/tools/filesystem-tools.test.ts @@ -72,7 +72,9 @@ describe("filesystem tools", () => { expect(result.ok).toBe(true); expect(entries.map((entry) => entry.path)).toContain("src/app.ts"); - expect(precomputed.map((entry) => entry.path)).toContain("src/app.ts"); + expect(precomputed.entries.map((entry) => entry.path)).toContain( + "src/app.ts" + ); }); test("reads files and checks existence in batches", async () => { @@ -99,6 +101,10 @@ describe("filesystem tools", () => { expect((readResult.data as any).files["exists.txt"]).toBe("hello"); expect((readResult.data as any).files["missing.txt"]).toBeNull(); + expect((readResult.data as any).readResults).toEqual({ + "exists.txt": { status: "read", bytesRead: 5, totalBytes: 5 }, + "missing.txt": { status: "error", reason: "not-found" }, + }); expect((existsResult.data as any).exists["exists.txt"]).toBe(true); expect((existsResult.data as any).exists["missing.txt"]).toBe(false); }); @@ -123,6 +129,27 @@ describe("filesystem tools", () => { ); }); + test("reports when a file read is truncated", async () => { + fs.writeFileSync(path.join(testDir, "large.txt"), "1234567890"); + + const result = await executeTool( + { + type: "tool", + operation: "read-files", + cwd: testDir, + params: { paths: ["large.txt"], maxBytes: 4 }, + }, + makeContext(testDir) + ); + + expect((result.data as any).files["large.txt"]).toBe("1234"); + expect((result.data as any).readResults["large.txt"]).toEqual({ + status: "truncated", + bytesRead: 4, + totalBytes: 10, + }); + }); + test("applies patchsets and injects auth tokens into env files", async () => { const result = await executeTool( { diff --git a/packages/cli/test/lib/init/tools/list-dir.test.ts b/packages/cli/test/lib/init/tools/list-dir.test.ts index 86b11bdf11..e59bda84fc 100644 --- a/packages/cli/test/lib/init/tools/list-dir.test.ts +++ b/packages/cli/test/lib/init/tools/list-dir.test.ts @@ -11,6 +11,7 @@ import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { listDir } from "../../../../src/lib/init/tools/list-dir.js"; import type { DirEntry, + DirectoryScanMetadata, ListDirPayload, } from "../../../../src/lib/init/types.js"; @@ -33,6 +34,15 @@ function entriesOf(result: Awaited>): DirEntry[] { return (result.data as { entries: DirEntry[] }).entries; } +function metadataOf( + result: Awaited> +): DirectoryScanMetadata { + if (!result.ok) { + throw new Error(`expected listDir to succeed, got: ${result.error}`); + } + return (result.data as { metadata: DirectoryScanMetadata }).metadata; +} + describe("listDir", () => { let testDir: string; @@ -241,10 +251,35 @@ describe("listDir", () => { writeFileSync(join(testDir, `f${i.toString().padStart(2, "0")}.ts`), ""); } - const entries = entriesOf( - await listDir(makePayload(testDir, { path: ".", maxEntries: 5 })) + const result = await listDir( + makePayload(testDir, { path: ".", maxEntries: 5 }) ); + const entries = entriesOf(result); expect(entries).toHaveLength(5); + expect(metadataOf(result).truncated).toBe(true); + }); + + test("reports why directory contents were omitted", async () => { + mkdirSync(join(testDir, ".private")); + mkdirSync(join(testDir, "node_modules")); + mkdirSync(join(testDir, "src")); + + const result = await listDir( + makePayload(testDir, { + path: ".", + recursive: true, + maxDepth: 0, + }) + ); + + expect(metadataOf(result)).toEqual({ + truncated: false, + omittedDirectories: expect.arrayContaining([ + { path: ".private", reason: "hidden" }, + { path: "node_modules", reason: "excluded" }, + { path: "src", reason: "max-depth" }, + ]), + }); }); test("throws on sandbox escape via `..`", async () => { diff --git a/packages/cli/test/lib/init/wizard-runner.test.ts b/packages/cli/test/lib/init/wizard-runner.test.ts index 690848f6c3..16d604d5a0 100644 --- a/packages/cli/test/lib/init/wizard-runner.test.ts +++ b/packages/cli/test/lib/init/wizard-runner.test.ts @@ -211,7 +211,10 @@ beforeEach(() => { }); precomputeDirListingSpy = vi .spyOn(workflowInputs, "precomputeDirListing") - .mockResolvedValue([]); + .mockResolvedValue({ + entries: [], + metadata: { truncated: false, omittedDirectories: [] }, + }); preReadCommonFilesSpy = vi .spyOn(workflowInputs, "preReadCommonFiles") .mockResolvedValue({}); @@ -841,7 +844,16 @@ describe("runWizard", () => { const fileCache = { "package.json": '{"name":"app"}' }; const detectedSentry = { status: "none" as const, signals: [] }; - precomputeDirListingSpy.mockResolvedValue(dirListing); + const dirListingMetadata = { + truncated: false, + omittedDirectories: [ + { path: "node_modules", reason: "excluded" as const }, + ], + }; + precomputeDirListingSpy.mockResolvedValue({ + entries: dirListing, + metadata: dirListingMetadata, + }); preReadCommonFilesSpy.mockResolvedValue(fileCache); precomputeSentryDetectionSpy.mockResolvedValue({ ok: true, @@ -867,6 +879,7 @@ describe("runWizard", () => { expect(args.inputData).not.toHaveProperty("fileCache"); expect(args.inputData).not.toHaveProperty("existingSentry"); expect(args.initialState?.dirListing).toEqual(dirListing); + expect(args.initialState?.dirListingMetadata).toEqual(dirListingMetadata); expect(args.initialState?.fileCache).toEqual(fileCache); expect(args.initialState?.existingSentry).toEqual(detectedSentry); });